From 05fb1fa8ccef41b38bb06ea463cdc2dcbd96e106 Mon Sep 17 00:00:00 2001 From: OuYun Date: Tue, 3 Mar 2020 10:48:55 +0800 Subject: [PATCH 001/181] add option to identify node category --- script/template/options.json | 8 ++++++++ src/bigbang/entry.cpp | 11 +++++++++-- src/xengine/docker/config.cpp | 26 ++++++++++++++++++++++---- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/script/template/options.json b/script/template/options.json index ff040107..0d380ad6 100755 --- a/script/template/options.json +++ b/script/template/options.json @@ -8,6 +8,14 @@ "format": "-testnet", "desc": "Use the test network" }, + { + "name": "nNodeCat", + "type": "int", + "opt": "nodecat", + "default": 0, + "format": "-nodecat=", + "desc": "Identify node category: 0 - BBCNODE; 1 - FORKNODE; 2 - DPOSNODE, 0 by default" + }, { "name": "fWallet", "type": "bool", diff --git a/src/bigbang/entry.cpp b/src/bigbang/entry.cpp index 2813547e..d072f898 100644 --- a/src/bigbang/entry.cpp +++ b/src/bigbang/entry.cpp @@ -91,6 +91,13 @@ bool CBbEntry::Initialize(int argc, char* argv[]) return false; } + // check node category + if (config.GetConfig()->nNodeCat < 0 || config.GetConfig()->nNodeCat > 2) + { + cerr << "Node Category should be 0(BBCNODE), 1(FORKNODE) or 2(DPOSNODE) but it is " << config.GetConfig()->nNodeCat << "\n"; + return false; + } + // purge if (config.GetConfig()->fPurge) { @@ -196,12 +203,12 @@ bool CBbEntry::Initialize(int argc, char* argv[]) if (config.GetConfig()->fTestNet) { HEIGHT_HASH_MULTI_SIGNER = HEIGHT_HASH_MULTI_SIGNER_TESTNET; - HEIGHT_HASH_TX_DATA = HEIGHT_HASH_TX_DATA_TESTNET; + HEIGHT_HASH_TX_DATA = HEIGHT_HASH_TX_DATA_TESTNET; } else { HEIGHT_HASH_MULTI_SIGNER = HEIGHT_HASH_MULTI_SIGNER_MAINNET; - HEIGHT_HASH_TX_DATA = HEIGHT_HASH_TX_DATA_MAINNET; + HEIGHT_HASH_TX_DATA = HEIGHT_HASH_TX_DATA_MAINNET; } // modules diff --git a/src/xengine/docker/config.cpp b/src/xengine/docker/config.cpp index 70dc0515..40d3fa33 100644 --- a/src/xengine/docker/config.cpp +++ b/src/xengine/docker/config.cpp @@ -45,19 +45,32 @@ bool CConfig::Load(int argc, char* argv[], const fs::path& pathDefault, const st { vector vecIgnoreCmd; - defaultDesc.add_options()("cmd", po::value>(&vecCommand))("help", po::value(&fHelp)->default_value(false))("daemon", po::value(&fDaemon)->default_value(false))("debug", po::value(&fDebug)->default_value(false))("datadir", po::value(&strRoot)->default_value(pathDefault.string()))("conf", po::value(&strConfig)->default_value(strConfile))("ignore", po::value>(&vecIgnoreCmd)); + defaultDesc.add_options() + ("cmd", po::value>(&vecCommand)) + ("help", po::value(&fHelp)->default_value(false)) + ("daemon", po::value(&fDaemon)->default_value(false)) + ("debug", po::value(&fDebug)->default_value(false)) + ("datadir", po::value(&strRoot)->default_value(pathDefault.string())) + ("conf", po::value(&strConfig)->default_value(strConfile)) + ("ignore", po::value>(&vecIgnoreCmd)); po::positional_options_description defaultPosDesc; defaultPosDesc.add("cmd", -1).add("ignore", ignoreCmd); - auto parser = po::command_line_parser(argc, argv).options(defaultDesc).style(defaultCmdStyle).extra_parser(CConfig::ExtraParser).positional(defaultPosDesc); + auto parser = po::command_line_parser(argc, argv) + .options(defaultDesc) + .style(defaultCmdStyle) + .extra_parser(CConfig::ExtraParser) + .positional(defaultPosDesc); po::store(parser.run(), vm); - po::notify(vm); + pathRoot = strRoot; pathConfile = strConfig; if (!STD_DEBUG) + { STD_DEBUG = fDebug; + } if (fHelp) { @@ -72,7 +85,12 @@ bool CConfig::Load(int argc, char* argv[], const fs::path& pathDefault, const st vector confToken; if (TokenizeConfile(pathConfile.string().c_str(), confToken)) { - po::store(po::command_line_parser(confToken).options(defaultDesc).style(defaultCmdStyle).extra_parser(CConfig::ExtraParser).allow_unregistered().run(), vm); + po::store(po::command_line_parser(confToken) + .options(defaultDesc) + .style(defaultCmdStyle) + .extra_parser(CConfig::ExtraParser) + .allow_unregistered() + .run(), vm); po::notify(vm); } } From dae0a2323503c001725a48077b7bf000333c73a4 Mon Sep 17 00:00:00 2001 From: OuYun Date: Tue, 3 Mar 2020 16:10:21 +0800 Subject: [PATCH 002/181] alter name of option to adapt for naming rule of generator --- CMakeLists.txt | 2 +- script/template/options.json | 16 ++++++++-------- src/bigbang/entry.cpp | 16 ++++++++-------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7712701d..0ed8c94c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,7 +38,7 @@ elseif(UNIX) set(CMAKE_CXX_EXTENSIONS OFF) else() - message(FATAL_ERROR "Unsupported plantform") + message(FATAL_ERROR "Unsupported platform") endif() # build type diff --git a/script/template/options.json b/script/template/options.json index 0d380ad6..f41f5472 100755 --- a/script/template/options.json +++ b/script/template/options.json @@ -8,14 +8,6 @@ "format": "-testnet", "desc": "Use the test network" }, - { - "name": "nNodeCat", - "type": "int", - "opt": "nodecat", - "default": 0, - "format": "-nodecat=", - "desc": "Identify node category: 0 - BBCNODE; 1 - FORKNODE; 2 - DPOSNODE, 0 by default" - }, { "name": "fWallet", "type": "bool", @@ -64,6 +56,14 @@ "format": "-blocknotify", "desc": "Execute command when the best block changes (%s in cmd is replaced by block hash)" }, + { + "name": "nCatOfNode", + "type": "int", + "opt": "catofnode", + "default": 0, + "format": "-catofnode=", + "desc": "Identify node category: 0 - BBCNODE; 1 - FORKNODE; 2 - DPOSNODE, 0 by default" + }, { "name": "nLogFileSize", "type": "int", diff --git a/src/bigbang/entry.cpp b/src/bigbang/entry.cpp index d072f898..b05e4592 100644 --- a/src/bigbang/entry.cpp +++ b/src/bigbang/entry.cpp @@ -91,10 +91,16 @@ bool CBbEntry::Initialize(int argc, char* argv[]) return false; } + // list config if in debug mode + if (config.GetConfig()->fDebug) + { + config.ListConfig(); + } + // check node category - if (config.GetConfig()->nNodeCat < 0 || config.GetConfig()->nNodeCat > 2) + if (config.GetConfig()->nCatOfNode < 0 || config.GetConfig()->nCatOfNode > 2) { - cerr << "Node Category should be 0(BBCNODE), 1(FORKNODE) or 2(DPOSNODE) but it is " << config.GetConfig()->nNodeCat << "\n"; + cerr << "Node Category should be 0(BBCNODE), 1(FORKNODE) or 2(DPOSNODE) but it is " << config.GetConfig()->nCatOfNode << "\n"; return false; } @@ -105,12 +111,6 @@ bool CBbEntry::Initialize(int argc, char* argv[]) return false; } - // list config if in debug mode - if (config.GetConfig()->fDebug) - { - config.ListConfig(); - } - // check log size if (config.GetConfig()->nLogFileSize < 1 || config.GetConfig()->nLogFileSize > 2048) { From fb48e294f4e0553151532936f81bbcb3935099c7 Mon Sep 17 00:00:00 2001 From: OuYun Date: Wed, 4 Mar 2020 16:56:37 +0800 Subject: [PATCH 003/181] add paho mqtt c++ library --- src/CMakeLists.txt | 1 + src/bigbang/CMakeLists.txt | 1 + src/bigbang/cmqcluster.cpp | 11 + src/bigbang/cmqcluster.h | 17 + src/mqtt/CHANGELOG.md | 46 ++ src/mqtt/CMakeLists.txt | 101 +++ src/mqtt/CODING_STYLE.md | 30 + src/mqtt/CONTRIBUTING.md | 47 ++ src/mqtt/README.md | 331 +++++++++ src/mqtt/about.html | 28 + src/mqtt/buildtst.sh | 66 ++ src/mqtt/cmake/CMakeLists.txt | 22 + src/mqtt/cmake/FindPahoMqttC.cmake | 31 + src/mqtt/cmake/PahoMqttCppConfig.cmake.in | 11 + src/mqtt/devenv.sh | 18 + src/mqtt/dist/Makefile | 11 + src/mqtt/dist/paho-cpp.spec | 64 ++ src/mqtt/edl-v10 | 15 + src/mqtt/epl-v10 | 70 ++ src/mqtt/install_catch2.sh | 22 + src/mqtt/install_paho_mqtt_c.sh | 16 + src/mqtt/m4/README | 3 + src/mqtt/m4/m4_ax_cxx_compile_stdcxx.m4 | 562 ++++++++++++++ src/mqtt/m4/m4_ax_cxx_compile_stdcxx_11.m4 | 39 + src/mqtt/notice.html | 108 +++ src/mqtt/src/CMakeLists.txt | 146 ++++ src/mqtt/src/async_client.cpp | 822 +++++++++++++++++++++ src/mqtt/src/client.cpp | 148 ++++ src/mqtt/src/connect_options.cpp | 209 ++++++ src/mqtt/src/disconnect_options.cpp | 67 ++ src/mqtt/src/iclient_persistence.cpp | 151 ++++ src/mqtt/src/message.cpp | 122 +++ src/mqtt/src/mqtt/async_client.h | 710 ++++++++++++++++++ src/mqtt/src/mqtt/buffer_ref.h | 305 ++++++++ src/mqtt/src/mqtt/buffer_view.h | 119 +++ src/mqtt/src/mqtt/callback.h | 91 +++ src/mqtt/src/mqtt/client.h | 377 ++++++++++ src/mqtt/src/mqtt/connect_options.h | 448 +++++++++++ src/mqtt/src/mqtt/delivery_token.h | 135 ++++ src/mqtt/src/mqtt/disconnect_options.h | 176 +++++ src/mqtt/src/mqtt/exception.h | 225 ++++++ src/mqtt/src/mqtt/iaction_listener.h | 84 +++ src/mqtt/src/mqtt/iasync_client.h | 419 +++++++++++ src/mqtt/src/mqtt/iclient_persistence.h | 135 ++++ src/mqtt/src/mqtt/message.h | 409 ++++++++++ src/mqtt/src/mqtt/properties.h | 389 ++++++++++ src/mqtt/src/mqtt/response_options.h | 121 +++ src/mqtt/src/mqtt/server_response.h | 202 +++++ src/mqtt/src/mqtt/ssl_options.h | 287 +++++++ src/mqtt/src/mqtt/string_collection.h | 241 ++++++ src/mqtt/src/mqtt/subscribe_options.h | 170 +++++ src/mqtt/src/mqtt/thread_queue.h | 332 +++++++++ src/mqtt/src/mqtt/token.h | 504 +++++++++++++ src/mqtt/src/mqtt/topic.h | 180 +++++ src/mqtt/src/mqtt/types.h | 184 +++++ src/mqtt/src/mqtt/will_options.h | 258 +++++++ src/mqtt/src/properties.cpp | 183 +++++ src/mqtt/src/response_options.cpp | 91 +++ src/mqtt/src/ssl_options.cpp | 146 ++++ src/mqtt/src/string_collection.cpp | 103 +++ src/mqtt/src/subscribe_options.cpp | 36 + src/mqtt/src/token.cpp | 297 ++++++++ src/mqtt/src/topic.cpp | 57 ++ src/mqtt/src/will_options.cpp | 133 ++++ 64 files changed, 10883 insertions(+) create mode 100644 src/bigbang/cmqcluster.cpp create mode 100644 src/bigbang/cmqcluster.h create mode 100644 src/mqtt/CHANGELOG.md create mode 100644 src/mqtt/CMakeLists.txt create mode 100644 src/mqtt/CODING_STYLE.md create mode 100644 src/mqtt/CONTRIBUTING.md create mode 100644 src/mqtt/README.md create mode 100644 src/mqtt/about.html create mode 100755 src/mqtt/buildtst.sh create mode 100644 src/mqtt/cmake/CMakeLists.txt create mode 100644 src/mqtt/cmake/FindPahoMqttC.cmake create mode 100644 src/mqtt/cmake/PahoMqttCppConfig.cmake.in create mode 100644 src/mqtt/devenv.sh create mode 100644 src/mqtt/dist/Makefile create mode 100644 src/mqtt/dist/paho-cpp.spec create mode 100644 src/mqtt/edl-v10 create mode 100644 src/mqtt/epl-v10 create mode 100755 src/mqtt/install_catch2.sh create mode 100755 src/mqtt/install_paho_mqtt_c.sh create mode 100644 src/mqtt/m4/README create mode 100644 src/mqtt/m4/m4_ax_cxx_compile_stdcxx.m4 create mode 100644 src/mqtt/m4/m4_ax_cxx_compile_stdcxx_11.m4 create mode 100644 src/mqtt/notice.html create mode 100644 src/mqtt/src/CMakeLists.txt create mode 100644 src/mqtt/src/async_client.cpp create mode 100644 src/mqtt/src/client.cpp create mode 100644 src/mqtt/src/connect_options.cpp create mode 100644 src/mqtt/src/disconnect_options.cpp create mode 100644 src/mqtt/src/iclient_persistence.cpp create mode 100644 src/mqtt/src/message.cpp create mode 100644 src/mqtt/src/mqtt/async_client.h create mode 100644 src/mqtt/src/mqtt/buffer_ref.h create mode 100644 src/mqtt/src/mqtt/buffer_view.h create mode 100644 src/mqtt/src/mqtt/callback.h create mode 100644 src/mqtt/src/mqtt/client.h create mode 100644 src/mqtt/src/mqtt/connect_options.h create mode 100644 src/mqtt/src/mqtt/delivery_token.h create mode 100644 src/mqtt/src/mqtt/disconnect_options.h create mode 100644 src/mqtt/src/mqtt/exception.h create mode 100644 src/mqtt/src/mqtt/iaction_listener.h create mode 100644 src/mqtt/src/mqtt/iasync_client.h create mode 100644 src/mqtt/src/mqtt/iclient_persistence.h create mode 100644 src/mqtt/src/mqtt/message.h create mode 100644 src/mqtt/src/mqtt/properties.h create mode 100644 src/mqtt/src/mqtt/response_options.h create mode 100644 src/mqtt/src/mqtt/server_response.h create mode 100644 src/mqtt/src/mqtt/ssl_options.h create mode 100644 src/mqtt/src/mqtt/string_collection.h create mode 100644 src/mqtt/src/mqtt/subscribe_options.h create mode 100644 src/mqtt/src/mqtt/thread_queue.h create mode 100644 src/mqtt/src/mqtt/token.h create mode 100644 src/mqtt/src/mqtt/topic.h create mode 100644 src/mqtt/src/mqtt/types.h create mode 100644 src/mqtt/src/mqtt/will_options.h create mode 100644 src/mqtt/src/properties.cpp create mode 100644 src/mqtt/src/response_options.cpp create mode 100644 src/mqtt/src/ssl_options.cpp create mode 100644 src/mqtt/src/string_collection.cpp create mode 100644 src/mqtt/src/subscribe_options.cpp create mode 100644 src/mqtt/src/token.cpp create mode 100644 src/mqtt/src/topic.cpp create mode 100644 src/mqtt/src/will_options.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 56d404f7..e2069371 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -18,3 +18,4 @@ add_subdirectory(network) add_subdirectory(leveldb) add_subdirectory(snappy) add_subdirectory(jsonrpc) +add_subdirectory(mqtt) \ No newline at end of file diff --git a/src/bigbang/CMakeLists.txt b/src/bigbang/CMakeLists.txt index 43558680..606a3fac 100755 --- a/src/bigbang/CMakeLists.txt +++ b/src/bigbang/CMakeLists.txt @@ -47,6 +47,7 @@ set(src wallet.cpp wallet.h blockchain.cpp blockchain.h forkmanager.cpp forkmanager.h + cmqcluster.cpp cmqcluster.h datastat.cpp datastat.h checkrepair.cpp checkrepair.h event.h diff --git a/src/bigbang/cmqcluster.cpp b/src/bigbang/cmqcluster.cpp new file mode 100644 index 00000000..b80a4894 --- /dev/null +++ b/src/bigbang/cmqcluster.cpp @@ -0,0 +1,11 @@ +// Copyright (c) 2019-2020 The Bigbang developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "cmqcluster.h" + +namespace bigbang +{ + + +} // namespace bigbang diff --git a/src/bigbang/cmqcluster.h b/src/bigbang/cmqcluster.h new file mode 100644 index 00000000..d1bb0d6a --- /dev/null +++ b/src/bigbang/cmqcluster.h @@ -0,0 +1,17 @@ +// Copyright (c) 2019-2020 The Bigbang developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BIGBANG_CMQCLUSTER_H +#define BIGBANG_CMQCLUSTER_H + +namespace bigbang +{ + +class CMQCluster +{ +}; + +} // namespace bigbang + +#endif //BIGBANG_CMQCLUSTER_H diff --git a/src/mqtt/CHANGELOG.md b/src/mqtt/CHANGELOG.md new file mode 100644 index 00000000..eb80fd4d --- /dev/null +++ b/src/mqtt/CHANGELOG.md @@ -0,0 +1,46 @@ +# Change Log + +## Version 1.1 (2019-10-12) + +This release was primarily to add MQTT v5 support and server responses. + +- MQTT v5 support: + - **Properties** + - New `property` class acts something like a std::variant to hold a property of any supported type. + - New `properties` class is a collection type to hold all the properties for a single transmitted packet. + - Properties can be added to outbound messages and obtained from received messages. + - Properties can also be obtained from server responses to requests such as from a _connect_ call. These are available in the `token` objects when they complete. + - The client object tracks the desired MQTT version that the app requested and/or is currently connected at. Internally this is now required by the `response_options` the need to distinguish between pre-v5 and post-v5 callback functions. + - MQTT v5 reason codes for requests are available via `token` objects when they complete. They are also available in `exception` objects that are thrown by tokens. + - Support for subscibe options, like no local subscriptions, etc. + - Sample applications were added showing how to do basic Remote Procedure Calls (RPC's) with MQTT v5 using the *RESPONSE_TOPIC* and *CORRELATION_DATA* properties. These are *rpc_math_cli* and *rpc_math_srvr* in the _src/samples_ directory. + - A sample "chat" application was added, showing how to use subscribe options, such as "no local". +- More descriptive error messages (PR #154), integrated into the `mqtt::exception` class. MQTT v5 reason codes are also included in the exceptions when an error occurs. +- Applications can (finally) get server responses from the various ACK packets. These are available through the tokens after they complete, as `connect_response`, `subscribe_response`, and `unsubscribe_response`. +- The `topic` objects can be used to subscribe. +- Applications can register individual callback functions instead of using a `callback` interface object. This allows easy use of lambda functions for callbacks. +- The connect options can take a LWT as a plain message, via `connect_options::set_will_message()` +- New unit tests have started using _Catch2_. +- Tested with Paho C v1.3.1 + +## Version 1.0.1 (2018-12-12) + +This is a bug-fix released aimed mainly at issues with the build system and working towards more "modern" usage of CMake. In addition: + +- Support for Paho C v1.2.1 +- Fixed a number of build issues, particularly on Windows +- Windows shared libraries (DLL's) now supported +- Several minor bug fixes + + +## Version 1.0.0 (2017-07-23) + +The initial Paho C++ Client library for memory-managed platforms (Linux, Windows, etc). + +- Requires Paho C Client Library v1.2. +- MQTT 3.1 & 3.1.1 +- SSL/TLS +- Asynchronous & Synchronous interfaces +- Persistence and off-line buffering +- Automatic reconnect +- High availability. diff --git a/src/mqtt/CMakeLists.txt b/src/mqtt/CMakeLists.txt new file mode 100644 index 00000000..7d115f5d --- /dev/null +++ b/src/mqtt/CMakeLists.txt @@ -0,0 +1,101 @@ +# CMakeLists.txt +# +# Top-level CMake file for the Paho C++ library. +# +#******************************************************************************* +# This is part of the Paho MQTT C++ client library. +# +# Copyright (c) 2016-2017, Guilherme Maciel Ferreira +# Copyright (c) 2017, Frank Pagliughi +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# and Eclipse Distribution License v1.0 which accompany this distribution. +# +# The Eclipse Public License is available at +# http://www.eclipse.org/legal/epl-v10.html +# and the Eclipse Distribution License is available at +# http://www.eclipse.org/org/documents/edl-v10.php. +# +# Contributors: +# Guilherme Maciel Ferreira - initial version +# Frank Pagliughi +#*******************************************************************************/ + +## Note: on OS X you should install XCode and the associated command-line tools + +## cmake flags +cmake_minimum_required(VERSION 3.5) + +## project name +project("paho-mqtt-cpp" + VERSION "1.1.0" + LANGUAGES CXX +) + +## --- Build options --- +message(STATUS "mqtt........................") +if(WIN32) + option(PAHO_BUILD_STATIC "Build static library" TRUE) + option(PAHO_BUILD_SHARED "Build shared library (DLL)" FALSE) + option(PAHO_WITH_SSL "Build SSL-enabled library" FALSE) + message(STATUS "win32") +else() + option(PAHO_BUILD_STATIC "Build static library" FALSE) + option(PAHO_BUILD_SHARED "Build shared library" TRUE) + #option(PAHO_BUILD_STATIC "Build static library" TRUE) + #option(PAHO_BUILD_SHARED "Build shared library" FALSE) + #set(PAHO_BUILD_STATIC ON) + #set(PAHO_BUILD_SHARED OFF) + option(PAHO_WITH_SSL "Build SSL-enabled library" TRUE) + message(STATUS "non-win32" " " "PAHO_BUILD_STATIC:" ${PAHO_BUILD_STATIC} " " "PAHO_BUILD_SHARED:" ${PAHO_BUILD_SHARED}) +endif() + +## --- C++11 build flags --- + +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +# Generate position-independent code (-fPIC on UNIX) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +# --- System Libraries --- + +include(GNUInstallDirs) + +if(WIN32) + set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) + set(LIBS_SYSTEM ws2_32) +elseif(UNIX) + set(LIBS_SYSTEM c stdc++) +endif() + +## --- Build directories --- + +# For the paho_mqtt_c module +list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) +add_subdirectory(src) + +# --- Default library --- + +if(PAHO_BUILD_SHARED) + set(PAHO_CPP_LIB paho-mqttpp3) + message(STATUS "shared......" ${PAHO_BUILD_SHARED}) + #set(PAHO_CPP_LIB paho-mqttpp3-static) +else() + set(PAHO_CPP_LIB paho-mqttpp3-static) + message(STATUS "static......") +endif() + +## --- Packaging settings --- + +if(WIN32) + set(CPACK_GENERATOR "ZIP") +elseif(UNIX) + set(CPACK_GENERATOR "TGZ") +endif() + +include(CPack) +add_subdirectory(cmake) + diff --git a/src/mqtt/CODING_STYLE.md b/src/mqtt/CODING_STYLE.md new file mode 100644 index 00000000..ba0ed06d --- /dev/null +++ b/src/mqtt/CODING_STYLE.md @@ -0,0 +1,30 @@ +# Eclipse Paho MQTT C++ Client Library +# Coding Styles + +This document describes the coding style and language conventions used by the Eclipse Paho C++ Client Library. + +## Language Standard + +The Paho C++ library uses Modern C++, adhering to the C++11 standard. + +Nothing in the library should prevent it from being built with a compiler adhering to a newer standard such as C++14 or C++17, but at this point the library itself doesn't use any newer language feature than C++11. Unfortunately, compilers vary wildly from version to version and platform to platform. So this is a constant struggle to keep true. + +Adherence to the base C++11 standard may change at some point in the future. Newer versions of the library will likely start using newer language features some time in the future as they are more widely adapted by compilers and projects. The next logical jump would probably be to C++17. + +## Naming Convention + +The Paho C++ library attempts to follow the naming conventions of the C++ standard library as much as possible, as this seems the most definitive standard for the language. + + - Class names are lower snake case: *classes_like_this* + - Function names are lower snake case: *functions_like_this* + - Variable names are lower camel case: *varsLikeThis* + - Class memeber are lower camel case with a trailing underscore: *memVarsLikeThis_* + - Constants are all caps: *CONSTANTS_LIKE_THIS* + +## Format Conventions + +The top-level project directory contains a _.editorconfig_ file with some basic hints as to formatting conventions for source files. + +A few minutes looking through the existing sources will reveal the basic styles used. Pull Requests should generally try to fit in to the existing style and not try to impose new ones. + +At some point in the future, a formatter may be added to the project (probably Clang format), but until then, any sources that can be fixed up with an automated code generator or beautifier would generally be accepted into the project. But at some point soon after would then be reformat to fit into the overall conventions. \ No newline at end of file diff --git a/src/mqtt/CONTRIBUTING.md b/src/mqtt/CONTRIBUTING.md new file mode 100644 index 00000000..5f799ef1 --- /dev/null +++ b/src/mqtt/CONTRIBUTING.md @@ -0,0 +1,47 @@ +Contributing to Paho +==================== + +Thanks for your interest in this project. + +Project description: +-------------------- + +The Paho project has been created to provide scalable open-source implementations of open and standard messaging protocols aimed at new, existing, and emerging applications for Machine-to-Machine (M2M) and Internet of Things (IoT). +Paho reflects the inherent physical and cost constraints of device connectivity. Its objectives include effective levels of decoupling between devices and applications, designed to keep markets open and encourage the rapid growth of scalable Web and Enterprise middleware and applications. Paho is being kicked off with MQTT publish/subscribe client implementations for use on embedded platforms, along with corresponding server support as determined by the community. + +- https://projects.eclipse.org/projects/technology.paho + +Developer resources: +-------------------- + +Information regarding source code management, builds, coding standards, and more. + +- https://projects.eclipse.org/projects/technology.paho/developer + +Contributor License Agreement: +------------------------------ + +Before your contribution can be accepted by the project, you need to create and electronically sign the Eclipse Foundation Contributor License Agreement (CLA). + +- http://www.eclipse.org/legal/CLA.php + +Contact: +-------- + +Contact the project developers via the project's "dev" list. + +- https://dev.eclipse.org/mailman/listinfo/paho-dev + +Search for bugs: +---------------- + +This project uses Bugzilla to track ongoing development and issues. + +- https://bugs.eclipse.org/bugs/buglist.cgi?product=Paho + +Create a new bug: +----------------- + +Be sure to search for existing bugs before you create another one. Remember that contributions are always welcome! + +- https://bugs.eclipse.org/bugs/enter_bug.cgi?product=Paho diff --git a/src/mqtt/README.md b/src/mqtt/README.md new file mode 100644 index 00000000..6584f59b --- /dev/null +++ b/src/mqtt/README.md @@ -0,0 +1,331 @@ +# Eclipse Paho MQTT C++ Client Library + +[![Build Status](https://travis-ci.org/eclipse/paho.mqtt.cpp.svg?branch=master)](https://travis-ci.org/eclipse/paho.mqtt.cpp) + +This repository contains the source code for the [Eclipse Paho](http://eclipse.org/paho) MQTT C++ client library on memory-managed operating systems such as Linux/Posix and Windows. + +This code builds a library which enables C++11 applications to connect to an [MQTT](http://mqtt.org) broker, publish messages to the broker, and to subscribe to topics and receive published messages. + +The library has the following features: + +- Support for MQTT v5, v3.1.1, and v 3.1 +- TCP, SSL/TLS, and WebSocket transports +- Message persistence +- Automatic reconnect +- Offline buffering +- High availability +- Blocking and non-blocking API's + +This code requires the [Paho C library](https://github.com/eclipse/paho.mqtt.c) by Ian Craggs, et al., specifically version 1.3.1 or possibly later. + +## Latest News + +To keep up with the latest announcements for this project, or to ask questions: + +**Twitter:** [@eclipsepaho](https://twitter.com/eclipsepaho) and [@fmpagliughi](https://twitter.com/fmpagliughi) + +**EMail:** [Eclipse Paho Mailing List](https://accounts.eclipse.org/mailing-list/paho-dev) + +**Mattermost:** [Eclipse Mattermost Paho Channel](https://mattermost.eclipse.org/eclipse/channels/paho) + + +### New Features in Paho C++ v1.1 + +- MQTT v5 support: + - **Properties** + - New `property` class acts something like a std::variant to hold a property of any supported type. + - New `properties` class is a collection type to hold all the properties for a single transmitted packet. + - Properties can be added to outbound messages and obtained from received messages. + - Properties can also be obtained from server responses to requests such as from a _connect_ call. These are available in the `token` objects when they complete. + - The client object tracks the desired MQTT version that the app requested and/or is currently connected at. Internally this is now required by the `response_options` the need to distinguish between pre-v5 and post-v5 callback functions. + - MQTT v5 reason codes for requests are available via `token` objects when they complete. They are also available in `exception` objects that are thrown by tokens. + - Support for subscibe options, like no local subscriptions, etc. + - Sample applications were added showing how to do basic Remote Procedure Calls (RPC's) with MQTT v5 using the *RESPONSE_TOPIC* and *CORRELATION_DATA* properties. These are *rpc_math_cli* and *rpc_math_srvr* in the _src/samples_ directory. + - A sample "chat" application was added, showing how to use subscribe options, such as "no local". +- More descriptive error messages (PR #154), integrated into the `mqtt::exception` class. MQTT v5 reason codes are also included in the exceptions when an error occurs. +- Applications can (finally) get server responses from the various ACK packets. These are available through the tokens after they complete, as `connect_response`, `subscribe_response`, and `unsubscribe_response`. +- The `topic` objects can be used to subscribe. +- Applications can register individual callback functions instead of using a `callback` interface object. This allows easy use of lambda functions for callbacks. +- The connect options can take a LWT as a plain message, via `connect_options::set_will_message()` + +### _Catch2_ Unit Tests + +Unit tests are being converted to use _Catch2_ for the test framework. The legacy unit tests are still using _CppUnit_, compiled into a separate test executable. If everything goes well with _Catch2_, the older unit tests will be ported to _Catch2_ as well. + +_Catch2_ can be found here: [Catch2](https://github.com/catchorg/Catch2) + +## Contributing + +Contributions to this project are gladly welcomed. Before submitting a Pull Request, please keep two things in mind: + + - This is an official Eclipse project, so it is required that all contributors sign an [Eclipse Contributor Agreement (ECA)](https://www.eclipse.org/legal/ECA.php) + - Please submit all Pull Requests against the _develop_ branch (not master). + + For full details, see [CONTRIBUTING.md](https://github.com/eclipse/paho.mqtt.cpp/blob/master/CONTRIBUTING.md). + +## Building from source + +*GNU Make and autotools were deprecated and removed in the v1.1 release.* + +_CMake_ is a cross-platform build system suitable for Unix and non-Unix platforms such as Microsoft Windows. It is now the only supported build system. + +The Paho C++ library requires the Paho C library, v1.3.1 or greater, to be built and installed first. More information below. + +CMake allows for options to direct the build. The following are specific to Paho C++: + +Variable | Default Value | Description +------------ | ------------- | ------------- +PAHO_BUILD_SHARED | TRUE (Linux), FALSE (Win32) | Whether to build the shared library +PAHO_BUILD_STATIC | FALSE (Linux), TRUE (Win32) | Whether to build the static library +PAHO_BUILD_DOCUMENTATION | FALSE | Create and install the HTML based API documentation (requires Doxygen) +PAHO_BUILD_SAMPLES | FALSE | Build sample programs +PAHO_BUILD_TESTS | FALSE | Build the unit tests. (This currently requires both _CppUnit_ and _Catch2_) +PAHO_WITH_SSL | TRUE (Linux), FALSE (Win32) | Flag that defines whether to build ssl-enabled binaries too + +In addition, the C++ build might commonly use `CMAKE_PREFIX_PATH` to help the build system find the location of the Paho C library. + +### Unix and Linux + +On *nix systems CMake creates Makefiles. + +The build process currently supports a number of Unix and Linux flavors. The build process requires the following tools: + + * CMake v3.5 or newer + * GCC v4.8 or newer or Clang v3.9 or newer + * GNU Make + +On Debian based systems this would mean that the following packages have to be installed: + +``` +$ sudo apt-get install build-essential gcc make cmake cmake-gui cmake-curses-gui +``` + +If you will be using secure sockets (and you probably should): + +``` +$ sudo apt-get install libssl-dev +``` + +Building the documentation requires doxygen and optionally graphviz to be installed: + +``` +$ sudo apt-get install doxygen graphviz +``` + +Unite tests are currently being built using both _CppUnit_ and _Catch2_. The _CppUnit_ tests are being deprecated and replaced with _Catch2_ equivalents. In the meantime, however, both systems are required to build the tests. + +``` +$ sudo apt-get install libcppunit-dev +``` + +_Catch2_ can be found here: [Catch2](https://github.com/catchorg/Catch2) + + +Before building the C++ library, first, build and install the Paho C library: + +``` +$ git clone https://github.com/eclipse/paho.mqtt.c.git +$ cd paho.mqtt.c +$ git checkout v1.3.1 + +$ cmake -Bbuild -H. -DPAHO_WITH_SSL=ON -DPAHO_ENABLE_TESTING=OFF +$ sudo cmake --build build/ --target install +$ sudo ldconfig +``` + +This builds with SSL/TLS enabled. If that is not desired, omit the `-DPAHO_WITH_SSL=ON`. + +If you installed the C library on a non-standard path, you might want to pass it as value to the `CMAKE_PREFIX_PATH` option. + +Using these variables CMake can be used to generate your Makefiles. The out-of-source build is the default on CMake. Therefore it is recommended to invoke all build commands inside your chosen build directory. + +An example build session might look like this: + +``` +$ git clone https://github.com/eclipse/paho.mqtt.cpp +$ cd paho.mqtt.cpp +$ cmake -Bbuild -H. -DPAHO_BUILD_DOCUMENTATION=TRUE -DPAHO_BUILD_SAMPLES=TRUE +$ sudo cmake --build build/ --target install +$ sudo ldconfig +``` + +If you did not install Paho C library to a default system location or you want to build against a different version, use the `CMAKE_PREFIX_PATH` to specify its install location: + +``` +$ cmake -Bbuild -H. -DPAHO_BUILD_DOCUMENTATION=TRUE -DPAHO_BUILD_SAMPLES=TRUE \ + -DCMAKE_PREFIX_PATH=../../paho.mqtt.c/build/install/usr/local +``` + +To use another compiler, either the CXX environment variable can be specified in the configuration step: + +``` +$ CXX=clang++ cmake .. +``` + +or the `CMAKE_CXX_COMPILER` flag can be used: + + +``` +$ cmake -DCMAKE_CXX_COMPILER=clang++ +``` + +#### Updating CMake on Ubuntu 14.04 or 16.04 + +The versions of CMake on Ubuntu 14.04 or 16.04 LTS are pretty old and have some problems with Paho C++ library. A newer version can be added by downloading the source and building it. If the older cmake can be removed from the system using the package manager, or it can be kept, using the Ububtu alternatives to chose between the versions. + +For example, here's how to install CMake v3.6 on Ubuntu 14.04, while keeping the older CMake available as _cmake-2.8:_ + +``` +$ wget http://www.cmake.org/files/v3.6/cmake-3.6.3.tar.gz +$ tar -xvzf cmake-3.6.3.tar.gz +$ cd cmake-3.6.3/ +$ ./configure +$ make +$ sudo make install +$ sudo mv /usr/bin/cmake /usr/bin/cmake-2.8 +$ sudo update-alternatives --install /usr/bin/cmake cmake /usr/local/bin/cmake 100 +$ sudo update-alternatives --install /usr/bin/cmake cmake /usr/bin/cmake-2.8 200 +$ cmake --version +cmake version 3.6.3 +``` + +You can speed up the CMake build on multi-core systems, by specifying parallel buid jobs for the configure and make steps, above, such as the following for a 4-core system: +``` +$ ./configure --parallel=4 +$ make -j4 +$ sudo make install +``` + +### Windows + +On Windows systems CMake creates Visual Studio project files. + +The build process currently supports a number Windows versions. The build process requires the following tools: + * CMake GUI v3.5 or newer + * Visual Studio 2015 or newer + +First install and open the cmake-gui application. This tutorial is based on cmake-gui 3.5.2. + +Second, select the path to the Paho MQTT C library (CMAKE_PREFIX_PATH) if not installed in a standard path. Remember that the Paho MQTT C must be installed on the system. Next, choose if it is supposed to build the documentation (PAHO_BUILD_DOCUMENTATION) and/or the sample applications (PAHO_BUILD_SAMPLES). + +Once the configuration is done, click on the Configure button, select the version of the Visual Studio, and then click on Generate button. + +At the end of this process you have a Visual Studio solution. + +Alternately, the libraries can be completely built at an MSBuild Command Prompt. Download the Paho C and C++ library sources, then open a command window and first compile the Paho C library: + +``` +> cd paho.mqtt.c +> cmake -Bbuild -H. -DCMAKE_INSTALL_PREFIX=C:\mqtt\paho-c +> cmake --build build/ --target install +``` + +Then build the C++ library: + +``` +> cd ..\paho.mqtt.cpp +> cmake -Bbuild -H. -DCMAKE_INSTALL_PREFIX=C:\mqtt\paho-cpp -DPAHO_BUILD_SAMPLES=ON -DPAHO_WITH_SSL=OFF -DCMAKE_PREFIX_PATH=C:\mqtt\paho-c +> cmake --build build/ --target install +``` +This builds and installs both libraries to a non-standard location under `C:\mqtt`. Modify this location as desired or use the default location, but either way, the C++ library will most likely need to be told where the C library was built using `CMAKE_PREFIX_PATH`. + +It seems quite odd, but even on a 64-bit system using a 64-bit compiler, MSVC seems to default to a 32-bit build target. + +The 64-bit target can be selected using tge CMake generator switch, *-G*, at configuration time. The full version must be provided. For Visual Studio 2015 which is v14 do this to first build the Paho C library: + +``` +> cmake -G "Visual Studio 14 Win64" -Bbuild -H. -DCMAKE_INSTALL_PREFIX=C:\mqtt\paho-c +... +``` + +Then use it to build the C++ library: + +``` +> cmake -G "Visual Studio 14 Win64" -Bbuild -H. -DCMAKE_INSTALL_PREFIX=C:\mqtt\paho-cpp -DPAHO_WITH_SSL=OFF -DCMAKE_PREFIX_PATH=C:\mqtt\paho-c +... +``` + +*Note that it is very important that you use the same generator (target) to build BOTH libraries, otherwise you will get lots of linker errors when you try to build the C++ library.* + + +## Example + +Sample applications can be found in the source repository at _src/samples_: +https://github.com/eclipse/paho.mqtt.cpp/tree/master/src/samples + +This is a partial example of what a typical example might look like: + +```cpp +int main(int argc, char* argv[]) +{ + sample_mem_persistence persist; + mqtt::client client(ADDRESS, CLIENTID, &persist); + + callback cb; + client.set_callback(cb); + + mqtt::connect_options connOpts; + connOpts.set_keep_alive_interval(20); + connOpts.set_clean_session(true); + + try { + client.connect(connOpts); + + // First use a message pointer. + + mqtt::message_ptr pubmsg = mqtt::make_message(PAYLOAD1); + pubmsg->set_qos(QOS); + client.publish(TOPIC, pubmsg); + + // Now try with itemized publish. + + client.publish(TOPIC, PAYLOAD2, strlen(PAYLOAD2)+1, 0, false); + + // Disconnect + + client.disconnect(); + } + catch (const mqtt::persistence_exception& exc) { + cerr << "Persistence Error: " << exc.what() << " [" + << exc.get_reason_code() << "]" << endl; + return 1; + } + catch (const mqtt::exception& exc) { + cerr << "Error: " << exc.what() << " [" + << exc.get_reason_code() << "]" << endl; + return 1; + } + + return 0; +} +``` + +----------- + +The original API organization and documentation were adapted from: + +The Paho Java library +by Dave Locke. +Copyright (c) 2012, IBM Corp + + All rights reserved. This program and the accompanying materials + are made available under the terms of the Eclipse Public License v1.0 + which accompanies this distribution, and is available at + http://www.eclipse.org/legal/epl-v10.html + +----------- + +This code requires: + +The Paho C library by Ian Craggs +Copyright (c) 2013-2018, IBM Corp. + + All rights reserved. This program and the accompanying materials + are made available under the terms of the Eclipse Public License v1.0 + and Eclipse Distribution License v1.0 which accompany this distribution. + + The Eclipse Public License is available at + http://www.eclipse.org/legal/epl-v10.html + and the Eclipse Distribution License is available at + http://www.eclipse.org/org/documents/edl-v10.php. + diff --git a/src/mqtt/about.html b/src/mqtt/about.html new file mode 100644 index 00000000..6555a44e --- /dev/null +++ b/src/mqtt/about.html @@ -0,0 +1,28 @@ + + + +About + + +

About This Content

+ +

December 9, 2013

+

License

+ +

The Eclipse Foundation makes available all content in this plug-in ("Content"). Unless otherwise +indicated below, the Content is provided to you under the terms and conditions of the +Eclipse Public License Version 1.0 ("EPL") and Eclipse Distribution License Version 1.0 ("EDL"). +A copy of the EPL is available at +http://www.eclipse.org/legal/epl-v10.html +and a copy of the EDL is available at +http://www.eclipse.org/org/documents/edl-v10.php. +For purposes of the EPL, "Program" will mean the Content.

+ +

If you did not receive this Content directly from the Eclipse Foundation, the Content is +being redistributed by another party ("Redistributor") and different terms and conditions may +apply to your use of any object code in the Content. Check the Redistributor's license that was +provided with the Content. If no such license exists, contact the Redistributor. Unless otherwise +indicated below, the terms and conditions of the EPL still apply to any source code in the Content +and such source code may be obtained at http://www.eclipse.org.

+ + diff --git a/src/mqtt/buildtst.sh b/src/mqtt/buildtst.sh new file mode 100755 index 00000000..1df494d3 --- /dev/null +++ b/src/mqtt/buildtst.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# +# buildtst.sh +# +# Build test for the Paho C++ library. +# +# This test the build with a few compilers on Linux. It does a build using +# CMake, for the library, tests, and examples, then runs the unit tests. +# This is repeated for each of the compilers in the list. If a particular +# compiler is not installed on the system, it is just skipped. +# +# This is not meant to replace Travis or other CI on the repo server, but +# is a quick test to use locally during development. +# + +COMPILERS="g++-5 g++-6 g++-7 g++-8 clang++-3.9 clang++-4.0 clang++-5.0 clang++-6.0 clang++-7 clang++-8" +[ "$#" -gt 0 ] && COMPILERS="$@" + +[ -z "${BUILD_JOBS}" ] && BUILD_JOBS=4 +[ -n "${PAHO_C_PATH}" ] && PAHO_C_SWITCH="-DCMAKE_PREFIX_PATH=${PAHO_C_PATH}" + +for COMPILER in $COMPILERS; do + if [ -z "$(which ${COMPILER})" ]; then + printf "Compiler not found: %s\n" "${COMPILER}" + else + printf "===== Testing: %s =====\n\n" "${COMPILER}" + rm -rf buildtst-build/ + mkdir buildtst-build ; pushd buildtst-build &> /dev/null + + if ! cmake -DCMAKE_CXX_COMPILER=${COMPILER} -DPAHO_WITH_SSL=ON -DPAHO_BUILD_SAMPLES=ON -DPAHO_BUILD_TESTS=ON ${PAHO_C_SWITCH} .. ; then + printf "\nCMake configuration failed for %s\n" "${COMPILER}" + exit 1 + fi + + if ! make -j${BUILD_JOBS} ; then + printf "\nCompilation failed for %s\n" "${COMPILER}" + exit 2 + fi + + printf "Running Catch2 Unit tests for %s:\n" "${COMPILER}" + if ! ./test/unit/unit_tests ; then + printf "\nCatch2 unit test failed for %s\n" "${COMPILER}" + exit 3 + fi + + printf "Running CppUnit tests for %s:\n" "${COMPILER}" + if ! ./test/cppunit/paho-mqttpp-test ; then + printf "\nUnit test failed for %s\n" "${COMPILER}" + exit 4 + fi + + popd &> /dev/null + fi + printf "\n" +done + +rm -rf buildtst-build/ +printf "\nAll builds completed successfully\n\n" + +if ! cppcheck --enable=all --std=c++11 --force --quiet src/*.cpp ; then + printf "\ncppcheck failed\n" + exit 5 +fi + +printf "\n===== All tests completed successfully =====\n\n" +exit 0 diff --git a/src/mqtt/cmake/CMakeLists.txt b/src/mqtt/cmake/CMakeLists.txt new file mode 100644 index 00000000..a9f89087 --- /dev/null +++ b/src/mqtt/cmake/CMakeLists.txt @@ -0,0 +1,22 @@ +set(package_name PahoMqttCpp) +configure_file(${package_name}Config.cmake.in ${package_name}Config.cmake @ONLY) + +include(CMakePackageConfigHelpers) +write_basic_package_version_file( + "${CMAKE_CURRENT_BINARY_DIR}/${package_name}ConfigVersion.cmake" + VERSION ${PROJECT_VERSION} + COMPATIBILITY SameMajorVersion) # TODO + +export(EXPORT ${package_name} + FILE "${CMAKE_CURRENT_BINARY_DIR}/${package_name}Targets.cmake" + NAMESPACE ${package_name}::) + +install(EXPORT ${package_name} DESTINATION lib/cmake/${package_name} + FILE ${package_name}Targets.cmake + NAMESPACE ${package_name}::) + +install(FILES + "${CMAKE_CURRENT_BINARY_DIR}/${package_name}Config.cmake" + FindPahoMqttC.cmake + "${CMAKE_CURRENT_BINARY_DIR}/${package_name}ConfigVersion.cmake" + DESTINATION lib/cmake/${package_name}) diff --git a/src/mqtt/cmake/FindPahoMqttC.cmake b/src/mqtt/cmake/FindPahoMqttC.cmake new file mode 100644 index 00000000..53f15a39 --- /dev/null +++ b/src/mqtt/cmake/FindPahoMqttC.cmake @@ -0,0 +1,31 @@ +# find the Paho MQTT C library +if(PAHO_WITH_SSL) + set(_PAHO_MQTT_C_LIB_NAME paho-mqtt3as) + find_package(OpenSSL REQUIRED) +else() + set(_PAHO_MQTT_C_LIB_NAME paho-mqtt3a) +endif() +# add suffix when using static Paho MQTT C library variant +if(PAHO_BUILD_STATIC) + set(_PAHO_MQTT_C_LIB_NAME ${_PAHO_MQTT_C_LIB_NAME}-static) +endif() + +find_library(PAHO_MQTT_C_LIBRARIES NAMES ${_PAHO_MQTT_C_LIB_NAME}) +unset(_PAHO_MQTT_C_LIB_NAME) +find_path(PAHO_MQTT_C_INCLUDE_DIRS NAMES MQTTAsync.h) + +add_library(PahoMqttC::PahoMqttC UNKNOWN IMPORTED) + +set_target_properties(PahoMqttC::PahoMqttC PROPERTIES + IMPORTED_LOCATION "${PAHO_MQTT_C_LIBRARIES}" + INTERFACE_INCLUDE_DIRECTORIES "${PAHO_MQTT_C_INCLUDE_DIRS}" + IMPORTED_LINK_INTERFACE_LANGUAGES "C") +if(PAHO_WITH_SSL) + set_target_properties(PahoMqttC::PahoMqttC PROPERTIES + INTERFACE_COMPILE_DEFINITIONS "OPENSSL=1" + INTERFACE_LINK_LIBRARIES "OpenSSL::SSL;OpenSSL::Crypto") +endif() + +include(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(PahoMqttC + REQUIRED_VARS PAHO_MQTT_C_LIBRARIES PAHO_MQTT_C_INCLUDE_DIRS) diff --git a/src/mqtt/cmake/PahoMqttCppConfig.cmake.in b/src/mqtt/cmake/PahoMqttCppConfig.cmake.in new file mode 100644 index 00000000..aff9472c --- /dev/null +++ b/src/mqtt/cmake/PahoMqttCppConfig.cmake.in @@ -0,0 +1,11 @@ +# save build-time options +set(PAHO_BUILD_STATIC @PAHO_BUILD_STATIC@) +set(PAHO_BUILD_SHARED @PAHO_BUILD_SHARED@) +set(PAHO_WITH_SSL @PAHO_WITH_SSL@) + +include(CMakeFindDependencyMacro) +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}) +find_dependency(PahoMqttC REQUIRED) +list(REMOVE_AT CMAKE_MODULE_PATH -1) + +include("${CMAKE_CURRENT_LIST_DIR}/@package_name@Targets.cmake") diff --git a/src/mqtt/devenv.sh b/src/mqtt/devenv.sh new file mode 100644 index 00000000..254626bc --- /dev/null +++ b/src/mqtt/devenv.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# +# devenv.sh +# +# Sets up a development environment for working on the the Paho C++ Library on Linux, +# using the development tree of the Paho C library. +# This is _not_ necessary for building applications against the library once it has +# been installed - it's just for library developers. +# +# Source it into the current shell as: +# $ source devenv.sh +# + +PAHO_C_DIR=$(readlink -e ../paho.mqtt.c) + +export DEVELOP=1 +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${PAHO_C_DIR}/build/output:${PWD}/lib + diff --git a/src/mqtt/dist/Makefile b/src/mqtt/dist/Makefile new file mode 100644 index 00000000..7c485e7f --- /dev/null +++ b/src/mqtt/dist/Makefile @@ -0,0 +1,11 @@ +VERSION=1.0.0 + +check: + rpmlint -i dist/paho-cpp.spec + +rpm-prep: + mkdir -p ${HOME}/rpmbuild/SOURCES/ + tar --transform="s/\./paho-cpp-${VERSION}/" -cf ${HOME}/rpmbuild/SOURCES/v${VERSION}.tar.gz --exclude=./build.paho.cpp --exclude=.git --exclude=*.bz ./ --gzip + +rpm: rpm-prep + rpmbuild -ba dist/paho-cpp.spec diff --git a/src/mqtt/dist/paho-cpp.spec b/src/mqtt/dist/paho-cpp.spec new file mode 100644 index 00000000..6a05157a --- /dev/null +++ b/src/mqtt/dist/paho-cpp.spec @@ -0,0 +1,64 @@ +Summary: MQTT CPP Client +Name: paho-cpp +Version: 1.0.0 +Release: 0%{?dist} +License: Eclipse Distribution License 1.0 and Eclipse Public License 1.0 +Group: Development/Tools +Source: https://github.com/eclipse/paho.mqtt.cpp/archive/v%{version}.tar.gz +URL: https://eclipse.org/paho/clients/cpp/ +BuildRequires: cmake3 +BuildRequires: gcc +BuildRequires: graphviz +BuildRequires: doxygen +BuildRequires: openssl-devel +BuildRequires: paho-c-devel +Requires: openssl +Requires: paho-c + + +%description +The Paho MQTT CPP Client is a fully fledged MQTT client written in ANSI standard C++ 11. + + +%package devel +Summary: MQTT CPP Client development kit +Group: Development/Libraries +Requires: paho-cpp + +%description devel +Development files and samples for the the Paho MQTT CPP Client. + + +%package devel-docs +Summary: MQTT CPP Client development kit documentation +Group: Development/Libraries + +%description devel-docs +Development documentation files for the the Paho MQTT CPP Client. + +%prep +%autosetup -n paho-cpp-%{version} + +%build +mkdir build.paho.cpp && cd build.paho.cpp +%cmake3 -DPAHO_WITH_SSL=TRUE -DPAHO_BUILD_DOCUMENTATION=TRUE -DPAHO_BUILD_SAMPLES=TRUE .. +make %{?_smp_mflags} + +%install +cd build.paho.cpp +make install DESTDIR=%{buildroot} + +%files +%doc edl-v10 epl-v10 +%{_libdir}/* + +%files devel +%{_bindir}/* +%{_includedir}/* + +%files devel-docs +%{_datadir}/* + +%changelog +* Wed Oct 11 2017 Julien Courtat - 1.0.0 +- Initial packaging diff --git a/src/mqtt/edl-v10 b/src/mqtt/edl-v10 new file mode 100644 index 00000000..cf989f14 --- /dev/null +++ b/src/mqtt/edl-v10 @@ -0,0 +1,15 @@ + +Eclipse Distribution License - v 1.0 + +Copyright (c) 2007, Eclipse Foundation, Inc. and its licensors. + +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + Neither the name of the Eclipse Foundation, Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/src/mqtt/epl-v10 b/src/mqtt/epl-v10 new file mode 100644 index 00000000..79e486c3 --- /dev/null +++ b/src/mqtt/epl-v10 @@ -0,0 +1,70 @@ +Eclipse Public License - v 1.0 + +THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS + +"Contribution" means: + +a) in the case of the initial Contributor, the initial code and documentation distributed under this Agreement, and +b) in the case of each subsequent Contributor: +i) changes to the Program, and +ii) additions to the Program; +where such changes and/or additions to the Program originate from and are distributed by that particular Contributor. A Contribution 'originates' from a Contributor if it was added to the Program by such Contributor itself or anyone acting on such Contributor's behalf. Contributions do not include additions to the Program which: (i) are separate modules of software distributed in conjunction with the Program under their own license agreement, and (ii) are not derivative works of the Program. +"Contributor" means any person or entity that distributes the Program. + +"Licensed Patents" mean patent claims licensable by a Contributor which are necessarily infringed by the use or sale of its Contribution alone or when combined with the Program. + +"Program" means the Contributions distributed in accordance with this Agreement. + +"Recipient" means anyone who receives the Program under this Agreement, including all Contributors. + +2. GRANT OF RIGHTS + +a) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, distribute and sublicense the Contribution of such Contributor, if any, and such derivative works, in source code and object code form. +b) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed Patents to make, use, sell, offer to sell, import and otherwise transfer the Contribution of such Contributor, if any, in source code and object code form. This patent license shall apply to the combination of the Contribution and the Program if, at the time the Contribution is added by the Contributor, such addition of the Contribution causes such combination to be covered by the Licensed Patents. The patent license shall not apply to any other combinations which include the Contribution. No hardware per se is licensed hereunder. +c) Recipient understands that although each Contributor grants the licenses to its Contributions set forth herein, no assurances are provided by any Contributor that the Program does not infringe the patent or other intellectual property rights of any other entity. Each Contributor disclaims any liability to Recipient for claims brought by any other entity based on infringement of intellectual property rights or otherwise. As a condition to exercising the rights and licenses granted hereunder, each Recipient hereby assumes sole responsibility to secure any other intellectual property rights needed, if any. For example, if a third party patent license is required to allow Recipient to distribute the Program, it is Recipient's responsibility to acquire that license before distributing the Program. +d) Each Contributor represents that to its knowledge it has sufficient copyright rights in its Contribution, if any, to grant the copyright license set forth in this Agreement. +3. REQUIREMENTS + +A Contributor may choose to distribute the Program in object code form under its own license agreement, provided that: + +a) it complies with the terms and conditions of this Agreement; and +b) its license agreement: +i) effectively disclaims on behalf of all Contributors all warranties and conditions, express and implied, including warranties or conditions of title and non-infringement, and implied warranties or conditions of merchantability and fitness for a particular purpose; +ii) effectively excludes on behalf of all Contributors all liability for damages, including direct, indirect, special, incidental and consequential damages, such as lost profits; +iii) states that any provisions which differ from this Agreement are offered by that Contributor alone and not by any other party; and +iv) states that source code for the Program is available from such Contributor, and informs licensees how to obtain it in a reasonable manner on or through a medium customarily used for software exchange. +When the Program is made available in source code form: + +a) it must be made available under this Agreement; and +b) a copy of this Agreement must be included with each copy of the Program. +Contributors may not remove or alter any copyright notices contained within the Program. + +Each Contributor must identify itself as the originator of its Contribution, if any, in a manner that reasonably allows subsequent Recipients to identify the originator of the Contribution. + +4. COMMERCIAL DISTRIBUTION + +Commercial distributors of software may accept certain responsibilities with respect to end users, business partners and the like. While this license is intended to facilitate the commercial use of the Program, the Contributor who includes the Program in a commercial product offering should do so in a manner which does not create potential liability for other Contributors. Therefore, if a Contributor includes the Program in a commercial product offering, such Contributor ("Commercial Contributor") hereby agrees to defend and indemnify every other Contributor ("Indemnified Contributor") against any losses, damages and costs (collectively "Losses") arising from claims, lawsuits and other legal actions brought by a third party against the Indemnified Contributor to the extent caused by the acts or omissions of such Commercial Contributor in connection with its distribution of the Program in a commercial product offering. The obligations in this section do not apply to any claims or Losses relating to any actual or alleged intellectual property infringement. In order to qualify, an Indemnified Contributor must: a) promptly notify the Commercial Contributor in writing of such claim, and b) allow the Commercial Contributor to control, and cooperate with the Commercial Contributor in, the defense and any related settlement negotiations. The Indemnified Contributor may participate in any such claim at its own expense. + +For example, a Contributor might include the Program in a commercial product offering, Product X. That Contributor is then a Commercial Contributor. If that Commercial Contributor then makes performance claims, or offers warranties related to Product X, those performance claims and warranties are such Commercial Contributor's responsibility alone. Under this section, the Commercial Contributor would have to defend claims against the other Contributors related to those performance claims and warranties, and if a court requires any other Contributor to pay any damages as a result, the Commercial Contributor must pay those damages. + +5. NO WARRANTY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the appropriateness of using and distributing the Program and assumes all risks associated with its exercise of rights under this Agreement , including but not limited to the risks and costs of program errors, compliance with applicable laws, damage to or loss of data, programs or equipment, and unavailability or interruption of operations. + +6. DISCLAIMER OF LIABILITY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +7. GENERAL + +If any provision of this Agreement is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this Agreement, and without further action by the parties hereto, such provision shall be reformed to the minimum extent necessary to make such provision valid and enforceable. + +If Recipient institutes patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Program itself (excluding combinations of the Program with other software or hardware) infringes such Recipient's patent(s), then such Recipient's rights granted under Section 2(b) shall terminate as of the date such litigation is filed. + +All Recipient's rights under this Agreement shall terminate if it fails to comply with any of the material terms or conditions of this Agreement and does not cure such failure in a reasonable period of time after becoming aware of such noncompliance. If all Recipient's rights under this Agreement terminate, Recipient agrees to cease use and distribution of the Program as soon as reasonably practicable. However, Recipient's obligations under this Agreement and any licenses granted by Recipient relating to the Program shall continue and survive. + +Everyone is permitted to copy and distribute copies of this Agreement, but in order to avoid inconsistency the Agreement is copyrighted and may only be modified in the following manner. The Agreement Steward reserves the right to publish new versions (including revisions) of this Agreement from time to time. No one other than the Agreement Steward has the right to modify this Agreement. The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation may assign the responsibility to serve as the Agreement Steward to a suitable separate entity. Each new version of the Agreement will be given a distinguishing version number. The Program (including Contributions) may always be distributed subject to the version of the Agreement under which it was received. In addition, after a new version of the Agreement is published, Contributor may elect to distribute the Program (including its Contributions) under the new version. Except as expressly stated in Sections 2(a) and 2(b) above, Recipient receives no rights or licenses to the intellectual property of any Contributor under this Agreement, whether expressly, by implication, estoppel or otherwise. All rights in the Program not expressly granted under this Agreement are reserved. + +This Agreement is governed by the laws of the State of New York and the intellectual property laws of the United States of America. No party to this Agreement will bring a legal action under this Agreement more than one year after the cause of action arose. Each party waives its rights to a jury trial in any resulting litigation. diff --git a/src/mqtt/install_catch2.sh b/src/mqtt/install_catch2.sh new file mode 100755 index 00000000..95c4290d --- /dev/null +++ b/src/mqtt/install_catch2.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# +# install_catch2.sh +# +# Travis CI build/test helper script for the Eclipse Paho C++ library. +# This installs Catch2 into the VM. +# + +set -ex + +VERSION=2.9.1 + +# Install Catch2 from sources +wget https://github.com/catchorg/Catch2/archive/v${VERSION}.tar.gz +tar -xf v${VERSION}.tar.gz +cd Catch2-${VERSION}/ +cmake -Bbuild -H. -DBUILD_TESTING=OFF + +# CMake bin is installed in a strange place where +# sudo can not find by default. +sudo env "PATH=$PATH" cmake --build build/ --target install + diff --git a/src/mqtt/install_paho_mqtt_c.sh b/src/mqtt/install_paho_mqtt_c.sh new file mode 100755 index 00000000..8fd7f4b4 --- /dev/null +++ b/src/mqtt/install_paho_mqtt_c.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# +# Installs the matching version of Paho MQTT C library required by the C++ lib. +# + +set -ex + +git clone https://github.com/eclipse/paho.mqtt.c.git +cd paho.mqtt.c +git checkout v1.3.1 +cmake -Bbuild -H. -DPAHO_WITH_SSL=ON -DPAHO_BUILD_STATIC=ON -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DPAHO_ENABLE_TESTING=OFF +sudo env "PATH=$PATH" cmake --build build/ --target install +sudo ldconfig +exit 0 + + diff --git a/src/mqtt/m4/README b/src/mqtt/m4/README new file mode 100644 index 00000000..cb81aba7 --- /dev/null +++ b/src/mqtt/m4/README @@ -0,0 +1,3 @@ +Fix the problem + +aclocal: error: couldn't open directory 'm4': No such file or directory diff --git a/src/mqtt/m4/m4_ax_cxx_compile_stdcxx.m4 b/src/mqtt/m4/m4_ax_cxx_compile_stdcxx.m4 new file mode 100644 index 00000000..2c18e49c --- /dev/null +++ b/src/mqtt/m4/m4_ax_cxx_compile_stdcxx.m4 @@ -0,0 +1,562 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CXX_COMPILE_STDCXX(VERSION, [ext|noext], [mandatory|optional]) +# +# DESCRIPTION +# +# Check for baseline language coverage in the compiler for the specified +# version of the C++ standard. If necessary, add switches to CXX and +# CXXCPP to enable support. VERSION may be '11' (for the C++11 standard) +# or '14' (for the C++14 standard). +# +# The second argument, if specified, indicates whether you insist on an +# extended mode (e.g. -std=gnu++11) or a strict conformance mode (e.g. +# -std=c++11). If neither is specified, you get whatever works, with +# preference for an extended mode. +# +# The third argument, if specified 'mandatory' or if left unspecified, +# indicates that baseline support for the specified C++ standard is +# required and that the macro should error out if no mode with that +# support is found. If specified 'optional', then configuration proceeds +# regardless, after defining HAVE_CXX${VERSION} if and only if a +# supporting mode is found. +# +# LICENSE +# +# Copyright (c) 2008 Benjamin Kosnik +# Copyright (c) 2012 Zack Weinberg +# Copyright (c) 2013 Roy Stogner +# Copyright (c) 2014, 2015 Google Inc.; contributed by Alexey Sokolov +# Copyright (c) 2015 Paul Norman +# Copyright (c) 2015 Moritz Klammler +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 4 + +dnl This macro is based on the code from the AX_CXX_COMPILE_STDCXX_11 macro +dnl (serial version number 13). + +AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl + m4_if([$1], [11], [], + [$1], [14], [], + [$1], [17], [m4_fatal([support for C++17 not yet implemented in AX_CXX_COMPILE_STDCXX])], + [m4_fatal([invalid first argument `$1' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$2], [], [], + [$2], [ext], [], + [$2], [noext], [], + [m4_fatal([invalid second argument `$2' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$3], [], [ax_cxx_compile_cxx$1_required=true], + [$3], [mandatory], [ax_cxx_compile_cxx$1_required=true], + [$3], [optional], [ax_cxx_compile_cxx$1_required=false], + [m4_fatal([invalid third argument `$3' to AX_CXX_COMPILE_STDCXX])]) + AC_LANG_PUSH([C++])dnl + ac_success=no + AC_CACHE_CHECK(whether $CXX supports C++$1 features by default, + ax_cv_cxx_compile_cxx$1, + [AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [ax_cv_cxx_compile_cxx$1=yes], + [ax_cv_cxx_compile_cxx$1=no])]) + if test x$ax_cv_cxx_compile_cxx$1 = xyes; then + ac_success=yes + fi + + m4_if([$2], [noext], [], [dnl + if test x$ac_success = xno; then + for switch in -std=gnu++$1 -std=gnu++0x; do + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + fi]) + + m4_if([$2], [ext], [], [dnl + if test x$ac_success = xno; then + dnl HP's aCC needs +std=c++11 according to: + dnl http://h21007.www2.hp.com/portal/download/files/unprot/aCxx/PDF_Release_Notes/769149-001.pdf + dnl Cray's crayCC needs "-h std=c++11" + for switch in -std=c++$1 -std=c++0x +std=c++$1 "-h std=c++$1"; do + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + fi]) + AC_LANG_POP([C++]) + if test x$ax_cxx_compile_cxx$1_required = xtrue; then + if test x$ac_success = xno; then + AC_MSG_ERROR([*** A compiler with support for C++$1 language features is required.]) + fi + fi + if test x$ac_success = xno; then + HAVE_CXX$1=0 + AC_MSG_NOTICE([No compiler with C++$1 support was found]) + else + HAVE_CXX$1=1 + AC_DEFINE(HAVE_CXX$1,1, + [define if the compiler supports basic C++$1 syntax]) + fi + AC_SUBST(HAVE_CXX$1) +]) + + +dnl Test body for checking C++11 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_11], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 +) + + +dnl Test body for checking C++14 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 +) + + +dnl Tests for new features in C++11 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_11], [[ + +// If the compiler admits that it is not ready for C++11, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201103L + +#error "This is not a C++11 compiler" + +#else + +namespace cxx11 +{ + + namespace test_static_assert + { + + template + struct check + { + static_assert(sizeof(int) <= sizeof(T), "not big enough"); + }; + + } + + namespace test_final_override + { + + struct Base + { + virtual void f() {} + }; + + struct Derived : public Base + { + virtual void f() override {} + }; + + } + + namespace test_double_right_angle_brackets + { + + template < typename T > + struct check {}; + + typedef check single_type; + typedef check> double_type; + typedef check>> triple_type; + typedef check>>> quadruple_type; + + } + + namespace test_decltype + { + + int + f() + { + int a = 1; + decltype(a) b = 2; + return a + b; + } + + } + + namespace test_type_deduction + { + + template < typename T1, typename T2 > + struct is_same + { + static const bool value = false; + }; + + template < typename T > + struct is_same + { + static const bool value = true; + }; + + template < typename T1, typename T2 > + auto + add(T1 a1, T2 a2) -> decltype(a1 + a2) + { + return a1 + a2; + } + + int + test(const int c, volatile int v) + { + static_assert(is_same::value == true, ""); + static_assert(is_same::value == false, ""); + static_assert(is_same::value == false, ""); + auto ac = c; + auto av = v; + auto sumi = ac + av + 'x'; + auto sumf = ac + av + 1.0; + static_assert(is_same::value == true, ""); + static_assert(is_same::value == true, ""); + static_assert(is_same::value == true, ""); + static_assert(is_same::value == false, ""); + static_assert(is_same::value == true, ""); + return (sumf > 0.0) ? sumi : add(c, v); + } + + } + + namespace test_noexcept + { + + int f() { return 0; } + int g() noexcept { return 0; } + + static_assert(noexcept(f()) == false, ""); + static_assert(noexcept(g()) == true, ""); + + } + + namespace test_constexpr + { + + template < typename CharT > + unsigned long constexpr + strlen_c_r(const CharT *const s, const unsigned long acc) noexcept + { + return *s ? strlen_c_r(s + 1, acc + 1) : acc; + } + + template < typename CharT > + unsigned long constexpr + strlen_c(const CharT *const s) noexcept + { + return strlen_c_r(s, 0UL); + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("1") == 1UL, ""); + static_assert(strlen_c("example") == 7UL, ""); + static_assert(strlen_c("another\0example") == 7UL, ""); + + } + + namespace test_rvalue_references + { + + template < int N > + struct answer + { + static constexpr int value = N; + }; + + answer<1> f(int&) { return answer<1>(); } + answer<2> f(const int&) { return answer<2>(); } + answer<3> f(int&&) { return answer<3>(); } + + void + test() + { + int i = 0; + const int c = 0; + static_assert(decltype(f(i))::value == 1, ""); + static_assert(decltype(f(c))::value == 2, ""); + static_assert(decltype(f(0))::value == 3, ""); + } + + } + + namespace test_uniform_initialization + { + + struct test + { + static const int zero {}; + static const int one {1}; + }; + + static_assert(test::zero == 0, ""); + static_assert(test::one == 1, ""); + + } + + namespace test_lambdas + { + + void + test1() + { + auto lambda1 = [](){}; + auto lambda2 = lambda1; + lambda1(); + lambda2(); + } + + int + test2() + { + auto a = [](int i, int j){ return i + j; }(1, 2); + auto b = []() -> int { return '0'; }(); + auto c = [=](){ return a + b; }(); + auto d = [&](){ return c; }(); + auto e = [a, &b](int x) mutable { + const auto identity = [](int y){ return y; }; + for (auto i = 0; i < a; ++i) + a += b--; + return x + identity(a + b); + }(0); + return a + b + c + d + e; + } + + int + test3() + { + const auto nullary = [](){ return 0; }; + const auto unary = [](int x){ return x; }; + using nullary_t = decltype(nullary); + using unary_t = decltype(unary); + const auto higher1st = [](nullary_t f){ return f(); }; + const auto higher2nd = [unary](nullary_t f1){ + return [unary, f1](unary_t f2){ return f2(unary(f1())); }; + }; + return higher1st(nullary) + higher2nd(nullary)(unary); + } + + } + + namespace test_variadic_templates + { + + template + struct sum; + + template + struct sum + { + static constexpr auto value = N0 + sum::value; + }; + + template <> + struct sum<> + { + static constexpr auto value = 0; + }; + + static_assert(sum<>::value == 0, ""); + static_assert(sum<1>::value == 1, ""); + static_assert(sum<23>::value == 23, ""); + static_assert(sum<1, 2>::value == 3, ""); + static_assert(sum<5, 5, 11>::value == 21, ""); + static_assert(sum<2, 3, 5, 7, 11, 13>::value == 41, ""); + + } + + // http://stackoverflow.com/questions/13728184/template-aliases-and-sfinae + // Clang 3.1 fails with headers of libstd++ 4.8.3 when using std::function + // because of this. + namespace test_template_alias_sfinae + { + + struct foo {}; + + template + using member = typename T::member_type; + + template + void func(...) {} + + template + void func(member*) {} + + void test(); + + void test() { func(0); } + + } + +} // namespace cxx11 + +#endif // __cplusplus >= 201103L + +]]) + + +dnl Tests for new features in C++14 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_14], [[ + +// If the compiler admits that it is not ready for C++14, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201402L + +#error "This is not a C++14 compiler" + +#else + +namespace cxx14 +{ + + namespace test_polymorphic_lambdas + { + + int + test() + { + const auto lambda = [](auto&&... args){ + const auto istiny = [](auto x){ + return (sizeof(x) == 1UL) ? 1 : 0; + }; + const int aretiny[] = { istiny(args)... }; + return aretiny[0]; + }; + return lambda(1, 1L, 1.0f, '1'); + } + + } + + namespace test_binary_literals + { + + constexpr auto ivii = 0b0000000000101010; + static_assert(ivii == 42, "wrong value"); + + } + + namespace test_generalized_constexpr + { + + template < typename CharT > + constexpr unsigned long + strlen_c(const CharT *const s) noexcept + { + auto length = 0UL; + for (auto p = s; *p; ++p) + ++length; + return length; + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("x") == 1UL, ""); + static_assert(strlen_c("test") == 4UL, ""); + static_assert(strlen_c("another\0test") == 7UL, ""); + + } + + namespace test_lambda_init_capture + { + + int + test() + { + auto x = 0; + const auto lambda1 = [a = x](int b){ return a + b; }; + const auto lambda2 = [a = lambda1(x)](){ return a; }; + return lambda2(); + } + + } + + namespace test_digit_seperators + { + + constexpr auto ten_million = 100'000'000; + static_assert(ten_million == 100000000, ""); + + } + + namespace test_return_type_deduction + { + + auto f(int& x) { return x; } + decltype(auto) g(int& x) { return x; } + + template < typename T1, typename T2 > + struct is_same + { + static constexpr auto value = false; + }; + + template < typename T > + struct is_same + { + static constexpr auto value = true; + }; + + int + test() + { + auto x = 0; + static_assert(is_same::value, ""); + static_assert(is_same::value, ""); + return x; + } + + } + +} // namespace cxx14 + +#endif // __cplusplus >= 201402L + +]]) diff --git a/src/mqtt/m4/m4_ax_cxx_compile_stdcxx_11.m4 b/src/mqtt/m4/m4_ax_cxx_compile_stdcxx_11.m4 new file mode 100644 index 00000000..0aadeafe --- /dev/null +++ b/src/mqtt/m4/m4_ax_cxx_compile_stdcxx_11.m4 @@ -0,0 +1,39 @@ +# ============================================================================ +# http://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx_11.html +# ============================================================================ +# +# SYNOPSIS +# +# AX_CXX_COMPILE_STDCXX_11([ext|noext], [mandatory|optional]) +# +# DESCRIPTION +# +# Check for baseline language coverage in the compiler for the C++11 +# standard; if necessary, add switches to CXX and CXXCPP to enable +# support. +# +# This macro is a convenience alias for calling the AX_CXX_COMPILE_STDCXX +# macro with the version set to C++11. The two optional arguments are +# forwarded literally as the second and third argument respectively. +# Please see the documentation for the AX_CXX_COMPILE_STDCXX macro for +# more information. If you want to use this macro, you also need to +# download the ax_cxx_compile_stdcxx.m4 file. +# +# LICENSE +# +# Copyright (c) 2008 Benjamin Kosnik +# Copyright (c) 2012 Zack Weinberg +# Copyright (c) 2013 Roy Stogner +# Copyright (c) 2014, 2015 Google Inc.; contributed by Alexey Sokolov +# Copyright (c) 2015 Paul Norman +# Copyright (c) 2015 Moritz Klammler +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 17 + +AX_REQUIRE_DEFINED([AX_CXX_COMPILE_STDCXX]) +AC_DEFUN([AX_CXX_COMPILE_STDCXX_11], [AX_CXX_COMPILE_STDCXX([11], [$1], [$2])]) diff --git a/src/mqtt/notice.html b/src/mqtt/notice.html new file mode 100644 index 00000000..f19c483b --- /dev/null +++ b/src/mqtt/notice.html @@ -0,0 +1,108 @@ + + + + + +Eclipse Foundation Software User Agreement + + + +

Eclipse Foundation Software User Agreement

+

February 1, 2011

+ +

Usage Of Content

+ +

THE ECLIPSE FOUNDATION MAKES AVAILABLE SOFTWARE, DOCUMENTATION, INFORMATION AND/OR OTHER MATERIALS FOR OPEN SOURCE PROJECTS + (COLLECTIVELY "CONTENT"). USE OF THE CONTENT IS GOVERNED BY THE TERMS AND CONDITIONS OF THIS AGREEMENT AND/OR THE TERMS AND + CONDITIONS OF LICENSE AGREEMENTS OR NOTICES INDICATED OR REFERENCED BELOW. BY USING THE CONTENT, YOU AGREE THAT YOUR USE + OF THE CONTENT IS GOVERNED BY THIS AGREEMENT AND/OR THE TERMS AND CONDITIONS OF ANY APPLICABLE LICENSE AGREEMENTS OR + NOTICES INDICATED OR REFERENCED BELOW. IF YOU DO NOT AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT AND THE TERMS AND + CONDITIONS OF ANY APPLICABLE LICENSE AGREEMENTS OR NOTICES INDICATED OR REFERENCED BELOW, THEN YOU MAY NOT USE THE CONTENT.

+ +

Applicable Licenses

+ +

Unless otherwise indicated, all Content made available by the Eclipse Foundation is provided to you under the terms and conditions of the Eclipse Public License Version 1.0 + ("EPL"). A copy of the EPL is provided with this Content and is also available at http://www.eclipse.org/legal/epl-v10.html. + For purposes of the EPL, "Program" will mean the Content.

+ +

Content includes, but is not limited to, source code, object code, documentation and other files maintained in the Eclipse Foundation source code + repository ("Repository") in software modules ("Modules") and made available as downloadable archives ("Downloads").

+ +
    +
  • Content may be structured and packaged into modules to facilitate delivering, extending, and upgrading the Content. Typical modules may include plug-ins ("Plug-ins"), plug-in fragments ("Fragments"), and features ("Features").
    • +
  • Each Plug-in or Fragment may be packaged as a sub-directory or JAR (Java™ ARchive) in a directory named "plugins".
    • +
  • A Feature is a bundle of one or more Plug-ins and/or Fragments and associated material. Each Feature may be packaged as a sub-directory in a directory named "features". Within a Feature, files named "feature.xml" may contain a list of the names and version numbers of the Plug-ins + and/or Fragments associated with that Feature.
    • +
  • Features may also include other Features ("Included Features"). Within a Feature, files named "feature.xml" may contain a list of the names and version numbers of Included Features.
    • +
+ +

The terms and conditions governing Plug-ins and Fragments should be contained in files named "about.html" ("Abouts"). The terms and conditions governing Features and +Included Features should be contained in files named "license.html" ("Feature Licenses"). Abouts and Feature Licenses may be located in any directory of a Download or Module +including, but not limited to the following locations:

+ +
    +
  • The top-level (root) directory
    • +
  • Plug-in and Fragment directories
    • +
  • Inside Plug-ins and Fragments packaged as JARs
    • +
  • Sub-directories of the directory named "src" of certain Plug-ins
    • +
  • Feature directories
    • +
+ +

Note: if a Feature made available by the Eclipse Foundation is installed using the Provisioning Technology (as defined below), you must agree to a license ("Feature Update License") during the +installation process. If the Feature contains Included Features, the Feature Update License should either provide you with the terms and conditions governing the Included Features or +inform you where you can locate them. Feature Update Licenses may be found in the "license" property of files named "feature.properties" found within a Feature. +Such Abouts, Feature Licenses, and Feature Update Licenses contain the terms and conditions (or references to such terms and conditions) that govern your use of the associated Content in +that directory.

+ +

THE ABOUTS, FEATURE LICENSES, AND FEATURE UPDATE LICENSES MAY REFER TO THE EPL OR OTHER LICENSE AGREEMENTS, NOTICES OR TERMS AND CONDITIONS. SOME OF THESE +OTHER LICENSE AGREEMENTS MAY INCLUDE (BUT ARE NOT LIMITED TO):

+ + + +

IT IS YOUR OBLIGATION TO READ AND ACCEPT ALL SUCH TERMS AND CONDITIONS PRIOR TO USE OF THE CONTENT. If no About, Feature License, or Feature Update License is provided, please +contact the Eclipse Foundation to determine what terms and conditions govern that particular Content.

+ + +

Use of Provisioning Technology

+ +

The Eclipse Foundation makes available provisioning software, examples of which include, but are not limited to, p2 and the Eclipse + Update Manager ("Provisioning Technology") for the purpose of allowing users to install software, documentation, information and/or + other materials (collectively "Installable Software"). This capability is provided with the intent of allowing such users to + install, extend and update Eclipse-based products. Information about packaging Installable Software is available at http://eclipse.org/equinox/p2/repository_packaging.html + ("Specification").

+ +

You may use Provisioning Technology to allow other parties to install Installable Software. You shall be responsible for enabling the + applicable license agreements relating to the Installable Software to be presented to, and accepted by, the users of the Provisioning Technology + in accordance with the Specification. By using Provisioning Technology in such a manner and making it available in accordance with the + Specification, you further acknowledge your agreement to, and the acquisition of all necessary rights to permit the following:

+ +
    +
  1. A series of actions may occur ("Provisioning Process") in which a user may execute the Provisioning Technology + on a machine ("Target Machine") with the intent of installing, extending or updating the functionality of an Eclipse-based + product.
    2. +
  2. During the Provisioning Process, the Provisioning Technology may cause third party Installable Software or a portion thereof to be + accessed and copied to the Target Machine.
    3. +
  3. Pursuant to the Specification, you will provide to the user the terms and conditions that govern the use of the Installable + Software ("Installable Software Agreement") and such Installable Software Agreement shall be accessed from the Target + Machine in accordance with the Specification. Such Installable Software Agreement must inform the user of the terms and conditions that govern + the Installable Software and must solicit acceptance by the end user in the manner prescribed in such Installable Software Agreement. Upon such + indication of agreement by the user, the provisioning Technology will complete installation of the Installable Software.
    4. +
+ +

Cryptography

+ +

Content may contain encryption software. The country in which you are currently may have restrictions on the import, possession, and use, and/or re-export to + another country, of encryption software. BEFORE using any encryption software, please check the country's laws, regulations and policies concerning the import, + possession, or use, and re-export of encryption software, to see if this is permitted.

+ +

Java and all Java-based trademarks are trademarks of Oracle Corporation in the United States, other countries, or both.

+ + diff --git a/src/mqtt/src/CMakeLists.txt b/src/mqtt/src/CMakeLists.txt new file mode 100644 index 00000000..850bc86e --- /dev/null +++ b/src/mqtt/src/CMakeLists.txt @@ -0,0 +1,146 @@ +# CMakeLists.txt +# +# CMake file for the Paho C++ core library. +# +# This is part of the Paho MQTT C++ client library. +# + +#******************************************************************************* +# Copyright (c) 2016-2017, Guilherme Maciel Ferreira +# Copyright (c) 2017-2018, Frank Pagliughi +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# and Eclipse Distribution License v1.0 which accompany this distribution. +# +# The Eclipse Public License is available at +# http://www.eclipse.org/legal/epl-v10.html +# and the Eclipse Distribution License is available at +# http://www.eclipse.org/org/documents/edl-v10.php. +# +# Contributors: +# Guilherme Maciel Ferreira - initial version +# Frank Pagliughi - made the shared library optional +#*******************************************************************************/ + +find_package(PahoMqttC REQUIRED) + +# --- The headers --- + +#add_subdirectory(mqtt) + +## --- Library dependencies --- + +set (THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Threads REQUIRED) + +#set(LIBS_SYSTEM Threads::Threads) +#if(WIN32) +# string(APPEND LIBS_SYSTEM " ws2_32") +#endif() + +## --- Use object library to optimize compilation --- + +add_library(paho-cpp-objs OBJECT + async_client.cpp + client.cpp + disconnect_options.cpp + iclient_persistence.cpp + message.cpp + properties.cpp + response_options.cpp + ssl_options.cpp + string_collection.cpp + subscribe_options.cpp + token.cpp + topic.cpp + connect_options.cpp + will_options.cpp +) + +## install the shared library +#install(TARGETS paho-cpp-objs EXPORT PahoMqttCpp +# OBJECTS DESTINATION ${CMAKE_INSTALL_LIBDIR} +#) + +# Object libraries can't use target_link_libraries in order to take advantage +# of transitive usage requirements until CMake 3.12. This is a workaround: +#target_include_directories(OBJS PRIVATE ${PAHO_MQTT_C_INCLUDE_DIRS}) + +target_include_directories(paho-cpp-objs + PUBLIC + $ + $ + PRIVATE + ${PAHO_MQTT_C_INCLUDE_DIRS} + src +) + + +## --- Build the shared library, if requested --- +message(STATUS "???" " " "PAHO_BUILD_STATIC:" ${PAHO_BUILD_STATIC} " " "PAHO_BUILD_SHARED:" ${PAHO_BUILD_SHARED}) + +if(PAHO_BUILD_SHARED) + ## create the shared library + add_library(paho-mqttpp3 SHARED $) + + ## add dependencies to the shared library + target_link_libraries(paho-mqttpp3 + PRIVATE ${LIBS_SYSTEM} + PUBLIC PahoMqttC::PahoMqttC Threads::Threads) + + # It would be nice to exort the include paths from the obj lib, but we + # get an export error. Perhaps in a future version? + # + # CMake Error: install(EXPORT "PahoMqttCpp" ...) includes target "paho-mqttpp3" + # which requires target "paho-cpp-objs" that is not in the export set. + + #target_include_directories(paho-mqttpp3 PUBLIC + # $ + #) + + target_include_directories(paho-mqttpp3 PUBLIC + $ + $ + ) + + ## set the shared library soname + set_target_properties(paho-mqttpp3 PROPERTIES + VERSION ${PROJECT_VERSION} + SOVERSION ${PROJECT_VERSION_MAJOR}) + + ## install the shared library + install(TARGETS paho-mqttpp3 EXPORT PahoMqttCpp + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +endif() + +## --- Build static version of the library, if requested --- + +if(PAHO_BUILD_STATIC) + ## create the static library + add_library(paho-mqttpp3-static STATIC $) + + ## add dependencies to the shared library + target_link_libraries(paho-mqttpp3-static + PRIVATE ${LIBS_SYSTEM} + PUBLIC PahoMqttC::PahoMqttC Threads::Threads) + + target_include_directories(paho-mqttpp3-static PUBLIC + $ + $ + ) + + ## install the static library + install(TARGETS paho-mqttpp3-static EXPORT PahoMqttCpp + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) + + ## Let the archive use the same name as the shared lib on *nix systems + if(UNIX) + set_target_properties(paho-mqttpp3-static PROPERTIES OUTPUT_NAME paho-mqttpp3) + endif() +endif() + + diff --git a/src/mqtt/src/async_client.cpp b/src/mqtt/src/async_client.cpp new file mode 100644 index 00000000..1a896621 --- /dev/null +++ b/src/mqtt/src/async_client.cpp @@ -0,0 +1,822 @@ +// async_client.cpp + +/******************************************************************************* + * Copyright (c) 2013-2019 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + * Frank Pagliughi - MQTT v5 support + *******************************************************************************/ + +#include "mqtt/async_client.h" +#include "mqtt/token.h" +#include "mqtt/message.h" +#include "mqtt/response_options.h" +#include "mqtt/disconnect_options.h" +#include +#include +#include +#include +#include +#include + +// TODO: Delete this when #680 is merged into the Paho C lib +#if !defined(MQTTAsync_createOptions_initializer5) + #define MQTTAsync_createOptions_initializer5 { {'M', 'Q', 'C', 'O'}, 0, 0, 100, MQTTVERSION_5 } +#endif + +namespace mqtt { + +///////////////////////////////////////////////////////////////////////////// +// Constructors + +async_client::async_client(const string& serverURI, const string& clientId, + const string& persistDir) + : async_client(serverURI, clientId, 0, persistDir) +{ +} + +async_client::async_client(const string& serverURI, const string& clientId, + iclient_persistence* persistence /*=nullptr*/) + : async_client(serverURI, clientId, 0, persistence) +{ +} + +async_client::async_client(const string& serverURI, const string& clientId, + int maxBufferedMessages, const string& persistDir) + : serverURI_(serverURI), clientId_(clientId), mqttVersion_(MQTTVERSION_DEFAULT), + persist_(nullptr), userCallback_(nullptr) +{ + MQTTAsync_createOptions opts MQTTAsync_createOptions_initializer5; + + if (maxBufferedMessages != 0) { + opts.sendWhileDisconnected = to_int(true); + opts.maxBufferedMessages = maxBufferedMessages; + } + + int rc = MQTTAsync_createWithOptions(&cli_, serverURI.c_str(), clientId.c_str(), + MQTTCLIENT_PERSISTENCE_DEFAULT, + const_cast(persistDir.c_str()), + &opts); + + if (rc != 0) + throw exception(rc); +} + +async_client::async_client(const string& serverURI, const string& clientId, + int maxBufferedMessages, iclient_persistence* persistence /*=nullptr*/) + : serverURI_(serverURI), clientId_(clientId), mqttVersion_(MQTTVERSION_DEFAULT), + persist_(nullptr), userCallback_(nullptr) +{ + MQTTAsync_createOptions opts MQTTAsync_createOptions_initializer5; + + if (maxBufferedMessages != 0) { + opts.sendWhileDisconnected = to_int(true); + opts.maxBufferedMessages = maxBufferedMessages; + } + + int rc = MQTTASYNC_SUCCESS; + + if (!persistence) { + rc = MQTTAsync_createWithOptions(&cli_, serverURI.c_str(), clientId.c_str(), + MQTTCLIENT_PERSISTENCE_NONE, nullptr, &opts); + } + else { + persist_.reset(new MQTTClient_persistence { + persistence, + &iclient_persistence::persistence_open, + &iclient_persistence::persistence_close, + &iclient_persistence::persistence_put, + &iclient_persistence::persistence_get, + &iclient_persistence::persistence_remove, + &iclient_persistence::persistence_keys, + &iclient_persistence::persistence_clear, + &iclient_persistence::persistence_containskey + }); + + rc = MQTTAsync_createWithOptions(&cli_, serverURI.c_str(), clientId.c_str(), + MQTTCLIENT_PERSISTENCE_USER, persist_.get(), + &opts); + } + if (rc != 0) + throw exception(rc); +} + +async_client::~async_client() +{ + MQTTAsync_destroy(&cli_); +} + +// -------------------------------------------------------------------------- +// Class static callbacks. +// These are the callbacks directly from the C-lib. In each case the +// 'context' should be the address of the async_client object that +// registered the callback. + + +// Callback for MQTTAsync_setConnected() +// This is installed with the normall callbacks and with a call to +// reconnect() to indicate that it succeeded. It is called after the token +// is notified of success on a normal connect with callbacks. +void async_client::on_connected(void* context, char* cause) +{ + if (context) { + async_client* cli = static_cast(context); + callback* cb = cli->userCallback_; + auto& connHandler = cli->connHandler_; + + string cause_str = cause ? string(cause) : string(); + + if (cb) + cb->connected(cause_str); + + if (connHandler) + connHandler(cause_str); + } +} + +// Callback for when the connection is lost. +// This is called from the MQTTAsync_connectionLost registered via +// MQTTAsync_setCallbacks(). +void async_client::on_connection_lost(void *context, char *cause) +{ + if (context) { + async_client* cli = static_cast(context); + callback* cb = cli->userCallback_; + consumer_queue_type& que = cli->que_; + auto& connLostHandler = cli->connLostHandler_; + + string cause_str = cause ? string(cause) : string(); + + if (cb) + cb->connection_lost(cause_str); + + if (connLostHandler) + connLostHandler(cause_str); + + if (que) + que->put(const_message_ptr{}); + } +} + +// Callback from the C lib for when a disconnect packet is received from +// the server. +void async_client::on_disconnected(void* context, MQTTProperties* cprops, + MQTTReasonCodes reasonCode) +{ + if (context) { + async_client* cli = static_cast(context); + auto& disconnectedHandler = cli->disconnectedHandler_; + + if (disconnectedHandler) { + properties props(*cprops); + disconnectedHandler(props, ReasonCode(reasonCode)); + } + } +} + +// Callback for when a subscribed message arrives. +// This is called from the MQTTAsync_messageArrived registered via +// MQTTAsync_setCallbacks(). +int async_client::on_message_arrived(void* context, char* topicName, int topicLen, + MQTTAsync_message* msg) +{ + if (context) { + async_client* cli = static_cast(context); + callback* cb = cli->userCallback_; + consumer_queue_type& que = cli->que_; + message_handler& msgHandler = cli->msgHandler_; + + if (cb || que || msgHandler) { + size_t len = (topicLen == 0) ? strlen(topicName) : size_t(topicLen); + + string topic(topicName, topicName+len); + auto m = message::create(std::move(topic), *msg); + + if (msgHandler) + msgHandler(m); + + if (cb) + cb->message_arrived(m); + + if (que) + que->put(m); + } + } + + MQTTAsync_freeMessage(&msg); + MQTTAsync_free(topicName); + + // TODO: Should the user code determine the return value? + // The Java version does doesn't seem to... + return to_int(true); +} + +// Callback to indicate that a message was delivered to the server. +// It is called for a message with a QOS >= 1, but it happens before the +// on_success() call for the token. Thus we don't have the underlying +// MQTTAsync_token of the outgoing message at the time of this callback. +// +// *** So using the Async C library we have no way to match this msgID with +// a delivery_token object. So this is useless to us. +// +// So, all in all, this callback in it's current implementation seems rather +// redundant. +// +#if 0 +void async_client::on_delivery_complete(void* context, MQTTAsync_token msgID) +{ + if (context) { + async_client* m = static_cast(context); + callback* cb = m->get_callback(); + if (cb) { + delivery_token_ptr tok = m->get_pending_delivery_token(msgID); + cb->delivery_complete(tok); + } + } +} +#endif + +// -------------------------------------------------------------------------- +// Private methods + +void async_client::add_token(token_ptr tok) +{ + if (tok) { + guard g(lock_); + pendingTokens_.push_back(tok); + } +} + +void async_client::add_token(delivery_token_ptr tok) +{ + if (tok) { + guard g(lock_); + pendingDeliveryTokens_.push_back(tok); + } +} + +// Note that we uniquely identify a token by the address of its raw pointer, +// since the message ID is not unique. + +void async_client::remove_token(token* tok) +{ + if (!tok) + return; + + guard g(lock_); + for (auto p=pendingDeliveryTokens_.begin(); + p!=pendingDeliveryTokens_.end(); ++p) { + if (p->get() == tok) { + delivery_token_ptr dtok = *p; + pendingDeliveryTokens_.erase(p); + + // If there's a user callback registered, we can now call + // delivery_complete() + + if (userCallback_) { + const_message_ptr msg = dtok->get_message(); + if (msg && msg->get_qos() > 0) { + callback* cb = userCallback_; + g.unlock(); + cb->delivery_complete(dtok); + } + } + return; + } + } + for (auto p=pendingTokens_.begin(); p!=pendingTokens_.end(); ++p) { + if (p->get() == tok) { + pendingTokens_.erase(p); + return; + } + } +} + + +// -------------------------------------------------------------------------- +// Callback management + +void async_client::set_callback(callback& cb) +{ + guard g(lock_); + userCallback_ = &cb; + + int rc = MQTTAsync_setConnected(cli_, this, &async_client::on_connected); + + if (rc == MQTTASYNC_SUCCESS) { + rc = MQTTAsync_setCallbacks(cli_, this, + &async_client::on_connection_lost, + &async_client::on_message_arrived, + nullptr /*&async_client::on_delivery_complete*/); + } + else + MQTTAsync_setConnected(cli_, nullptr, nullptr); + + if (rc != MQTTASYNC_SUCCESS) { + userCallback_ = nullptr; + throw exception(rc); + } +} + +void async_client::disable_callbacks() +{ + // TODO: It would be nice to disable callbacks at the C library level, + // but the setCallback function currently does not accept a nullptr for + // the "message arrived" parameter. So, for now we send it an empty + // lambda function. + int rc = MQTTAsync_setCallbacks(cli_, this, nullptr, + [](void*,char*,int,MQTTAsync_message*) -> int {return to_int(true);}, + nullptr); + + if (rc != MQTTASYNC_SUCCESS) + throw exception(rc); +} + +void async_client::set_connected_handler(connection_handler cb) +{ + connHandler_ = cb; + check_ret(::MQTTAsync_setConnected(cli_, this, + &async_client::on_connected)); +} + +void async_client::set_connection_lost_handler(connection_handler cb) +{ + connLostHandler_ = cb; + check_ret(::MQTTAsync_setConnectionLostCallback(cli_, this, + &async_client::on_connection_lost)); +} + +void async_client::set_disconnected_handler(disconnected_handler cb) +{ + disconnectedHandler_ = cb; + check_ret(::MQTTAsync_setDisconnected(cli_, this, + &async_client::on_disconnected)); +} + +void async_client::set_message_callback(message_handler cb) +{ + msgHandler_ = cb; + check_ret(::MQTTAsync_setMessageArrivedCallback(cli_, this, + &async_client::on_message_arrived)); +} + +// -------------------------------------------------------------------------- +// Connect + +token_ptr async_client::connect() +{ + return connect(connect_options()); +} + +token_ptr async_client::connect(connect_options opts) +{ + // TODO: We really should get (or update) this value from the response + // (when the server confirms the requested version) + mqttVersion_ = opts.opts_.MQTTVersion; + + // TODO: If connTok_ is non-null, there could be a pending connect + // which might complete after creating/assigning a new one. If that + // happened, the callback would have the context address of the previous + // token which was destroyed. So for now, keep the old one alive within + // this function, and check the behavior of the C library... + auto tmpTok = connTok_; + connTok_ = token::create(token::Type::CONNECT, *this); + add_token(connTok_); + + opts.set_token(connTok_); + + int rc = MQTTAsync_connect(cli_, &opts.opts_); + + if (rc != MQTTASYNC_SUCCESS) { + remove_token(connTok_); + connTok_.reset(); + throw exception(rc); + } + + return connTok_; +} + +token_ptr async_client::connect(connect_options opts, void* userContext, + iaction_listener& cb) +{ + // TODO: We really should get this value from the response (when + // the server confirms the requested version) + mqttVersion_ = opts.opts_.MQTTVersion; + + auto tmpTok = connTok_; + connTok_ = token::create(token::Type::CONNECT, *this, userContext, cb); + add_token(connTok_); + + opts.set_token(connTok_); + + int rc = MQTTAsync_connect(cli_, &opts.opts_); + + if (rc != MQTTASYNC_SUCCESS) { + remove_token(connTok_); + connTok_.reset(); + throw exception(rc); + } + + return connTok_; +} + +// -------------------------------------------------------------------------- +// Re-connect + +token_ptr async_client::reconnect() +{ + auto tok = connTok_; + + if (!tok) + throw exception(MQTTASYNC_FAILURE, "Can't reconnect before a successful connect"); + + tok->reset(); + add_token(tok); + + int rc = MQTTAsync_setConnected(cli_, this, &async_client::on_connected); + + if (rc == MQTTASYNC_SUCCESS) + rc = MQTTAsync_reconnect(cli_); + + if (rc != MQTTASYNC_SUCCESS) { + remove_token(tok); + throw exception(rc); + } + + return tok; +} + +// -------------------------------------------------------------------------- +// Disconnect + +token_ptr async_client::disconnect(disconnect_options opts) +{ + auto tok = token::create(token::Type::DISCONNECT, *this); + add_token(tok); + + opts.set_token(tok, mqttVersion_); + + int rc = MQTTAsync_disconnect(cli_, &opts.opts_); + + if (rc != MQTTASYNC_SUCCESS) { + remove_token(tok); + throw exception(rc); + } + + return tok; +} + +token_ptr async_client::disconnect(int timeout, void* userContext, iaction_listener& cb) +{ + auto tok = token::create(token::Type::DISCONNECT, *this, userContext, cb); + add_token(tok); + + disconnect_options opts(timeout); + opts.set_token(tok, mqttVersion_); + + int rc = MQTTAsync_disconnect(cli_, &opts.opts_); + + if (rc != MQTTASYNC_SUCCESS) { + remove_token(tok); + throw exception(rc); + } + + return tok; +} + +// -------------------------------------------------------------------------- +// Queries + +delivery_token_ptr async_client::get_pending_delivery_token(int msgID) const +{ + // Messages with QOS=1 or QOS=2 that require a response/acknowledge should + // have a non-zero 16-bit message ID. The library keeps the token objects + // for all of these messages that are in flight. When the acknowledge comes + // back from the broker, the C++ library can look up the token from the + // msgID and signal it, indicating completion. + + if (msgID > 0) { + guard g(lock_); + for (const auto& t : pendingDeliveryTokens_) { + if (t->get_message_id() == msgID) + return t; + } + } + return delivery_token_ptr(); +} + +std::vector async_client::get_pending_delivery_tokens() const +{ + std::vector toks; + guard g(lock_); + for (const auto& t : pendingDeliveryTokens_) { + if (t->get_message_id() > 0) { + toks.push_back(t); + } + } + return toks; +} + +// -------------------------------------------------------------------------- +// Publish + +delivery_token_ptr async_client::publish(string_ref topic, const void* payload, + size_t n, int qos, bool retained) +{ + auto msg = message::create(std::move(topic), payload, n, qos, retained); + return publish(std::move(msg)); +} + +delivery_token_ptr async_client::publish(string_ref topic, binary_ref payload, + int qos, bool retained) +{ + auto msg = message::create(std::move(topic), std::move(payload), qos, retained); + return publish(std::move(msg)); +} + +delivery_token_ptr async_client::publish(string_ref topic, + const void* payload, size_t n, + int qos, bool retained, void* userContext, + iaction_listener& cb) +{ + auto msg = message::create(std::move(topic), payload, n, qos, retained); + return publish(std::move(msg), userContext, cb); +} + +delivery_token_ptr async_client::publish(const_message_ptr msg) +{ + auto tok = delivery_token::create(*this, msg); + add_token(tok); + + delivery_response_options rspOpts(tok, mqttVersion_); + + int rc = MQTTAsync_sendMessage(cli_, msg->get_topic().c_str(), + &(msg->msg_), &rspOpts.opts_); + + if (rc == MQTTASYNC_SUCCESS) { + tok->set_message_id(rspOpts.opts_.token); + } + else { + remove_token(tok); + throw exception(rc); + } + + return tok; +} + +delivery_token_ptr async_client::publish(const_message_ptr msg, + void* userContext, iaction_listener& cb) +{ + delivery_token_ptr tok = delivery_token::create(*this, msg, userContext, cb); + add_token(tok); + + delivery_response_options rspOpts(tok, mqttVersion_); + + int rc = MQTTAsync_sendMessage(cli_, msg->get_topic().c_str(), + &(msg->msg_), &rspOpts.opts_); + + if (rc == MQTTASYNC_SUCCESS) { + tok->set_message_id(rspOpts.opts_.token); + } + else { + remove_token(tok); + throw exception(rc); + } + + return tok; +} + +// -------------------------------------------------------------------------- +// Subscribe + +token_ptr async_client::subscribe(const string& topicFilter, int qos, + const subscribe_options& opts /*=subscribe_options()*/) +{ + auto tok = token::create(token::Type::SUBSCRIBE, *this, topicFilter); + tok->set_num_expected(0); // Indicates non-array response for single val + add_token(tok); + + response_options rspOpts(tok, mqttVersion_); + rspOpts.set_subscribe_options(opts); + + int rc = MQTTAsync_subscribe(cli_, topicFilter.c_str(), qos, &rspOpts.opts_); + + if (rc != MQTTASYNC_SUCCESS) { + remove_token(tok); + throw exception(rc); + } + + return tok; +} + +token_ptr async_client::subscribe(const string& topicFilter, int qos, + void* userContext, iaction_listener& cb, + const subscribe_options& opts /*=subscribe_options()*/) +{ + auto tok = token::create(token::Type::SUBSCRIBE, *this, topicFilter, + userContext, cb); + tok->set_num_expected(0); + add_token(tok); + + response_options rspOpts(tok, mqttVersion_); + rspOpts.set_subscribe_options(opts); + + int rc = MQTTAsync_subscribe(cli_, topicFilter.c_str(), qos, &rspOpts.opts_); + + if (rc != MQTTASYNC_SUCCESS) { + remove_token(tok); + throw exception(rc); + } + + return tok; +} + +token_ptr async_client::subscribe(const_string_collection_ptr topicFilters, + const qos_collection& qos, + const std::vector& opts + /*=std::vector()*/) +{ + size_t n = topicFilters->size(); + + if (n != qos.size()) + throw std::invalid_argument("Collection sizes don't match"); + + auto tok = token::create(token::Type::SUBSCRIBE, *this, topicFilters); + tok->set_num_expected(n); + + add_token(tok); + + response_options rspOpts(tok, mqttVersion_); + rspOpts.set_subscribe_options(opts); + + int rc = MQTTAsync_subscribeMany(cli_, int(n), topicFilters->c_arr(), + const_cast(qos.data()), &rspOpts.opts_); + + if (rc != MQTTASYNC_SUCCESS) { + remove_token(tok); + throw exception(rc); + } + + return tok; +} + +token_ptr async_client::subscribe(const_string_collection_ptr topicFilters, + const qos_collection& qos, + void* userContext, iaction_listener& cb, + const std::vector& opts + /*=std::vector()*/) +{ + size_t n = topicFilters->size(); + + if (n != qos.size()) + throw std::invalid_argument("Collection sizes don't match"); + + auto tok = token::create(token::Type::SUBSCRIBE, *this, + topicFilters, userContext, cb); + tok->set_num_expected(n); + add_token(tok); + + response_options rspOpts(tok, mqttVersion_); + rspOpts.set_subscribe_options(opts); + + int rc = MQTTAsync_subscribeMany(cli_, int(n), topicFilters->c_arr(), + const_cast(qos.data()), &rspOpts.opts_); + + if (rc != MQTTASYNC_SUCCESS) { + remove_token(tok); + throw exception(rc); + } + + return tok; +} + +// -------------------------------------------------------------------------- +// Unsubscribe + +token_ptr async_client::unsubscribe(const string& topicFilter) +{ + auto tok = token::create(token::Type::UNSUBSCRIBE, *this, topicFilter); + tok->set_num_expected(0); // Indicates non-array response for single val + add_token(tok); + + response_options rspOpts(tok, mqttVersion_); + + int rc = MQTTAsync_unsubscribe(cli_, topicFilter.c_str(), &rspOpts.opts_); + + if (rc != MQTTASYNC_SUCCESS) { + remove_token(tok); + throw exception(rc); + } + + return tok; +} + +token_ptr async_client::unsubscribe(const_string_collection_ptr topicFilters) +{ + size_t n = topicFilters->size(); + + auto tok = token::create(token::Type::UNSUBSCRIBE, *this, topicFilters); + tok->set_num_expected(n); + add_token(tok); + + response_options rspOpts(tok, mqttVersion_); + + int rc = MQTTAsync_unsubscribeMany(cli_, int(n), + topicFilters->c_arr(), &rspOpts.opts_); + + if (rc != MQTTASYNC_SUCCESS) { + remove_token(tok); + throw exception(rc); + } + + return tok; +} + +token_ptr async_client::unsubscribe(const_string_collection_ptr topicFilters, + void* userContext, iaction_listener& cb) +{ + size_t n = topicFilters->size(); + + auto tok = token::create(token::Type::UNSUBSCRIBE, *this, topicFilters, + userContext, cb); + tok->set_num_expected(n); + add_token(tok); + + response_options rspOpts(tok, mqttVersion_); + + int rc = MQTTAsync_unsubscribeMany(cli_, int(n), topicFilters->c_arr(), &rspOpts.opts_); + + if (rc != MQTTASYNC_SUCCESS) { + remove_token(tok); + throw exception(rc); + } + + return tok; +} + +token_ptr async_client::unsubscribe(const string& topicFilter, + void* userContext, iaction_listener& cb) +{ + auto tok = token::create(token::Type::UNSUBSCRIBE , *this, topicFilter, + userContext, cb); + add_token(tok); + + response_options rspOpts(tok, mqttVersion_); + + int rc = MQTTAsync_unsubscribe(cli_, topicFilter.c_str(), &rspOpts.opts_); + + if (rc != MQTTASYNC_SUCCESS) { + remove_token(tok); + throw exception(rc); + } + + return tok; +} + +// -------------------------------------------------------------------------- + +void async_client::start_consuming() +{ + // Make sure callbacks don't happen while we update the que, etc + disable_callbacks(); + + // TODO: Should we replace user callback? + //userCallback_ = nullptr; + + que_.reset(new thread_queue); + + int rc = MQTTAsync_setCallbacks(cli_, this, + &async_client::on_connection_lost, + &async_client::on_message_arrived, + nullptr); + + if (rc != MQTTASYNC_SUCCESS) + throw exception(rc); +} + +void async_client::stop_consuming() +{ + try { + disable_callbacks(); + que_.reset(); + } + catch (...) { + que_.reset(); + throw; + } +} + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + diff --git a/src/mqtt/src/client.cpp b/src/mqtt/src/client.cpp new file mode 100644 index 00000000..8cd45771 --- /dev/null +++ b/src/mqtt/src/client.cpp @@ -0,0 +1,148 @@ +// client.cpp +// Implementation of the client class for the mqtt C++ client library. + +/******************************************************************************* + * Copyright (c) 2013-2017 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + *******************************************************************************/ + +#include "mqtt/client.h" +#include +#include + +namespace mqtt { + +const std::chrono::minutes client::DFLT_TIMEOUT = std::chrono::minutes(5); + +#if __cplusplus < 201703L + constexpr int client::DFLT_QOS; +#endif + +///////////////////////////////////////////////////////////////////////////// + + +client::client(const string& serverURI, const string& clientId, + iclient_persistence* persistence /*=nullptr*/) + : cli_(serverURI, clientId, persistence), + timeout_(DFLT_TIMEOUT), userCallback_(nullptr) +{ +} + +client::client(const string& serverURI, const string& clientId, + const string& persistDir) + : cli_(serverURI, clientId, persistDir), + timeout_(DFLT_TIMEOUT), userCallback_(nullptr) +{ +} + +client::client(const string& serverURI, const string& clientId, + int maxBufferedMessages, iclient_persistence* persistence /*=nullptr*/) + : cli_(serverURI, clientId, maxBufferedMessages, persistence), + timeout_(DFLT_TIMEOUT), userCallback_(nullptr) +{ +} + +client::client(const string& serverURI, const string& clientId, + int maxBufferedMessages, const string& persistDir) + : cli_(serverURI, clientId, maxBufferedMessages, persistDir), + timeout_(DFLT_TIMEOUT), userCallback_(nullptr) +{ +} + +void client::set_callback(callback& cb) +{ + userCallback_ = &cb; + cli_.set_callback(*this); +} + +connect_response client::connect() +{ + cli_.start_consuming(); + auto tok = cli_.connect(); + tok->wait_for(timeout_); + return tok->get_connect_response(); +} + +connect_response client::connect(connect_options opts) +{ + cli_.start_consuming(); + auto tok = cli_.connect(std::move(opts)); + tok->wait_for(timeout_); + return tok->get_connect_response(); +} + +connect_response client::reconnect() +{ + auto tok = cli_.reconnect(); + tok->wait_for(timeout_); + return tok->get_connect_response(); +} + +subscribe_response client::subscribe(const string& topicFilter) +{ + auto tok = cli_.subscribe(topicFilter, DFLT_QOS); + tok->wait_for(timeout_); + return tok->get_subscribe_response(); +} + +subscribe_response client::subscribe(const string& topicFilter, int qos) +{ + auto tok = cli_.subscribe(topicFilter, qos); + tok->wait_for(timeout_); + return tok->get_subscribe_response(); +} + +subscribe_response client::subscribe(const string_collection& topicFilters) +{ + qos_collection qos; + for (size_t i=0; iwait_for(timeout_); + return tok->get_subscribe_response(); +} + +subscribe_response client::subscribe(const string_collection& topicFilters, + const qos_collection& qos) +{ + auto tok = cli_.subscribe(ptr(topicFilters), qos); + tok->wait_for(timeout_); + return tok->get_subscribe_response(); +} + +unsubscribe_response client::unsubscribe(const string& topicFilter) +{ + auto tok = cli_.unsubscribe(topicFilter); + tok->wait_for(timeout_); + return tok->get_unsubscribe_response(); +} + +unsubscribe_response client::unsubscribe(const string_collection& topicFilters) +{ + auto tok = cli_.unsubscribe(ptr(topicFilters)); + tok->wait_for(timeout_); + return tok->get_unsubscribe_response(); +} + +void client::disconnect() +{ + cli_.disconnect()->wait_for(timeout_); + cli_.stop_consuming(); +} + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + diff --git a/src/mqtt/src/connect_options.cpp b/src/mqtt/src/connect_options.cpp new file mode 100644 index 00000000..0ce5a21a --- /dev/null +++ b/src/mqtt/src/connect_options.cpp @@ -0,0 +1,209 @@ +/******************************************************************************* + * Copyright (c) 2016 Guilherme M. Ferreira + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Guilherme M. Ferreira - initial implementation and documentation + * Frank Pagliughi - Copy and move operations. Bug fixes. + *******************************************************************************/ + +#include "mqtt/connect_options.h" +#include + +namespace mqtt { + +///////////////////////////////////////////////////////////////////////////// + +const MQTTAsync_connectOptions connect_options::DFLT_C_STRUCT = MQTTAsync_connectOptions_initializer; + +connect_options::connect_options() : opts_(DFLT_C_STRUCT) +{ +} + +connect_options::connect_options(string_ref userName, binary_ref password) + : connect_options() +{ + set_user_name(userName); + set_password(password); +} + +connect_options::connect_options(const connect_options& opt) : opts_(opt.opts_) +{ + if (opts_.will) + set_will(opt.will_); + + if (opts_.ssl) + set_ssl(opt.ssl_); + + set_user_name(opt.userName_); + set_password(opt.password_); +} + +connect_options::connect_options(connect_options&& opt) : opts_(opt.opts_), + will_(std::move(opt.will_)), + ssl_(std::move(opt.ssl_)), + userName_(std::move(opt.userName_)), + password_(std::move(opt.password_)) +{ + if (opts_.will) { + opts_.will = &will_.opts_; + opts_.willProperties = const_cast(&will_.props_.c_struct()); + } + + if (opts_.ssl) + opts_.ssl = &ssl_.opts_; + + opts_.username = c_str(userName_); + set_password(password_); +} + +connect_options& connect_options::operator=(const connect_options& opt) +{ + opts_ = opt.opts_; + + if (opts_.will) + set_will(opt.will_); + + if (opts_.ssl) + set_ssl(opt.ssl_); + + set_user_name(opt.userName_); + set_password(opt.password_); + + return *this; +} + +connect_options& connect_options::operator=(connect_options&& opt) +{ + opts_ = opt.opts_; + + if (opts_.will) + set_will(std::move(opt.will_)); + + if (opts_.ssl) + set_ssl(std::move(opt.ssl_)); + + userName_ = std::move(opt.userName_); + opts_.username = c_str(userName_); + + password_ = std::move(opt.password_); + set_password(password_); + + return *this; +} + +void connect_options::set_will(const will_options& will) +{ + will_ = will; + opts_.will = &will_.opts_; + opts_.willProperties = will_.get_properties().empty() + ? nullptr : const_cast(&will_.props_.c_struct()); +} + +void connect_options::set_will(will_options&& will) +{ + will_ = will; + opts_.will = &will_.opts_; + opts_.willProperties = will_.get_properties().empty() + ? nullptr : const_cast(&will_.props_.c_struct()); +} + +void connect_options::set_user_name(string_ref userName) +{ + userName_ = std::move(userName); + opts_.username = c_str(userName_); +} + +void connect_options::set_password(binary_ref password) +{ + password_ = std::move(password); + + if (password_.empty()) { + opts_.binarypwd.len = 0; + opts_.binarypwd.data = nullptr; + } + else { + opts_.binarypwd.len = (int) password_.size(); + opts_.binarypwd.data = password_.data(); + } +} + +void connect_options::set_ssl(const ssl_options& ssl) +{ + ssl_ = ssl; + opts_.ssl = &ssl_.opts_; +} + +void connect_options::set_ssl(ssl_options&& ssl) +{ + ssl_ = ssl; + opts_.ssl = &ssl_.opts_; +} + +void connect_options::set_token(const token_ptr& tok) +{ + tok_ = tok; + opts_.context = tok_.get(); + + opts_.onSuccess = nullptr; + opts_.onFailure = nullptr; + + opts_.onSuccess5 = nullptr; + opts_.onFailure5 = nullptr; + + if (tok) { + if (opts_.MQTTVersion < MQTTVERSION_5) { + opts_.onSuccess = &token::on_success; + opts_.onFailure = &token::on_failure; + } + else { + opts_.onSuccess5 = &token::on_success5; + opts_.onFailure5 = &token::on_failure5; + } + } +} + +void connect_options::set_servers(const_string_collection_ptr serverURIs) +{ + if (serverURIs) { + serverURIs_ = std::move(serverURIs); + opts_. serverURIcount = (int) serverURIs_->size(); + opts_.serverURIs = serverURIs_->c_arr(); + } + else { + serverURIs_.reset(); + opts_.serverURIcount = 0; + opts_.serverURIs = nullptr; + } +} + +void connect_options::set_mqtt_version(int mqttVersion) { + opts_.MQTTVersion = mqttVersion; + + if (mqttVersion < MQTTVERSION_5) + opts_.cleanstart = 0; + else + opts_.cleansession = 0; +} + +void connect_options::set_automatic_reconnect(int minRetryInterval, + int maxRetryInterval) +{ + opts_.automaticReconnect = to_int(true); + opts_.minRetryInterval = minRetryInterval; + opts_.maxRetryInterval = maxRetryInterval; +} + + +///////////////////////////////////////////////////////////////////////////// + +} // end namespace mqtt + diff --git a/src/mqtt/src/disconnect_options.cpp b/src/mqtt/src/disconnect_options.cpp new file mode 100644 index 00000000..8f52ee53 --- /dev/null +++ b/src/mqtt/src/disconnect_options.cpp @@ -0,0 +1,67 @@ +// disconnect_options.cpp + +#include "mqtt/disconnect_options.h" +#include +#include + +namespace mqtt { + +///////////////////////////////////////////////////////////////////////////// + +const MQTTAsync_disconnectOptions disconnect_options::DFLT_C_STRUCT = MQTTAsync_disconnectOptions_initializer; + +disconnect_options::disconnect_options() : opts_(DFLT_C_STRUCT) +{ +} + +disconnect_options::disconnect_options(const disconnect_options& opt) + : opts_(opt.opts_), tok_(opt.tok_) +{ +} + +disconnect_options::disconnect_options(disconnect_options&& opt) + : opts_(opt.opts_), tok_(std::move(opt.tok_)) +{ +} + +disconnect_options& disconnect_options::operator=(const disconnect_options& opt) +{ + opts_ = opt.opts_; + tok_ = opt.tok_; + return *this; +} + +disconnect_options& disconnect_options::operator=(disconnect_options&& opt) +{ + opts_ = opt.opts_; + tok_ = std::move(opt.tok_); + return *this; +} + +void disconnect_options::set_token(const token_ptr& tok, int mqttVersion) +{ + tok_ = tok; + opts_.context = tok_.get(); + + opts_.onSuccess = nullptr; + opts_.onFailure = nullptr; + + opts_.onSuccess5 = nullptr; + opts_.onFailure5 = nullptr; + + if (tok) { + if (mqttVersion >= MQTTVERSION_5) { + opts_.onSuccess5 = &token::on_success5; + opts_.onFailure5 = &token::on_failure5; + } + else { + opts_.onSuccess = &token::on_success; + opts_.onFailure = &token::on_failure; + } + } +} + +///////////////////////////////////////////////////////////////////////////// +// end namespace 'mqtt' +} + diff --git a/src/mqtt/src/iclient_persistence.cpp b/src/mqtt/src/iclient_persistence.cpp new file mode 100644 index 00000000..3f8b1a03 --- /dev/null +++ b/src/mqtt/src/iclient_persistence.cpp @@ -0,0 +1,151 @@ +// iclient_persistence.cpp + +/******************************************************************************* + * Copyright (c) 2013-2016 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + *******************************************************************************/ + +#include "mqtt/types.h" +#include "mqtt/iclient_persistence.h" +#include +#include + +namespace mqtt { + +///////////////////////////////////////////////////////////////////////////// +// Functions to transition C persistence calls to the C++ persistence object. + +// Upon the call to persistence_open(), the 'context' has the address of the +// C++ persistence object, which is reassigned to the 'handle'. Subsequent +// calls have the object address as the handle. + +int iclient_persistence::persistence_open(void** handle, const char* clientID, + const char* serverURI, void* context) +{ + try { + if (handle && clientID && serverURI && context) { + static_cast(context)->open(clientID, serverURI); + *handle = context; + return MQTTASYNC_SUCCESS; + } + } + catch (...) {} + + return MQTTCLIENT_PERSISTENCE_ERROR; +} + +int iclient_persistence::persistence_close(void* handle) +{ + try { + if (handle) { + static_cast(handle)->close(); + return MQTTASYNC_SUCCESS; + } + } + catch (...) {} + + return MQTTCLIENT_PERSISTENCE_ERROR; +} + +int iclient_persistence::persistence_put(void* handle, char* key, int bufcount, + char* buffers[], int buflens[]) +{ + try { + if (handle && bufcount > 0 && buffers && buflens) { + std::vector vec; + for (int i=0; i(handle)->put(key, vec); + return MQTTASYNC_SUCCESS; + } + } + catch (...) {} + + return MQTTCLIENT_PERSISTENCE_ERROR; +} + +int iclient_persistence::persistence_get(void* handle, char* key, + char** buffer, int* buflen) +{ + try { + if (handle && key && buffer && buflen) { + auto sv = static_cast(handle)->get(key); + *buffer = const_cast(sv.data()); + *buflen = (int) sv.length(); + return MQTTASYNC_SUCCESS; + } + } + catch (...) {} + + return MQTTCLIENT_PERSISTENCE_ERROR; +} + +int iclient_persistence::persistence_remove(void* handle, char* key) +{ + try { + if (handle && key) { + static_cast(handle)->remove(key); + return MQTTASYNC_SUCCESS; + } + } + catch (...) {} + + return MQTTCLIENT_PERSISTENCE_ERROR; +} + +int iclient_persistence::persistence_keys(void* handle, char*** keys, int* nkeys) +{ + try { + if (handle && keys && nkeys) { + auto& k = static_cast(handle)->keys(); + size_t n = k.size(); + *nkeys = n; + *keys = (n == 0) ? nullptr : const_cast(k.c_arr()); + return MQTTASYNC_SUCCESS; + } + } + catch (...) {} + + return MQTTCLIENT_PERSISTENCE_ERROR; +} + +int iclient_persistence::persistence_clear(void* handle) +{ + try { + if (handle) { + static_cast(handle)->clear(); + return MQTTASYNC_SUCCESS; + } + } + catch (...) {} + + return MQTTCLIENT_PERSISTENCE_ERROR; +} + +int iclient_persistence::persistence_containskey(void* handle, char* key) +{ + try { + if (handle && key && + static_cast(handle)->contains_key(key)) + return MQTTASYNC_SUCCESS; + } + catch (...) {} + + return MQTTCLIENT_PERSISTENCE_ERROR; +} + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + diff --git a/src/mqtt/src/message.cpp b/src/mqtt/src/message.cpp new file mode 100644 index 00000000..0a3a59c1 --- /dev/null +++ b/src/mqtt/src/message.cpp @@ -0,0 +1,122 @@ +// message.cpp + +/******************************************************************************* + * Copyright (c) 2013-2017 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + *******************************************************************************/ + + +#include "mqtt/message.h" +#include +#include + +namespace mqtt { + +///////////////////////////////////////////////////////////////////////////// + +#if __cplusplus < 201703L + constexpr int message::DFLT_QOS; + constexpr bool message::DFLT_RETAINED; +#endif + +const MQTTAsync_message message::DFLT_C_STRUCT = MQTTAsync_message_initializer; + +// -------------------------------------------------------------------------- + +message::message() : msg_(DFLT_C_STRUCT) +{ +} + +message::message(string_ref topic, const void* payload, size_t len, int qos, bool retained) + : msg_(DFLT_C_STRUCT), topic_(std::move(topic)) +{ + set_payload(payload, len); + set_qos(qos); + set_retained(retained); +} + +message::message(string_ref topic, binary_ref payload, int qos, bool retained) + : msg_(DFLT_C_STRUCT), topic_(std::move(topic)) +{ + set_payload(std::move(payload)); + set_qos(qos); + set_retained(retained); +} + +message::message(string_ref topic, const MQTTAsync_message& msg) + : msg_(msg), topic_(std::move(topic)), props_(msg.properties) +{ + msg_.properties = props_.c_struct(); + set_payload(msg.payload, msg.payloadlen); +} + +message::message(const message& other) : msg_(other.msg_), topic_(other.topic_) +{ + set_payload(other.payload_); +} + +message::message(message&& other) : msg_(other.msg_), topic_(std::move(other.topic_)) +{ + set_payload(std::move(other.payload_)); + other.msg_.payloadlen = 0; + other.msg_.payload = nullptr; +} + +message& message::operator=(const message& rhs) +{ + if (&rhs != this) { + msg_ = rhs.msg_; + topic_ = rhs.topic_; + set_payload(rhs.payload_); + } + return *this; +} + +message& message::operator=(message&& rhs) +{ + if (&rhs != this) { + msg_ = rhs.msg_; + topic_ = std::move(rhs.topic_); + set_payload(std::move(rhs.payload_)); + rhs.msg_.payloadlen = 0; + rhs.msg_.payload = nullptr; + } + return *this; +} + +void message::clear_payload() +{ + payload_.reset(); + msg_.payload = nullptr; + msg_.payloadlen = 0; +} + +void message::set_payload(binary_ref payload) +{ + payload_ = std::move(payload); + + if (payload_.empty()) { + msg_.payload = nullptr; + msg_.payloadlen = 0; + } + else { + msg_.payload = const_cast(payload_.data()); + msg_.payloadlen = payload_.length(); + } +} + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + diff --git a/src/mqtt/src/mqtt/async_client.h b/src/mqtt/src/mqtt/async_client.h new file mode 100644 index 00000000..4bbd95c1 --- /dev/null +++ b/src/mqtt/src/mqtt/async_client.h @@ -0,0 +1,710 @@ +///////////////////////////////////////////////////////////////////////////// +/// @file async_client.h +/// Declaration of MQTT async_client class +/// @date May 1, 2013 +/// @author Frank Pagliughi +///////////////////////////////////////////////////////////////////////////// + +/******************************************************************************* + * Copyright (c) 2013-2019 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + * Frank Pagliughi - MQTT v5 support + *******************************************************************************/ + +#ifndef __mqtt_async_client_h +#define __mqtt_async_client_h + +#include "MQTTAsync.h" +#include "mqtt/types.h" +#include "mqtt/token.h" +#include "mqtt/string_collection.h" +#include "mqtt/delivery_token.h" +#include "mqtt/iclient_persistence.h" +#include "mqtt/iaction_listener.h" +#include "mqtt/properties.h" +#include "mqtt/exception.h" +#include "mqtt/message.h" +#include "mqtt/callback.h" +#include "mqtt/thread_queue.h" +#include "mqtt/iasync_client.h" +#include +#include +#include +#include +#include +#include + +namespace mqtt { + +// OBSOLETE: The legacy constants that lacked the "PAHO_MQTTPP_" prefix +// clashed with #define's from other libraries and will be removed at the +// next major version upgrade. + +#if defined(PAHO_MQTTPP_VERSIONS) + /** The version number for the client library. */ + const uint32_t PAHO_MQTTPP_VERSION = 0x01010000; + /** The version string for the client library */ + const string PAHO_MQTTPP_VERSION_STR("Paho MQTT C++ (mqttpp) v. 1.1"); + /** Copyright notice for the client library */ + const string PAHO_MQTTPP_COPYRIGHT("Copyright (c) 2013-2019 Frank Pagliughi"); +#else + /** The version number for the client library. */ + const uint32_t VERSION = 0x01010000; + /** The version string for the client library */ + const string VERSION_STR("Paho MQTT C++ (mqttpp) v. 1.1"); + /** Copyright notice for the client library */ + const string COPYRIGHT("Copyright (c) 2013-2019 Frank Pagliughi"); +#endif + +///////////////////////////////////////////////////////////////////////////// + +/** + * Lightweight client for talking to an MQTT server using non-blocking + * methods that allow an operation to run in the background. + */ +class async_client : public virtual iasync_client +{ +public: + /** Smart/shared pointer for an object of this class */ + using ptr_t = std::shared_ptr; + /** Type for a thread-safe queue to consume messages synchronously */ + using consumer_queue_type = std::unique_ptr>; + + /** Handler type for registering an individual message callback */ + using message_handler = std::function; + /** Handler type for when a connecion is made or lost */ + using connection_handler = std::function; + /** Handler type for when a disconnect packet is received */ + using disconnected_handler = std::function; + +private: + /** Lock guard type for this class */ + using guard = std::unique_lock; + /** Unique lock type for this class */ + using unique_lock = std::unique_lock; + + /** Object monitor mutex */ + mutable std::mutex lock_; + /** The underlying C-lib client. */ + MQTTAsync cli_; + /** The server URI string. */ + string serverURI_; + /** The client ID string that we provided to the server. */ + string clientId_; + /** The MQTT protocol version we're connected at */ + int mqttVersion_; + /** A user persistence wrapper (if any) */ + std::unique_ptr persist_; + /** Callback supplied by the user (if any) */ + callback* userCallback_; + /** Connection handler */ + connection_handler connHandler_; + /** Connection lost handler */ + connection_handler connLostHandler_; + /** Disconnected handler */ + disconnected_handler disconnectedHandler_; + /** Message handler (if any) */ + message_handler msgHandler_; + /** Copy of connect token (for re-connects) */ + token_ptr connTok_; + /** A list of tokens that are in play */ + std::list pendingTokens_; + /** A list of delivery tokens that are in play */ + std::list pendingDeliveryTokens_; + /** A queue of messages for consumer API */ + consumer_queue_type que_; + + /** Callbacks from the C library */ + static void on_connected(void* context, char* cause); + static void on_connection_lost(void *context, char *cause); + static void on_disconnected(void* context, MQTTProperties* cprops, + MQTTReasonCodes reasonCode); + static int on_message_arrived(void* context, char* topicName, int topicLen, + MQTTAsync_message* msg); + static void on_delivery_complete(void* context, MQTTAsync_token tok); + + /** Manage internal list of active tokens */ + friend class token; + virtual void add_token(token_ptr tok); + virtual void add_token(delivery_token_ptr tok); + virtual void remove_token(token* tok) override; + virtual void remove_token(token_ptr tok) { remove_token(tok.get()); } + void remove_token(delivery_token_ptr tok) { remove_token(tok.get()); } + + /** Non-copyable */ + async_client() =delete; + async_client(const async_client&) =delete; + async_client& operator=(const async_client&) =delete; + + /** Checks a function return code and throws on error. */ + static void check_ret(int rc) { + if (rc != MQTTASYNC_SUCCESS) + throw exception(rc); + } + +public: + /** + * Create an async_client that can be used to communicate with an MQTT + * server. + * This uses file-based persistence in the specified directory. + * @param serverURI the address of the server to connect to, specified + * as a URI. + * @param clientId a client identifier that is unique on the server + * being connected to + * @param persistDir The directory to use for persistence data + * @throw exception if an argument is invalid + */ + async_client(const string& serverURI, const string& clientId, + const string& persistDir); + /** + * Create an async_client that can be used to communicate with an MQTT + * server. + * This allows the caller to specify a user-defined persistence object, + * or use no persistence. + * @param serverURI the address of the server to connect to, specified + * as a URI. + * @param clientId a client identifier that is unique on the server + * being connected to + * @param persistence The user persistence structure. If this is null, + * then no persistence is used. + * @throw exception if an argument is invalid + */ + async_client(const string& serverURI, const string& clientId, + iclient_persistence* persistence=nullptr); + /** + * Create an async_client that can be used to communicate with an MQTT + * server, which allows for off-line message buffering. + * This uses file-based persistence in the specified directory. + * @param serverURI the address of the server to connect to, specified + * as a URI. + * @param clientId a client identifier that is unique on the server + * being connected to + * @param maxBufferedMessages the maximum number of messages allowed to + * be buffered while not connected + * @param persistDir The directory to use for persistence data + * @throw exception if an argument is invalid + */ + async_client(const string& serverURI, const string& clientId, + int maxBufferedMessages, const string& persistDir); + /** + * Create an async_client that can be used to communicate with an MQTT + * server, which allows for off-line message buffering. + * This allows the caller to specify a user-defined persistence object, + * or use no persistence. + * @param serverURI the address of the server to connect to, specified + * as a URI. + * @param clientId a client identifier that is unique on the server + * being connected to + * @param maxBufferedMessages the maximum number of messages allowed to + * be buffered while not connected + * @param persistence The user persistence structure. If this is null, + * then no persistence is used. + * @throw exception if an argument is invalid + */ + async_client(const string& serverURI, const string& clientId, + int maxBufferedMessages, iclient_persistence* persistence=nullptr); + /** + * Destructor + */ + ~async_client() override; + /** + * Sets a callback listener to use for events that happen + * asynchronously. + * @param cb callback receiver which will be invoked for certain + * asynchronous events + */ + void set_callback(callback& cb) override; + /** + * Stops callbacks. + * This is not normally called by the application. It should be used + * cautiously as it may cause the application to lose messages. + */ + void disable_callbacks() override; + /** + * Callback for when a connection is made. + * @param cb Callback functor for when the connection is made. + */ + void set_connected_handler(connection_handler cb) /*override*/; + /** + * Callback for when a connection is lost. + * @param cb Callback functor for when the connection is lost. + */ + void set_connection_lost_handler(connection_handler cb) /*override*/; + /** + * Callback for when a disconnect packet is received from the server. + * @param cb Callback for when the disconnect packet is received. + */ + void set_disconnected_handler(disconnected_handler cb) /*override*/; + /** + * Sets the callback for when a message arrives from the broker. + * Note that the application can only have one message handler which can + * be installed individually using this method, or installled as a + * listener object. + * @param cb The callback functor to register with the library. + */ + void set_message_callback(message_handler cb) /*override*/; + /** + * Connects to an MQTT server using the default options. + * @return token used to track and wait for the connect to complete. The + * token will be passed to any callback that has been set. + * @throw exception for non security related problems + * @throw security_exception for security related problems + */ + token_ptr connect() override; + /** + * Connects to an MQTT server using the provided connect options. + * @param options a set of connection parameters that override the + * defaults. + * @return token used to track and wait for the connect to complete. The + * token will be passed to any callback that has been set. + * @throw exception for non security related problems + * @throw security_exception for security related problems + */ + token_ptr connect(connect_options options) override; + /** + * Connects to an MQTT server using the specified options. + * @param options a set of connection parameters that override the + * defaults. + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param cb callback listener that will be notified when the connect + * completes. + * @return token used to track and wait for the connect to complete. The + * token will be passed to any callback that has been set. + * @throw exception for non security related problems + * @throw security_exception for security related problems + */ + token_ptr connect(connect_options options, void* userContext, + iaction_listener& cb) override; + /** + * + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param cb callback listener that will be notified when the connect + * completes. + * @return token used to track and wait for the connect to complete. The + * token will be passed to any callback that has been set. + * @throw exception for non security related problems + * @throw security_exception for security related problems + */ + token_ptr connect(void* userContext, iaction_listener& cb) override { + return connect(connect_options{}, userContext, cb); + } + /** + * Reconnects the client using options from the previous connect. + * The client must have previously called connect() for this to work. + * @return token used to track the progress of the reconnect. + */ + token_ptr reconnect() override; + /** + * Disconnects from the server. + * @return token used to track and wait for the disconnect to complete. + * The token will be passed to any callback that has been set. + * @throw exception for problems encountered while disconnecting + */ + token_ptr disconnect() override { return disconnect(disconnect_options()); } + /** + * Disconnects from the server. + * @param opts Options for disconnecting. + * @return token used to track and wait for the disconnect to complete. + * The token will be passed to any callback that has been set. + * @throw exception for problems encountered while disconnecting + */ + token_ptr disconnect(disconnect_options opts) override; + /** + * Disconnects from the server. + * @param timeout the amount of time in milliseconds to allow for + * existing work to finish before disconnecting. A value + * of zero or less means the client will not quiesce. + * @return Token used to track and wait for disconnect to complete. The + * token will be passed to the callback methods if a callback is + * set. + * @throw exception for problems encountered while disconnecting + */ + token_ptr disconnect(int timeout) override { + return disconnect(disconnect_options(timeout)); + } + /** + * Disconnects from the server. + * @param timeout the amount of time in milliseconds to allow for + * existing work to finish before disconnecting. A value + * of zero or less means the client will not quiesce. + * @return Token used to track and wait for disconnect to complete. The + * token will be passed to the callback methods if a callback is + * set. + * @throw exception for problems encountered while disconnecting + */ + template + token_ptr disconnect(const std::chrono::duration& timeout) { + // TODO: check range + return disconnect((int) to_milliseconds_count(timeout)); + } + /** + * Disconnects from the server. + * @param timeout the amount of time in milliseconds to allow for + * existing work to finish before disconnecting. A value + * of zero or less means the client will not quiesce. + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param cb callback listener that will be notified when the disconnect + * completes. + * @return token_ptr Token used to track and wait for disconnect to + * complete. The token will be passed to the callback methods if + * a callback is set. + * @throw exception for problems encountered while disconnecting + */ + token_ptr disconnect(int timeout, void* userContext, + iaction_listener& cb) override; + /** + * Disconnects from the server. + * @param timeout the amount of time in milliseconds to allow for + * existing work to finish before disconnecting. A value + * of zero or less means the client will not quiesce. + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param cb callback listener that will be notified when the disconnect + * completes. + * @return token_ptr Token used to track and wait for disconnect to + * complete. The token will be passed to the callback methods if + * a callback is set. + * @throw exception for problems encountered while disconnecting + */ + template + token_ptr disconnect(const std::chrono::duration& timeout, + void* userContext, iaction_listener& cb) { + // TODO: check range + return disconnect((int) to_milliseconds_count(timeout), userContext, cb); + } + /** + * Disconnects from the server. + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param cb callback listener that will be notified when the disconnect + * completes. + * @return token_ptr Token used to track and wait for disconnect to + * complete. The token will be passed to the callback methods if + * a callback is set. + * @throw exception for problems encountered while disconnecting + */ + token_ptr disconnect(void* userContext, iaction_listener& cb) override { + return disconnect(0L, userContext, cb); + } + /** + * Returns the delivery token for the specified message ID. + * @return delivery_token + */ + delivery_token_ptr get_pending_delivery_token(int msgID) const override; + /** + * Returns the delivery tokens for any outstanding publish operations. + * @return delivery_token[] + */ + std::vector get_pending_delivery_tokens() const override; + /** + * Returns the client ID used by this client. + * @return The client ID used by this client. + */ + string get_client_id() const override { return clientId_; } + /** + * Returns the address of the server used by this client. + * @return The server's address, as a URI String. + */ + string get_server_uri() const override { return serverURI_; } + /** + * Determines if this client is currently connected to the server. + * @return true if connected, false otherwise. + */ + bool is_connected() const override { return to_bool(MQTTAsync_isConnected(cli_)); } + /** + * Publishes a message to a topic on the server + * @param topic The topic to deliver the message to + * @param payload the bytes to use as the message payload + * @param n the number of bytes in the payload + * @param qos the Quality of Service to deliver the message at. Valid + * values are 0, 1 or 2. + * @param retained whether or not this message should be retained by the + * server. + * @return token used to track and wait for the publish to complete. The + * token will be passed to callback methods if set. + */ + delivery_token_ptr publish(string_ref topic, const void* payload, size_t n, + int qos, bool retained) override; + /** + * Publishes a message to a topic on the server + * @param topic The topic to deliver the message to + * @param payload the bytes to use as the message payload + * @param n the number of bytes in the payload + * @return token used to track and wait for the publish to complete. The + * token will be passed to callback methods if set. + */ + delivery_token_ptr publish(string_ref topic, const void* payload, size_t n) override { + return publish(std::move(topic), payload, n, + message::DFLT_QOS, message::DFLT_RETAINED); + } + /** + * Publishes a message to a topic on the server + * @param topic The topic to deliver the message to + * @param payload the bytes to use as the message payload + * @param qos the Quality of Service to deliver the message at. Valid + * values are 0, 1 or 2. + * @param retained whether or not this message should be retained by the + * server. + * @return token used to track and wait for the publish to complete. The + * token will be passed to callback methods if set. + */ + delivery_token_ptr publish(string_ref topic, binary_ref payload, + int qos, bool retained) override; + /** + * Publishes a message to a topic on the server + * @param topic The topic to deliver the message to + * @param payload the bytes to use as the message payload + * @return token used to track and wait for the publish to complete. The + * token will be passed to callback methods if set. + */ + delivery_token_ptr publish(string_ref topic, binary_ref payload) override { + return publish(std::move(topic), std::move(payload), + message::DFLT_QOS, message::DFLT_RETAINED); + } + /** + * Publishes a message to a topic on the server + * @param topic The topic to deliver the message to + * @param payload the bytes to use as the message payload + * @param n the number of bytes in the payload + * @param qos the Quality of Service to deliver the message at. Valid + * values are 0, 1 or 2. + * @param retained whether or not this message should be retained by the + * server. + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param cb + * @return token used to track and wait for the publish to complete. The + * token will be passed to callback methods if set. + */ + delivery_token_ptr publish(string_ref topic, + const void* payload, size_t n, + int qos, bool retained, + void* userContext, iaction_listener& cb) override; + /** + * Publishes a message to a topic on the server Takes an Message + * message and delivers it to the server at the requested quality of + * service. + * @param msg the message to deliver to the server + * @return token used to track and wait for the publish to complete. The + * token will be passed to callback methods if set. + */ + delivery_token_ptr publish(const_message_ptr msg) override; + /** + * Publishes a message to a topic on the server. + * @param msg the message to deliver to the server + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param cb callback optional listener that will be notified when message + * delivery has completed to the requested quality of + * service + * @return token used to track and wait for the publish to complete. The + * token will be passed to callback methods if set. + */ + delivery_token_ptr publish(const_message_ptr msg, + void* userContext, iaction_listener& cb) override; + /** + * Subscribe to a topic, which may include wildcards. + * @param topicFilter the topic to subscribe to, which can include + * wildcards. + * @param qos + * + * @return token used to track and wait for the subscribe to complete. + * The token will be passed to callback methods if set. + */ + token_ptr subscribe(const string& topicFilter, int qos, + const subscribe_options& opts=subscribe_options()) override; + /** + * Subscribe to a topic, which may include wildcards. + * @param topicFilter the topic to subscribe to, which can include + * wildcards. + * @param qos the maximum quality of service at which to subscribe. + * Messages published at a lower quality of service will be + * received at the published QoS. Messages published at a + * higher quality of service will be received using the QoS + * specified on the subscribe. + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param cb listener that will be notified when subscribe has completed + * @return token used to track and wait for the subscribe to complete. + * The token will be passed to callback methods if set. + */ + token_ptr subscribe(const string& topicFilter, int qos, + void* userContext, iaction_listener& cb, + const subscribe_options& opts=subscribe_options()) override; + /** + * Subscribe to multiple topics, each of which may include wildcards. + * @param topicFilters + * @param qos the maximum quality of service at which to subscribe. + * Messages published at a lower quality of service will be + * received at the published QoS. Messages published at a + * higher quality of service will be received using the QoS + * specified on the subscribe. + * @return token used to track and wait for the subscribe to complete. + * The token will be passed to callback methods if set. + */ + token_ptr subscribe(const_string_collection_ptr topicFilters, + const qos_collection& qos, + const std::vector& opts=std::vector()) override; + /** + * Subscribes to multiple topics, each of which may include wildcards. + * @param topicFilters + * @param qos the maximum quality of service at which to subscribe. + * Messages published at a lower quality of service will be + * received at the published QoS. Messages published at a + * higher quality of service will be received using the QoS + * specified on the subscribe. + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param cb listener that will be notified when subscribe has completed + * @return token used to track and wait for the subscribe to complete. + * The token will be passed to callback methods if set. + */ + token_ptr subscribe(const_string_collection_ptr topicFilters, + const qos_collection& qos, + void* userContext, iaction_listener& cb, + const std::vector& opts=std::vector()) override; + /** + * Requests the server unsubscribe the client from a topic. + * @param topicFilter the topic to unsubscribe from. It must match a + * topicFilter specified on an earlier subscribe. + * @return token used to track and wait for the unsubscribe to complete. + * The token will be passed to callback methods if set. + */ + token_ptr unsubscribe(const string& topicFilter) override; + /** + * Requests the server unsubscribe the client from one or more topics. + * @param topicFilters one or more topics to unsubscribe from. Each + * topicFilter must match one specified on an + * earlier subscribe. + * @return token used to track and wait for the unsubscribe to complete. + * The token will be passed to callback methods if set. + */ + token_ptr unsubscribe(const_string_collection_ptr topicFilters) override; + /** + * Requests the server unsubscribe the client from one or more topics. + * @param topicFilters + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param cb listener that will be notified when unsubscribe has + * completed + * @return token used to track and wait for the unsubscribe to complete. + * The token will be passed to callback methods if set. + */ + token_ptr unsubscribe(const_string_collection_ptr topicFilters, + void* userContext, iaction_listener& cb) override; + /** + * Requests the server unsubscribe the client from a topics. + * @param topicFilter the topic to unsubscribe from. It must match a + * topicFilter specified on an earlier subscribe. + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param cb listener that will be notified when unsubscribe has + * completed + * @return token used to track and wait for the unsubscribe to complete. + * The token will be passed to callback methods if set. + */ + token_ptr unsubscribe(const string& topicFilter, + void* userContext, iaction_listener& cb) override; + + /** + * Start consuming messages. + * This initializes the client to receive messages through a queue that + * can be read synchronously. + */ + void start_consuming(); + /** + * Stop consuming messages. + * This shuts down the internal callback and discards any unread + * messages. + */ + void stop_consuming(); + /** + * Read the next message from the queue. + * This blocks until a new message arrives. + * @return The message and topic. + */ + const_message_ptr consume_message() { return que_->get(); } + /** + * Try to read the next message from the queue without blocking. + * @param msg Pointer to the value to receive the message + * @return @em true is a message was read, @em false if no message was + * available. + */ + bool try_consume_message(const_message_ptr* msg) { + return que_->try_get(msg); + } + /** + * Waits a limited time for a message to arrive. + * @param msg Pointer to the value to receive the message + * @param relTime The maximum amount of time to wait for a message. + * @return @em true if a message was read, @em false if a timeout + * occurred. + */ + template + bool try_consume_message_for(const_message_ptr* msg, + const std::chrono::duration& relTime) { + return que_->try_get_for(msg, relTime); + } + /** + * Waits a limited time for a message to arrive. + * @param relTime The maximum amount of time to wait for a message. + * @return A shared pointer to the message that was received. It will be + * empty on timeout. + */ + template + const_message_ptr try_consume_message_for(const std::chrono::duration& relTime) { + const_message_ptr msg; + que_->try_get_for(&msg, relTime); + return msg; + } + /** + * Waits until a specific time for a message to appear. + * @param msg Pointer to the value to receive the message + * @param absTime The time point to wait until, before timing out. + * @return @em true if a message was read, @em false if a timeout + * occurred. + */ + template + bool try_consume_message_until(const_message_ptr* msg, + const std::chrono::time_point& absTime) { + return que_->try_get_until(msg, absTime); + } + /** + * Waits until a specific time for a message to appear. + * @param msg Pointer to the value to receive the message + * @param absTime The time point to wait until, before timing out. + * @return @em true if a message was read, @em false if a timeout + * occurred. + */ + template + const_message_ptr try_consume_message_until(const std::chrono::time_point& absTime) { + const_message_ptr msg; + que_->try_get_until(msg, absTime); + return msg; + } + +}; + +/** Smart/shared pointer to an asynchronous MQTT client object */ +using async_client_ptr = async_client::ptr_t; + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + +#endif // __mqtt_async_client_h + diff --git a/src/mqtt/src/mqtt/buffer_ref.h b/src/mqtt/src/mqtt/buffer_ref.h new file mode 100644 index 00000000..4db9c747 --- /dev/null +++ b/src/mqtt/src/mqtt/buffer_ref.h @@ -0,0 +1,305 @@ +///////////////////////////////////////////////////////////////////////////// +/// @file buffer_ref.h +/// Buffer reference type for the Paho MQTT C++ library. +/// @date April 18, 2017 +/// @author Frank Pagliughi +///////////////////////////////////////////////////////////////////////////// + +/******************************************************************************* + * Copyright (c) 2017 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + *******************************************************************************/ + +#ifndef __mqtt_buffer_ref_h +#define __mqtt_buffer_ref_h + +#include "mqtt/types.h" +#include +#include + +namespace mqtt { + +///////////////////////////////////////////////////////////////////////////// + +/** + * A reference object for holding immutable data buffers, with cheap copy + * semantics and lifetime management. + * + * Each object of this class contains a reference-counted pointer to an + * immutable data buffer. Objects can be copied freely and easily, even + * across threads, since all instances promise not to modify the contents + * of the buffer. + * + * The buffer is immutable but the reference itself acts like a normal + * variable. It can be reassigned to point to a different buffer. + * + * If no value has been assigned to a reference, then it is in a default + * "null" state. It is not safe to call any member functions on a null + * reference, other than to check if the object is null or empty. + * @verbatim + * string_ref sr; + * if (!sr) + * cout << "null reference" << endl; + * else + * cout.write(sr.data(), sr.size()); + * @endverbatim + */ +template +class buffer_ref +{ +public: + /** + * The underlying type for the buffer. + * Normally byte-wide data (char or uint8_t) for Paho. + */ + using value_type = T; + /** + * The type for the buffer. + * We use basic_string for compatibility with string data. + */ + using blob = std::basic_string; + /** + * The pointer we use. + * Note that it is a pointer to a _const_ blob. + */ + using pointer_type = std::shared_ptr; + +private: + /** Our data is a shared pointer to a const buffer */ + pointer_type data_; + +public: + /** + * Default constructor creates a null reference. + */ + buffer_ref() =default; + /** + * Copy constructor only copies a shared pointer. + * @param buf Another buffer reference. + */ + buffer_ref(const buffer_ref& buf) =default; + /** + * Move constructor only moves a shared pointer. + * @param buf Another buffer reference. + */ + buffer_ref(buffer_ref&& buf) =default; + /** + * Creates a reference to a new buffer by copying data. + * @param b A string from which to create a new buffer. + */ + buffer_ref(const blob& b) : data_{std::make_shared(b)} {} + /** + * Creates a reference to a new buffer by moving a string into the + * buffer. + * @param b A string from which to create a new buffer. + */ + buffer_ref(blob&& b) : data_{std::make_shared(std::move(b))} {} + /** + * Creates a reference to an existing buffer by copying the shared + * pointer. + * Note that it is up to the caller to insure that there are no mutable + * references to the buffer. + * @param p A shared pointer to a string. + */ + buffer_ref(const pointer_type& p) : data_(p) {} + /** + * Creates a reference to an existing buffer by moving the shared + * pointer. + * Note that it is up to the caller to insure that there are no mutable + * references to the buffer. + * @param p A shared pointer to a string. + */ + buffer_ref(pointer_type&& p) : data_(std::move(p)) {} + /** + * Creates a reference to a new buffer containing a copy of the data. + * @param buf The memory to copy + * @param n The number of bytes to copy. + */ + buffer_ref(const value_type* buf, size_t n) : data_{std::make_shared(buf,n)} {} + /** + * Creates a reference to a new buffer containing a copy of the + * NUL-terminated char array. + * @param buf A NUL-terminated char array (C string). + */ + buffer_ref(const char* buf) : buffer_ref(reinterpret_cast(buf), + std::strlen(buf)) { + static_assert(sizeof(char) == sizeof(T), "can only use C arr with char or byte buffers"); + } + + /** + * Copy the reference to the buffer. + * @param rhs Another buffer + * @return A reference to this object + */ + buffer_ref& operator=(const buffer_ref& rhs) =default; + /** + * Move a reference to a buffer. + * @param rhs The other reference to move. + * @return A reference to this object. + */ + buffer_ref& operator=(buffer_ref&& rhs) =default; + /** + * Copy a string into this object, creating a new buffer. + * Modifies the reference for this object, pointing it to a + * newly-created buffer. Other references to the old object remain + * unchanges, so this follows copy-on-write semantics. + * @param b A new blob/string to copy. + * @return A reference to this object. + */ + buffer_ref& operator=(const blob& b) { + data_.reset(new blob(b)); + return *this; + } + /** + * Move a string into this object, creating a new buffer. + * Modifies the reference for this object, pointing it to a + * newly-created buffer. Other references to the old object remain + * unchanges, so this follows copy-on-write semantics. + * @param b A new blob/string to move. + * @return A reference to this object. + */ + buffer_ref& operator=(blob&& b) { + data_.reset(new blob(std::move(b))); + return *this; + } + /** + * Copy a NUL-terminated C char array into a new buffer + * @param cstr A NUL-terminated C string. + * @return A reference to this object + */ + buffer_ref& operator=(const char* cstr) { + static_assert(sizeof(char) == sizeof(T), "can only use C arr with char or byte buffers"); + data_.reset(new blob(reinterpret_cast(cstr), strlen(cstr))); + return *this; + } + /** + * Copy another type of buffer reference to this one. + * This can copy a buffer of different types, provided that the size of + * the data elements are the same. This is typically used to convert + * from char to byte, where the data is the same, but the interpretation + * is different. Note that this copies the underlying buffer. + * @param rhs A reference to a different type of buffer. + * @return A reference to this object. + */ + template + buffer_ref& operator=(const buffer_ref& rhs) { + static_assert(sizeof(OT) == sizeof(T), "Can only assign buffers if values the same size"); + data_.reset(new blob(reinterpret_cast(rhs.data()), rhs.size())); + return *this; + } + /** + * Clears the reference to nil. + */ + void reset() { data_.reset(); } + /** + * Determines if the reference is valid. + * If the reference is invalid then it is not safe to call @em any + * member functions other than @ref is_null() and @ref empty() + * @return @em true if referring to a valid buffer, @em false if the + * reference (pointer) is null. + */ + explicit operator bool() const { return bool(data_); } + /** + * Determines if the reference is invalid. + * If the reference is invalid then it is not safe to call @em any + * member functions other than @ref is_null() and @ref empty() + * @return @em true if the reference is null, @em false if it is + * referring to a valid buffer, + */ + bool is_null() const { return !data_; } + /** + * Determines if the buffer is empty. + * @return @em true if the buffer is empty or thr reference is null, @em + * false if the buffer contains data. + */ + bool empty() const { return !data_ || data_->empty(); } + /** + * Gets a const pointer to the data buffer. + * @return A pointer to the data buffer. + */ + const value_type* data() const { return data_->data(); } + /** + * Gets the size of the data buffer. + * @return The size of the data buffer. + */ + size_t size() const { return data_->size(); } + /** + * Gets the size of the data buffer. + * @return The size of the data buffer. + */ + size_t length() const { return data_->length(); } + /** + * Gets the data buffer as a string. + * @return The data buffer as a string. + */ + const blob& str() const { return *data_; } + /** + * Gets the data buffer as a string. + * @return The data buffer as a string. + */ + const blob& to_string() const { return str(); } + /** + * Gets the data buffer as NUL-terminated C string. + * Note that the reference must be set to call this function. + * @return The data buffer as a string. + */ + const char* c_str() const { return data_->c_str(); } + /** + * Gets a shared pointer to the (const) data buffer. + * @return A shared pointer to the (const) data buffer. + */ + const pointer_type& ptr() const { return data_; } + /** + * Gets elemental access to the data buffer (read only) + * @param i The index into the buffer. + * @return The value at the specified index. + */ + const value_type& operator[](size_t i) const { return (*data_)[i]; } +}; + +/** + * Stream inserter for a buffer reference. + * This does a binary write of the data in the buffer. + * @param os The output stream. + * @param buf The buffer reference to write. + * @return A reference to the output stream. + */ +template +std::ostream& operator<<(std::ostream& os, const buffer_ref& buf) { + if (!buf.empty()) + os.write(buf.data(), buf.size()); + return os; +} + +///////////////////////////////////////////////////////////////////////////// + +/** + * A refernce to a text buffer. + */ +using string_ref = buffer_ref; + +/** + * A reference to a binary buffer. + * Note that we're using char for the underlying data type to allow + * efficient moves to and from std::string's. Using a separate type + * indicates that the data may be arbitrary binary. + */ +using binary_ref = buffer_ref; + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + +#endif // __mqtt_buffer_ref_h + diff --git a/src/mqtt/src/mqtt/buffer_view.h b/src/mqtt/src/mqtt/buffer_view.h new file mode 100644 index 00000000..9284ecf0 --- /dev/null +++ b/src/mqtt/src/mqtt/buffer_view.h @@ -0,0 +1,119 @@ +///////////////////////////////////////////////////////////////////////////// +/// @file buffer_view.h +/// Buffer reference type for the Paho MQTT C++ library. +/// @date April 18, 2017 +/// @author Frank Pagliughi +///////////////////////////////////////////////////////////////////////////// + +/******************************************************************************* + * Copyright (c) 2017 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + *******************************************************************************/ + +#ifndef __mqtt_buffer_view_h +#define __mqtt_buffer_view_h + +#include "mqtt/types.h" + +namespace mqtt { + +///////////////////////////////////////////////////////////////////////////// + +/** + * A reference to a contiguous sequence of items, with no ownership. + * This simply contains a pointer to a const array of items, and a size. + * This is a similar, but simplified version of the std::string_view + * class(es) in the C++17 standard. + */ +template +class buffer_view +{ +public: + /** The type of items to be held in the queue. */ + using value_type = T; + /** The type used to specify number of items in the container. */ + using size_type = size_t; + +private: + /** Const pointer to the data array */ + const value_type* data_; + /** The size of the array */ + size_type sz_; + +public: + /** + * Constructs a buffer view. + * @param data The data pointer + * @param n The number of items + */ + buffer_view(const value_type* data, size_type n) + : data_(data), sz_(n) {} + /** + * Constructs a buffer view to a whole string. + * This the starting pointer and length of the whole string. + * @param str The string. + */ + buffer_view(const std::basic_string& str) + : data_(str.data()), sz_(str.size()) {} + /** + * Gets a pointer the first item in the view. + * @return A const pointer the first item in the view. + */ + const value_type* data() const { return data_; } + /** + * Gets the number of items in the view. + * @return The number of items in the view. + */ + size_type size() const { return sz_; } + /** + * Gets the number of items in the view. + * @return The number of items in the view. + */ + size_type length() const { return sz_; } + /** + * Access an item in the view. + * @param i The index of the item. + * @return A const reference to the requested item. + */ + const value_type& operator[](size_t i) const { return data_[i]; } + /** + * Gets a copy of the view as a string. + * @return A copy of the view as a string. + */ + std::basic_string str() const { + return std::basic_string(data_, sz_); + } + /** + * Gets a copy of the view as a string. + * @return A copy of the view as a string. + */ + string to_string() const { + static_assert(sizeof(char) == sizeof(T), "can only get string for char or byte buffers"); + return string(reinterpret_cast(data_), sz_); + } +}; + +/** A buffer view for character string data. */ +using string_view = buffer_view; + +/** A buffer view for binary data */ +using binary_view = buffer_view; + + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + +#endif // __mqtt_buffer_view_h + diff --git a/src/mqtt/src/mqtt/callback.h b/src/mqtt/src/mqtt/callback.h new file mode 100644 index 00000000..434b9796 --- /dev/null +++ b/src/mqtt/src/mqtt/callback.h @@ -0,0 +1,91 @@ +///////////////////////////////////////////////////////////////////////////// +/// @file callback.h +/// Declaration of MQTT callback class +/// @date May 1, 2013 +/// @author Frank Pagliughi +///////////////////////////////////////////////////////////////////////////// + +/******************************************************************************* + * Copyright (c) 2013-2019 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + *******************************************************************************/ + +#ifndef __mqtt_callback_h +#define __mqtt_callback_h + +#include "MQTTAsync.h" +#include "mqtt/delivery_token.h" +#include "mqtt/types.h" +#include +#include + +namespace mqtt { + +///////////////////////////////////////////////////////////////////////////// + +/** + * Provides a mechanism for tracking the completion of an asynchronous + * action. + */ +class callback +{ +public: + /** Smart/shared pointer to an object of this type */ + using ptr_t = std::shared_ptr; + /** Smart/shared pointer to a const object of this type */ + using const_ptr_t = std::shared_ptr; + + /** + * Virtual destructor. + */ + virtual ~callback() {} + + /** + * This method is called when the client is connected. + * Note that, in response to an initial connect(), the token from the + * connect call is also signaled with an on_success(). That occurs just + * before this is called. + * @param cause + */ + virtual void connected(const string& /*cause*/) {} + /** + * This method is called when the connection to the server is lost. + * @param cause + */ + virtual void connection_lost(const string& /*cause*/) {} + /** + * This method is called when a message arrives from the server. + * @param msg The message + */ + virtual void message_arrived(const_message_ptr /*msg*/) {} + /** + * Called when delivery for a message has been completed, and all + * acknowledgments have been received. + * @param tok The token tracking the message delivery. + */ + virtual void delivery_complete(delivery_token_ptr /*tok*/) {} +}; + +/** Smart/shared pointer to a callback object */ +using callback_ptr = callback::ptr_t; + +/** Smart/shared pointer to a const callback object */ +using const_callback_ptr = callback::const_ptr_t; + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + +#endif // __mqtt_callback_h + diff --git a/src/mqtt/src/mqtt/client.h b/src/mqtt/src/mqtt/client.h new file mode 100644 index 00000000..93d5b589 --- /dev/null +++ b/src/mqtt/src/mqtt/client.h @@ -0,0 +1,377 @@ +///////////////////////////////////////////////////////////////////////////// +/// @file client.h +/// Declaration of MQTT client class +/// @date May 1, 2013 +/// @author Frank Pagliughi +///////////////////////////////////////////////////////////////////////////// + +/******************************************************************************* + * Copyright (c) 2013-2017 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + *******************************************************************************/ + +#ifndef __mqtt_client_h +#define __mqtt_client_h + +#include "mqtt/async_client.h" +#include + +namespace mqtt { + +///////////////////////////////////////////////////////////////////////////// + +/** + * Lightweight client for talking to an MQTT server using methods that block + * until an operation completes. + */ +class client : private callback +{ + /** An arbitrary, but relatively long timeout */ + static const std::chrono::minutes DFLT_TIMEOUT; + /** The default quality of service */ + static constexpr int DFLT_QOS = 1; + + /** The actual client */ + async_client cli_; + /** The longest time to wait for an operation to complete. */ + std::chrono::milliseconds timeout_; + /** Callback supplied by the user (if any) */ + callback* userCallback_; + + /** + * Creates a shared pointer to an existing non-heap object. + * The shared pointer is given a no-op deleter, so it will not try to + * destroy the object when it goes out of scope. It is up to the caller + * to ensure that the object remains in memory for as long as there may + * be pointers to it. + * @param val A value which may live anywhere in memory (stack, + * file-scope, etc). + * @return A shared pointer to the object. + */ + template + std::shared_ptr ptr(const T& val) { + return std::shared_ptr(const_cast(&val), [](T*){}); + } + + // User callbacks + // Most are launched in a separate thread, for convenience, except + // message_arrived, for performance. + void connected(const string& cause) override { + std::async(std::launch::async, &callback::connected, userCallback_, cause); + } + void connection_lost(const string& cause) override { + std::async(std::launch::async, + &callback::connection_lost, userCallback_, cause); + } + void message_arrived(const_message_ptr msg) override { + userCallback_->message_arrived(msg); + } + void delivery_complete(delivery_token_ptr tok) override { + std::async(std::launch::async, &callback::delivery_complete, userCallback_, tok); + } + + /** Non-copyable */ + client() =delete; + client(const async_client&) =delete; + client& operator=(const async_client&) =delete; + +public: + /** Smart pointer type for this object */ + using ptr_t = std::shared_ptr; + /** Type for a collection of QOS values */ + using qos_collection = async_client::qos_collection; + + /** + * Create a client that can be used to communicate with an MQTT server. + * This allows the caller to specify a user-defined persistence object, + * or use no persistence. + * @param serverURI the address of the server to connect to, specified + * as a URI. + * @param clientId a client identifier that is unique on the server + * being connected to + * @param persistence The user persistence structure. If this is null, + * then no persistence is used. + */ + client(const string& serverURI, const string& clientId, + iclient_persistence* persistence=nullptr); + /** + * Create an async_client that can be used to communicate with an MQTT + * server. + * This uses file-based persistence in the specified directory. + * @param serverURI the address of the server to connect to, specified + * as a URI. + * @param clientId a client identifier that is unique on the server + * being connected to + * @param persistDir The directory to use for persistence data + */ + client(const string& serverURI, const string& clientId, + const string& persistDir); + /** + * Create a client that can be used to communicate with an MQTT server, + * which allows for off-line message buffering. + * This allows the caller to specify a user-defined persistence object, + * or use no persistence. + * @param serverURI the address of the server to connect to, specified + * as a URI. + * @param clientId a client identifier that is unique on the server + * being connected to + * @param maxBufferedMessages the maximum number of messages allowed to + * be buffered while not connected + * @param persistence The user persistence structure. If this is null, + * then no persistence is used. + */ + client(const string& serverURI, const string& clientId, + int maxBufferedMessages, iclient_persistence* persistence=nullptr); + /** + * Create a client that can be used to communicate with an MQTT server, + * which allows for off-line message buffering. + * This uses file-based persistence in the specified directory. + * @param serverURI the address of the server to connect to, specified + * as a URI. + * @param clientId a client identifier that is unique on the server + * being connected to + * @param maxBufferedMessages the maximum number of messages allowed to + * be buffered while not connected + * @param persistDir The directory to use for persistence data + */ + client(const string& serverURI, const string& clientId, + int maxBufferedMessages, const string& persistDir); + /** + * Virtual destructor + */ + virtual ~client() {} + /** + * Connects to an MQTT server using the default options. + */ + virtual connect_response connect(); + /** + * Connects to an MQTT server using the specified options. + * @param opts + */ + virtual connect_response connect(connect_options opts); + /** + * Reconnects the client using options from the previous connect. + * The client must have previously called connect() for this to work. + */ + virtual connect_response reconnect(); + /** + * Disconnects from the server. + */ + virtual void disconnect(); + /** + * Disconnects from the server. + * @param timeoutMS the amount of time in milliseconds to allow for + * existing work to finish before disconnecting. A value + * of zero or less means the client will not quiesce. + */ + virtual void disconnect(int timeoutMS) { + cli_.stop_consuming(); + cli_.disconnect(timeoutMS)->wait_for(timeout_); + } + /** + * Disconnects from the server. + * @param to the amount of time in milliseconds to allow for + * existing work to finish before disconnecting. A value + * of zero or less means the client will not quiesce. + */ + template + void disconnect(const std::chrono::duration& to) { + disconnect((int) to_milliseconds_count(to)); + } + /** + * Gets the client ID used by this client. + * @return The client ID used by this client. + */ + virtual string get_client_id() const { return cli_.get_client_id(); } + /** + * Gets the address of the server used by this client. + * @return The address of the server used by this client, as a URI. + */ + virtual string get_server_uri() const { return cli_.get_server_uri(); } + /** + * Return the maximum time to wait for an action to complete. + * @return int + */ + virtual std::chrono::milliseconds get_timeout() const { return timeout_; } + /** + * Get a topic object which can be used to publish messages on this + * client. + * @param top The topic name + * @return A topic attached to this client. + */ + virtual topic get_topic(const string& top) { return topic(cli_, top); } + /** + * Determines if this client is currently connected to the server. + * @return @em true if the client is currently connected, @em false if + * not. + */ + virtual bool is_connected() const { return cli_.is_connected(); } + + /** + * Publishes a message to a topic on the server and return once it is + * delivered. + * @param top The topic to publish + * @param payload The data to publish + * @param n The size in bytes of the data + * @param qos The QoS for message delivery + * @param retained Whether the broker should retain the message + */ + virtual void publish(string_ref top, const void* payload, size_t n, + int qos, bool retained) { + cli_.publish(std::move(top), payload, n, qos, retained)->wait_for(timeout_); + } + /** + * Publishes a message to a topic on the server and return once it is + * delivered. + * @param top The topic to publish + * @param payload The data to publish + * @param n The size in bytes of the data + */ + virtual void publish(string_ref top, const void* payload, size_t n) { + cli_.publish(std::move(top), payload, n)->wait_for(timeout_); + } + /** + * Publishes a message to a topic on the server. + * @param msg The message + */ + virtual void publish(const_message_ptr msg) { + cli_.publish(msg)->wait_for(timeout_); + } + /** + * Publishes a message to a topic on the server. + * This version will not timeout since that could leave the library with + * a reference to memory that could disappear while the library is still + * using it. + * @param msg The message + */ + virtual void publish(const message& msg) { + cli_.publish(ptr(msg))->wait(); + } + /** + * Sets the callback listener to use for events that happen + * asynchronously. + * @param cb The callback functions + */ + virtual void set_callback(callback& cb); + /** + * Set the maximum time to wait for an action to complete. + * @param timeoutMS The timeout in milliseconds + */ + virtual void set_timeout(int timeoutMS) { + timeout_ = std::chrono::milliseconds(timeoutMS); + } + /** + * Set the maximum time to wait for an action to complete. + * @param to The timeout as a std::chrono duration. + */ + template + void set_timeout(const std::chrono::duration& to) { + timeout_ = to_milliseconds(to); + } + /** + * Subscribe to a topic, which may include wildcards using a QoS of 1. + * @param topicFilter + */ + virtual subscribe_response subscribe(const string& topicFilter); + /** + * Subscribe to a topic, which may include wildcards. + * @param topicFilter A single topic to subscribe + * @param qos The QoS of the subscription + */ + virtual subscribe_response subscribe(const string& topicFilter, int qos); + /** + * Subscribes to a one or more topics, which may include wildcards using + * a QoS of 1. + * @param topicFilters A set of topics to subscribe + */ + virtual subscribe_response subscribe(const string_collection& topicFilters); + /** + * Subscribes to multiple topics, each of which may include wildcards. + * @param topicFilters A collection of topics to subscribe + * @param qos A collection of QoS for each topic + */ + virtual subscribe_response subscribe(const string_collection& topicFilters, + const qos_collection& qos); + /** + * Requests the server unsubscribe the client from a topic. + * @param topicFilter A single topic to unsubscribe. + */ + virtual unsubscribe_response unsubscribe(const string& topicFilter); + /** + * Requests the server unsubscribe the client from one or more topics. + * @param topicFilters A collection of topics to unsubscribe. + */ + virtual unsubscribe_response unsubscribe(const string_collection& topicFilters); + /** + * Start consuming messages. + * This initializes the client to receive messages through a queue that + * can be read synchronously. + */ + void start_consuming() { cli_.start_consuming(); } + /** + * Stop consuming messages. + * This shuts down the internal callback and discards any unread + * messages. + */ + void stop_consuming() { cli_.stop_consuming(); } + /** + * Read the next message from the queue. + * This blocks until a new message arrives. + * @return The message and topic. + */ + const_message_ptr consume_message() { return cli_.consume_message(); } + /** + * Try to read the next message from the queue without blocking. + * @param msg Pointer to the value to receive the message + * @return @em true is a message was read, @em false if no message was + * available. + */ + bool try_consume_message(const_message_ptr* msg) { + return cli_.try_consume_message(msg); + } + /** + * Waits a limited time for a message to arrive. + * @param msg Pointer to the value to receive the message + * @param relTime The maximum amount of time to wait for a message. + * @return @em true if a message was read, @em false if a timeout + * occurred. + */ + template + bool try_consume_message_for(const_message_ptr* msg, + const std::chrono::duration& relTime) { + return cli_.try_consume_message_for(msg, relTime); + } + /** + * Waits until a specific time for a message to occur. + * @param msg Pointer to the value to receive the message + * @param absTime The time point to wait until, before timing out. + * @return @em true if a message was read, @em false if a timeout + * occurred. + */ + template + bool try_consume_message_until(const_message_ptr* msg, + const std::chrono::time_point& absTime) { + return cli_.try_consume_message_until(msg, absTime); + } +}; + +/** Smart/shared pointer to an MQTT synchronous client object */ +using client_ptr = client::ptr_t; + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + +#endif // __mqtt_client_h + diff --git a/src/mqtt/src/mqtt/connect_options.h b/src/mqtt/src/mqtt/connect_options.h new file mode 100644 index 00000000..891bd013 --- /dev/null +++ b/src/mqtt/src/mqtt/connect_options.h @@ -0,0 +1,448 @@ +///////////////////////////////////////////////////////////////////////////// +/// @file connect_options.h +/// Declaration of MQTT connect_options class +/// @date May 1, 2013 +/// @author Frank Pagliughi +///////////////////////////////////////////////////////////////////////////// + +/******************************************************************************* + * Copyright (c) 2013-2016 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + *******************************************************************************/ + +#ifndef __mqtt_connect_options_h +#define __mqtt_connect_options_h + +#include "MQTTAsync.h" +#include "mqtt/types.h" +#include "mqtt/message.h" +#include "mqtt/topic.h" +#include "mqtt/token.h" +#include "mqtt/string_collection.h" +#include "mqtt/will_options.h" +#include "mqtt/ssl_options.h" +#include +#include + +namespace mqtt { + +///////////////////////////////////////////////////////////////////////////// + +/** + * Holds the set of options that control how the client connects to a + * server. + */ +class connect_options +{ + /** The default C struct */ + static const MQTTAsync_connectOptions DFLT_C_STRUCT ; + + /** The underlying C connection options */ + MQTTAsync_connectOptions opts_; + + /** The LWT options */ + will_options will_; + + /** The SSL options */ + ssl_options ssl_; + + /** The user name to use for the connection. */ + string_ref userName_; + + /** The password to use for the connection. */ + binary_ref password_; + + /** Shared token pointer for context, if any */ + token_ptr tok_; + + /** Collection of server URIs, if any */ + const_string_collection_ptr serverURIs_; + + /** The connect properties */ + properties props_; + + /** The client has special access */ + friend class async_client; + friend class connect_options_test; + friend class token_test; + + /** + * Gets a pointer to the C-language NUL-terminated strings for the + * struct. + * @note In the connect options, by default, the Paho C treats + * nullptr char arrays as unset values, so we keep that semantic and + * only set those char arrays if the string is non-empty. + * @param str The C++ string object. + * @return Pointer to a NUL terminated string. This is only valid until + * the next time the string is updated. + */ + const char* c_str(const string_ref& sr) { + return sr.empty() ? nullptr : sr.c_str(); + } + +public: + /** Smart/shared pointer to an object of this class. */ + using ptr_t = std::shared_ptr; + /** Smart/shared pointer to a const object of this class. */ + using const_ptr_t = std::shared_ptr; + + /** + * Constructs a new object using the default values. + */ + connect_options(); + /** + * Constructs a new object using the specified user name and password. + * @param userName The name of the user for connecting to the server + * @param password The password for connecting to the server + */ + connect_options(string_ref userName, binary_ref password); + /** + * Copy constructor. + * @param opt Another object to copy. + */ + connect_options(const connect_options& opt); + /** + * Move constructor. + * @param opt Another object to move into this new one. + */ + connect_options(connect_options&& opt); + /** + * Copy assignment. + * @param opt Another object to copy. + */ + connect_options& operator=(const connect_options& opt); + /** + * Move assignment. + * @param opt Another object to move into this new one. + */ + connect_options& operator=(connect_options&& opt); + /** + * Gets the "keep alive" interval. + * @return The keep alive interval in seconds. + */ + std::chrono::seconds get_keep_alive_interval() const { + return std::chrono::seconds(opts_.keepAliveInterval); + } + /** + * Gets the connection timeout. + * This is the amount of time the underlying library will wait for a + * timeout before failing. + * @return The connect timeout in seconds. + */ + std::chrono::seconds get_connect_timeout() const { + return std::chrono::seconds(opts_.connectTimeout); + } + /** + * Gets the user name to use for the connection. + * @return The user name to use for the connection. + */ + string get_user_name() const { return userName_ ? userName_.to_string() : string(); } + /** + * Gets the password to use for the connection. + * @return The password to use for the connection. + */ + binary_ref get_password() const { return password_; } + /** + * Gets the password to use for the connection. + * @return The password to use for the connection. + */ + string get_password_str() const { + return password_ ? password_.to_string() : string(); + } + /** + * Gets the maximum number of messages that can be in-flight + * simultaneously. + * @return The maximum number of inflight messages. + */ + int get_max_inflight() const { return opts_.maxInflight; } + /** + * Gets the topic to be used for last will and testament (LWT). + * @return The topic to be used for last will and testament (LWT). + */ + string get_will_topic() const { + return will_.get_topic(); + } + /** + * Gets the message to be sent as last will and testament (LWT). + * @return The message to be sent as last will and testament (LWT). + */ + const_message_ptr get_will_message() const { + return will_.get_message(); + } + /** + * Get the LWT options to use for the connection. + * @return The LWT options to use for the connection. + */ + const will_options& get_will_options() const { return will_; } + /** + * Get the SSL options to use for the connection. + * @return The SSL options to use for the connection. + */ + const ssl_options& get_ssl_options() const { return ssl_; } + /** + * Sets the SSL for the connection. + * These will only have an effect if compiled against the SSL version of + * the Paho C library. + * @param ssl The SSL options. + */ + void set_ssl(const ssl_options& ssl); + void set_ssl(ssl_options&& ssl); + /** + * Returns whether the server should remember state for the client + * across reconnects. + * @return @em true if requesting a clean session, @em false if not. + */ + bool is_clean_session() const { return to_bool(opts_.cleansession); } + /** + * Gets the token used as the callback context. + * @return The delivery token used as the callback context. + */ + token_ptr get_token() const { return tok_; } + /** + * Gets the list of servers to which the client will connect. + * @return A collection of server URI's. Each entry should be of the + * form @em protocol://host:port where @em protocol must be tcp + * or @em ssl. For @em host, you can specify either an IP + * address or a domain name. + */ + const_string_collection_ptr get_servers() const { return serverURIs_; } + /** + * Gets the version of MQTT to be used on the connect. + * @return + * @li MQTTVERSION_DEFAULT (0) = default: start with 3.1.1, and if that + * fails, fall back to 3.1 + * @li MQTTVERSION_3_1 (3) = only try version 3.1 + * @li MQTTVERSION_3_1_1 (4) = only try version 3.1.1 + */ + int get_mqtt_version() const { return opts_.MQTTVersion; } + /** + * Determines if the options have been configured for automatic + * reconnect. + * @return @em true if configured for automatic reconnect, @em false if + * not. + */ + bool get_automatic_reconnect() const { return to_bool(opts_.automaticReconnect); } + /** + * Gets the minimum retry interval for automatic reconnect. + * @return The minimum retry interval for automatic reconnect, in + * seconds. + */ + std::chrono::seconds get_min_retry_interval() const { + return std::chrono::seconds(opts_.minRetryInterval); + } + /** + * Gets the maximum retry interval for automatic reconnect. + * @return The maximum retry interval for automatic reconnect, in + * seconds. + */ + std::chrono::seconds get_max_retry_interval() const { + return std::chrono::seconds(opts_.maxRetryInterval); + } + + /** + * Sets whether the server should remember state for the client across + * reconnects. + * @param cleanSession + */ + void set_clean_session(bool cleanSession) { + opts_.cleansession = to_int(cleanSession); + } + /** + * Sets the "keep alive" interval. + * This is the maximum time that should pass without communications + * between client and server. If no massages pass in this time, the + * client will ping the broker. + * @param keepAliveInterval The keep alive interval in seconds. + */ + void set_keep_alive_interval(int keepAliveInterval) { + opts_.keepAliveInterval = keepAliveInterval; + } + /** + * Sets the "keep alive" interval with a chrono duration. + * This is the maximum time that should pass without communications + * between client and server. If no massages pass in this time, the + * client will ping the broker. + * @param interval The keep alive interval. + */ + template + void set_keep_alive_interval(const std::chrono::duration& interval) { + // TODO: Check range + set_keep_alive_interval((int) to_seconds_count(interval)); + } + /** + * Sets the connect timeout in seconds. + * This is the maximum time that the underlying library will wait for a + * connection before failing. + * @param timeout The connect timeout in seconds. + */ + void set_connect_timeout(int timeout) { + opts_.connectTimeout = timeout; + } + /** + * Sets the connect timeout with a chrono duration. + * This is the maximum time that the underlying library will wait for a + * connection before failing. + * @param timeout The connect timeout in seconds. + */ + template + void set_connect_timeout(const std::chrono::duration& timeout) { + // TODO: check range + set_connect_timeout((int) to_seconds_count(timeout)); + } + /** + * Sets the user name to use for the connection. + * @param userName + */ + void set_user_name(string_ref userName); + /** + * Sets the password to use for the connection. + */ + void set_password(binary_ref password); + /** + * Sets the maximum number of messages that can be in-flight + * simultaneously. + * @param n The maximum number of inflight messages. + */ + void set_max_inflight(int n) { opts_.maxInflight = n; } + /** + * Sets the "Last Will and Testament" (LWT) for the connection. + * @param will The LWT options. + */ + void set_will(const will_options& will); + void set_will(will_options&& will); + /** + * Sets the "Last Will and Testament" (LWT) as a message + * @param msg The LWT message + */ + void set_will_message(const message& msg) { + set_will(will_options(msg)); + } + /** + * Sets the "Last Will and Testament" (LWT) as a message + * @param msg Pointer to a LWT message + */ + void set_will_message(const_message_ptr msg) { + if (msg) set_will(will_options(*msg)); + } + /** + * Sets the callback context to a delivery token. + * @param tok The delivery token to be used as the callback context. + */ + void set_token(const token_ptr& tok); + /** + * Sets the list of servers to which the client will connect. + * @param serverURIs A pointer to a collection of server URI's. Each + * entry should be of the form @em + * protocol://host:port where @em protocol must be + * @em tcp or @em ssl. For @em host, you can specify + * either an IP address or a domain name. + */ + void set_servers(const_string_collection_ptr serverURIs); + /** + * Sets the version of MQTT to be used on the connect. + * + * This will also set other connect options to legal values dependent on + * the selected version. + * + * @param mqttVersion The MQTT version to use for the connection: + * @li MQTTVERSION_DEFAULT (0) = default: start with 3.1.1, and if + * that fails, fall back to 3.1 + * @li MQTTVERSION_3_1 (3) = only try version 3.1 + * @li MQTTVERSION_3_1_1 (4) = only try version 3.1.1 + * @li MQTTVERSION_5 (5) = only try version 5 + */ + void set_mqtt_version(int mqttVersion); + /** + * Enable or disable automatic reconnects. + * The retry intervals are not affected. + * @param on Whether to turn reconnects on or off + */ + void set_automatic_reconnect(bool on) { + opts_.automaticReconnect = to_int(on); + } + /** + * Enable or disable automatic reconnects. + * @param minRetryInterval Minimum retry interval in seconds. Doubled + * on each failed retry. + * @param maxRetryInterval Maximum retry interval in seconds. The + * doubling stops here on failed retries. + */ + void set_automatic_reconnect(int minRetryInterval, int maxRetryInterval); + /** + * Enable or disable automatic reconnects. + * @param minRetryInterval Minimum retry interval. Doubled on each + * failed retry. + * @param maxRetryInterval Maximum retry interval. The doubling stops + * here on failed retries. + */ + template + void set_automatic_reconnect(const std::chrono::duration& minRetryInterval, + const std::chrono::duration& maxRetryInterval) { + set_automatic_reconnect((int) to_seconds_count(minRetryInterval), + (int) to_seconds_count(maxRetryInterval)); + } + /** + * Determines if the 'clean start' flag is set for the connect. + * @return @em true if the 'clean start' flag is set for the connect, @em + * false if not. + */ + bool is_clean_start() const { + return to_bool(opts_.cleanstart); + } + /** + * Sets the 'clean start' flag for the connection. + * @param cleanStart Whether to set the 'clean start' flag for the connect. + */ + void set_clean_start(bool cleanStart) { + opts_.cleanstart = to_int(cleanStart); + } + /** + * Gets the connect properties. + * @return A const reference to the properties for the connect. + */ + const properties& get_properties() const { + return props_; + } + /** + * Sets the properties for the connect. + * @param props The properties to place into the message. + */ + void set_properties(const properties& props) { + props_ = props; + opts_.connectProperties = const_cast(&props_.c_struct()); + } + /** + * Moves the properties for the connect. + * @param props The properties to move into the connect object. + */ + void set_properties(properties&& props) { + props_ = props; + opts_.connectProperties = const_cast(&props_.c_struct()); + } + /** + * Gets a string representation of the object. + * @return A string representation of the object. + */ + string to_string() const; +}; + +/** Smart/shared pointer to a connection options object. */ +using connect_options_ptr = connect_options::ptr_t; + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + +#endif // __mqtt_connect_options_h + diff --git a/src/mqtt/src/mqtt/delivery_token.h b/src/mqtt/src/mqtt/delivery_token.h new file mode 100644 index 00000000..b70d5822 --- /dev/null +++ b/src/mqtt/src/mqtt/delivery_token.h @@ -0,0 +1,135 @@ +///////////////////////////////////////////////////////////////////////////// +/// @file delivery_token.h +/// Declaration of MQTT delivery_token class +/// @date May 1, 2013 +/// @author Frank Pagliughi +///////////////////////////////////////////////////////////////////////////// + +/******************************************************************************* + * Copyright (c) 2013-2016 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + *******************************************************************************/ + +#ifndef __mqtt_delivery_token_h +#define __mqtt_delivery_token_h + +#include "MQTTAsync.h" +#include "mqtt/token.h" +#include "mqtt/message.h" +#include + +namespace mqtt { + +///////////////////////////////////////////////////////////////////////////// + +/** + * Provides a mechanism to track the delivery progress of a message. + * Used to track the the delivery progress of a message when a publish is + * executed in a non-blocking manner (run in the background) action. + */ +class delivery_token : public token +{ + /** The message being tracked. */ + const_message_ptr msg_; + + /** Client has special access. */ + friend class async_client; + + /** + * Sets the message to which this token corresponds. + * @param msg + */ + void set_message(const_message_ptr msg) { msg_ = msg; } + +public: + /** Smart/shared pointer to an object of this class */ + using ptr_t = std::shared_ptr; + /** Smart/shared pointer to a const object of this class */ + using const_ptr_t = std::shared_ptr; + /** Weak pointer to an object of this class */ + using weak_ptr_t = std::weak_ptr; + + /** + * Creates an empty delivery token connected to a particular client. + * @param cli The asynchronous client object. + */ + delivery_token(iasync_client& cli) : token(token::Type::PUBLISH, cli) {} + /** + * Creates a delivery token connected to a particular client. + * @param cli The asynchronous client object. + * @param msg The message being tracked. + */ + delivery_token(iasync_client& cli, const_message_ptr msg) + : token(token::Type::PUBLISH, cli, msg->get_topic()), msg_(std::move(msg)) {} + /** + * Creates a delivery token connected to a particular client. + * @param cli The asynchronous client object. + * @param msg The message data. + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param cb callback optional listener that will be notified when message + * delivery has completed to the requested quality of + * service + */ + delivery_token(iasync_client& cli, const_message_ptr msg, + void* userContext, iaction_listener& cb) + : token(token::Type::PUBLISH, cli, msg->get_topic(), userContext, cb), msg_(std::move(msg)) {} + /** + * Creates an empty delivery token connected to a particular client. + * @param cli The asynchronous client object. + */ + static ptr_t create(iasync_client& cli) { + return std::make_shared(cli); + } + /** + * Creates a delivery token connected to a particular client. + * @param cli The asynchronous client object. + * @param msg The message data. + */ + static ptr_t create(iasync_client& cli, const_message_ptr msg) { + return std::make_shared(cli, msg); + } + /** + * Creates a delivery token connected to a particular client. + * @param cli The asynchronous client object. + * @param msg The message data. + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param cb callback optional listener that will be notified when message + * delivery has completed to the requested quality of + * service + */ + static ptr_t create(iasync_client& cli, const_message_ptr msg, + void* userContext, iaction_listener& cb) { + return std::make_shared(cli, msg, userContext, cb); + } + /** + * Gets the message associated with this token. + * @return The message associated with this token. + */ + virtual const_message_ptr get_message() const { return msg_; } +}; + +/** Smart/shared pointer to a delivery_token */ +using delivery_token_ptr = delivery_token::ptr_t; + +/** Smart/shared pointer to a const delivery_token */ +using const_delivery_token_ptr = delivery_token::const_ptr_t; + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + +#endif // __mqtt_delivery_token_h + diff --git a/src/mqtt/src/mqtt/disconnect_options.h b/src/mqtt/src/mqtt/disconnect_options.h new file mode 100644 index 00000000..d2ebdbf7 --- /dev/null +++ b/src/mqtt/src/mqtt/disconnect_options.h @@ -0,0 +1,176 @@ +///////////////////////////////////////////////////////////////////////////// +/// @file disconnect_options.h +/// Implementation of the class 'disconnect_options' +/// @date 26-Aug-2016 +///////////////////////////////////////////////////////////////////////////// + +/**************************************************************************** + * Copyright (c) 2016-2017 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + ***************************************************************************/ + +#ifndef __mqtt_disconnect_options_h +#define __mqtt_disconnect_options_h + +#include "MQTTAsync.h" +#include "mqtt/types.h" +#include "mqtt/token.h" +#include "mqtt/properties.h" +#include + +namespace mqtt { + +///////////////////////////////////////////////////////////////////////////// + +/** + * Options for disconnecting from an MQTT broker. + */ +class disconnect_options +{ + /** The default C struct */ + static const MQTTAsync_disconnectOptions DFLT_C_STRUCT; + + /** The underlying C disconnect options */ + MQTTAsync_disconnectOptions opts_; + + /** Shared token pointer for context, if any */ + token_ptr tok_; + + /** Disconnect message properties */ + properties props_; + + /** The client has special access */ + friend class async_client; + friend class disconnect_options_test; + +public: + /** + * Create an empty delivery response object. + */ + disconnect_options(); + /** + * Creates disconnect options tied to the specific token. + * @param timeout The timeout (in milliseconds). + */ + disconnect_options(int timeout) : disconnect_options() { + set_timeout(timeout); + } + /** + * Creates disconnect options tied to the specific token. + * @param to The timeout. + */ + template + disconnect_options(const std::chrono::duration& to) + : disconnect_options() { + set_timeout(to); + } + /** + * Copy constructor. + * @param opt Another object to copy. + */ + disconnect_options(const disconnect_options& opt); + /** + * Move constructor. + * @param opt Another object to move into this new one. + */ + disconnect_options(disconnect_options&& opt); + /** + * Copy assignment. + * @param opt Another object to copy. + */ + disconnect_options& operator=(const disconnect_options& opt); + /** + * Move assignment. + * @param opt Another object to move into this new one. + */ + disconnect_options& operator=(disconnect_options&& opt); + /** + * Gets the timeout used for disconnecting. + * @return The timeout for disconnecting (in milliseconds). + */ + std::chrono::milliseconds get_timeout() const { + return std::chrono::milliseconds(opts_.timeout); + } + /** + * Sets the timeout for disconnecting. + * This allows for any remaining in-flight messages to be delivered. + * @param timeout The timeout (in milliseconds). + */ + void set_timeout(int timeout) { opts_.timeout = timeout; } + /** + * Sets the connect timeout with a chrono duration. + * This is the maximum time that the underlying library will wait for a + * connection before failing. + * @param to The connect timeout in seconds. + */ + template + void set_timeout(const std::chrono::duration& to) { + // TODO: check range + set_timeout((int) to_milliseconds_count(to)); + } + /** + * Sets the callback context to a delivery token. + * @param tok The delivery token to be used as the callback context. + * @param mqttVersion The version of MQTT we're using for the + * connection. + */ + void set_token(const token_ptr& tok, int mqttVersion); + /** + * Gets the callback context to a delivery token. + * @return The delivery token to be used as the callback context. + */ + token_ptr get_token() const { return tok_; } + /** + * Gets the connect properties. + * @return A const reference to the properties for the connect. + */ + const properties& get_properties() const { return props_; } + /** + * Sets the properties for the connect. + * @param props The properties to place into the message. + */ + void set_properties(const properties& props) { + props_ = props; + opts_.properties = props_.c_struct(); + } + /** + * Moves the properties for the connect. + * @param props The properties to move into the connect object. + */ + void set_properties(properties&& props) { + props_ = props; + opts_.properties = props_.c_struct(); + } + /** + * Gets the reason code for the disconnect. + * @return The reason code for the disconnect. + */ + ReasonCode get_reason_code() const { + return ReasonCode(opts_.reasonCode); + } + /** + * Sets the reason code for the disconnect. + * @param code The reason code for the disconnect. + */ + void set_reason_code(ReasonCode code) { + opts_.reasonCode = MQTTReasonCodes(code); + } +}; + +///////////////////////////////////////////////////////////////////////////// +// end namespace 'mqtt' +} + +#endif // __mqtt_disconnect_options_h + diff --git a/src/mqtt/src/mqtt/exception.h b/src/mqtt/src/mqtt/exception.h new file mode 100644 index 00000000..cdcd2fc4 --- /dev/null +++ b/src/mqtt/src/mqtt/exception.h @@ -0,0 +1,225 @@ +///////////////////////////////////////////////////////////////////////////// +/// @file exception.h +/// Declaration of MQTT exception class +/// @date May 1, 2013 +/// @author Frank Pagliughi +///////////////////////////////////////////////////////////////////////////// + +/******************************************************************************* + * Copyright (c) 2013-2019 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + *******************************************************************************/ + +#ifndef __mqtt_exception_h +#define __mqtt_exception_h + +#include "MQTTAsync.h" +#include "mqtt/types.h" +#include +#include +#include +#include + +namespace mqtt { + +using bad_cast = std::bad_cast; + +///////////////////////////////////////////////////////////////////////////// + +/** + * Base mqtt::exception. + * This wraps the error codes which originate from the underlying C library. + */ +class exception : public std::runtime_error +{ +protected: + /** The error return code from the C library */ + int rc_; + /** The reason code from the server */ + ReasonCode reasonCode_; + /** The error message from the C library */ + string msg_; + +public: + /** + * Creates an MQTT exception. + * @param rc The error return code from the C library. + */ + explicit exception(int rc) + : exception(rc, error_str(rc)) {} + /** + * Creates an MQTT exception. + * @param rc The error return code from the C library. + * @param reasonCode The reason code from the server response. + */ + explicit exception(int rc, ReasonCode reasonCode) + : exception(rc, reasonCode, error_str(rc)) {} + /** + * Creates an MQTT exception. + * @param rc The error return code from the C library. + * @param msg The text message for the error. + */ + exception(int rc, const string& msg) + : std::runtime_error(printable_error(rc, ReasonCode::SUCCESS, msg)), + rc_(rc), reasonCode_(ReasonCode::SUCCESS), msg_(msg) {} + /** + * Creates an MQTT exception. + * @param rc The error return code from the C library. + * @param reasonCode The reason code from the server + * @param msg The text message for the error. + */ + exception(int rc, ReasonCode reasonCode, const string& msg) + : std::runtime_error(printable_error(rc, reasonCode, msg)), + rc_(rc), reasonCode_(reasonCode), msg_(msg) {} + /** + * Gets an error message from an error code. + * @param rc The error code from the C lib + * @return A string explanation of the error + */ + static string error_str(int rc) { + const char *msg = ::MQTTAsync_strerror(rc); + return msg ? string(msg) : string(); + } + /** + * Gets a string describing the MQTT v5 reason code. + * @param reasonCode The MQTT v5 reason code. + * @return A string describing the reason code. + */ + static string reason_code_str(int reasonCode) { + if (reasonCode != MQTTPP_V3_CODE) { + auto msg = ::MQTTReasonCode_toString(MQTTReasonCodes(reasonCode)); + if (msg) return string(msg); + } + return string(); + } + /** + * Gets a detailed error message for an error code. + * @param rc The error code from the C lib + * @param msg An optional additional message. If none is provided, the + * error_str message is used. + * @return A string error message that includes the error code and an + * explanation message. + */ + static string printable_error(int rc, int reasonCode=ReasonCode::SUCCESS, + const string& msg=string()) { + string s = "MQTT error [" + std::to_string(rc) + "]"; + if (!msg.empty()) + s += string(": ") + msg; + if (reasonCode != MQTTPP_V3_CODE && reasonCode != ReasonCode::SUCCESS) + s += string(". Reason: ") + reason_code_str(reasonCode); + return s; + } + /** + * Returns the return code for this exception. + */ + int get_return_code() const { return rc_; } + /** + * Gets a string of the error code. + * @return A string of the error code. + */ + string get_error_str() const { return error_str(rc_); } + /** + * Returns the reason code for this exception. + * For MQTT v3 connections, this is actually the return code. + */ + int get_reason_code() const { + return reasonCode_ == MQTTPP_V3_CODE ? rc_ : reasonCode_; + } + /** + * Gets a string for the reason code. + * @return A string for the reason code. + */ + string get_reason_code_str() const { + return reason_code_str(reasonCode_); + } + /** + * Returns the error message for this exception. + */ + string get_message() const { return msg_; } + /** + * Gets a string representation of this exception. + * @return A string representation of this exception. + */ + string to_string() const { return string(what()); } +}; + +///////////////////////////////////////////////////////////////////////////// + +class missing_response : public std::runtime_error +{ +public: + missing_response(const string& rsp) + : std::runtime_error("Missing "+rsp+" response") {} +}; + +///////////////////////////////////////////////////////////////////////////// + +/** + * This exception is thrown by the implementor of the persistence interface + * if there is a problem reading or writing persistent data. + */ +class persistence_exception : public exception +{ +public: + /** + * Creates an MQTT persistence exception. + */ + persistence_exception() : exception(MQTTCLIENT_PERSISTENCE_ERROR) {} + /** + * Creates an MQTT persistence exception. + * @param code The error code from the C library. + */ + explicit persistence_exception(int code) : exception(code) {} + /** + * Creates an MQTT persistence exception. + * @param msg The text message for the error. + */ + explicit persistence_exception(const string& msg) + : exception(MQTTCLIENT_PERSISTENCE_ERROR, msg) {} + /** + * Creates an MQTT persistence exception. + * @param code The error code + * @param msg The text message for the error. + */ + persistence_exception(int code, const string& msg) + : exception(code, msg) {} +}; + +///////////////////////////////////////////////////////////////////////////// + +/** + * Thrown when a client is not authorized to perform an operation, or if + * there is a problem with the security configuration. + */ +class security_exception : public exception +{ +public: + /** + * Creates an MQTT security exception + * @param code The error code. + */ + explicit security_exception(int code) : exception(code) {} + /** + * Creates an MQTT security exception + * @param code The error code. + * @param msg The text message for the error. + */ + security_exception(int code, const string& msg) : exception(code, msg) {} +}; + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + +#endif // __mqtt_token_h diff --git a/src/mqtt/src/mqtt/iaction_listener.h b/src/mqtt/src/mqtt/iaction_listener.h new file mode 100644 index 00000000..1e345771 --- /dev/null +++ b/src/mqtt/src/mqtt/iaction_listener.h @@ -0,0 +1,84 @@ +///////////////////////////////////////////////////////////////////////////// +/// @file iaction_listener.h +/// Declaration of MQTT iaction_listener class +/// @date May 1, 2013 +/// @author Frank Pagliughi +///////////////////////////////////////////////////////////////////////////// + +/******************************************************************************* + * Copyright (c) 2013-2016 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + *******************************************************************************/ + +#ifndef __mqtt_iaction_listener_h +#define __mqtt_iaction_listener_h + +#include "MQTTAsync.h" +#include "mqtt/types.h" +#include + +namespace mqtt { + +class token; + +///////////////////////////////////////////////////////////////////////////// + +/** + * Provides a mechanism for tracking the completion of an asynchronous + * action. + * + * A listener is registered on a token and that token is associated with + * an action like connect or publish. When used with tokens on the + * async_client the listener will be called back on the MQTT client's + * thread. The listener will be informed if the action succeeds or fails. It + * is important that the listener returns control quickly otherwise the + * operation of the MQTT client will be stalled. + */ +class iaction_listener +{ +public: + /** Smart/shared pointer to an object of this class. */ + using ptr_t = std::shared_ptr; + /** Smart/shared pointer to a const object of this class. */ + using const_ptr_t = std::shared_ptr; + + /** + * Virtual base destructor. + */ + virtual ~iaction_listener() {} + /** + * This method is invoked when an action fails. + * @param asyncActionToken + */ + virtual void on_failure(const token& asyncActionToken) =0; + /** + * This method is invoked when an action has completed successfully. + * @param asyncActionToken + */ + virtual void on_success(const token& asyncActionToken) =0; +}; + +/** Smart/shared pointer to an action listener */ +using iaction_listener_ptr = iaction_listener::ptr_t; + +/** Smart/shared pointer to a const action listener */ +using const_iaction_listener_ptr = iaction_listener::const_ptr_t; + + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + +#endif // __mqtt_iaction_listener_h + diff --git a/src/mqtt/src/mqtt/iasync_client.h b/src/mqtt/src/mqtt/iasync_client.h new file mode 100644 index 00000000..d47ab77c --- /dev/null +++ b/src/mqtt/src/mqtt/iasync_client.h @@ -0,0 +1,419 @@ +///////////////////////////////////////////////////////////////////////////// +/// @file iasync_client.h +/// Implementation of the interface for the asynchronous clients, +/// 'iasync_client' +/// @date 25-Aug-2016 +///////////////////////////////////////////////////////////////////////////// + +/******************************************************************************* + * Copyright (c) 2013-2016 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + *******************************************************************************/ + + +#ifndef __mqtt_iasync_client_h +#define __mqtt_iasync_client_h + +#include "mqtt/types.h" +#include "mqtt/token.h" +#include "mqtt/delivery_token.h" +#include "mqtt/iclient_persistence.h" +#include "mqtt/iaction_listener.h" +#include "mqtt/connect_options.h" +#include "mqtt/disconnect_options.h" +#include "mqtt/subscribe_options.h" +#include "mqtt/exception.h" +#include "mqtt/message.h" +#include "mqtt/callback.h" +#include + +namespace mqtt { + +///////////////////////////////////////////////////////////////////////////// + +/** + * Enables an application to communicate with an MQTT server using + * non-blocking methods. + * + * It provides applications a simple programming interface to all features + * of the MQTT version 3.1 specification including: + * + * @li connect + * @li publish + * @li subscribe + * @li unsubscribe + * @li disconnect + */ +class iasync_client +{ + friend class token; + virtual void remove_token(token* tok) =0; + +public: + /** Type for a collection of QOS values */ + using qos_collection = std::vector; + + /** + * Virtual destructor + */ + virtual ~iasync_client() {} + /** + * Connects to an MQTT server using the default options. + * @return token used to track and wait for the connect to complete. The + * token will be passed to any callback that has been set. + * @throw exception for non security related problems + * @throw security_exception for security related problems + */ + virtual token_ptr connect() =0; + /** + * Connects to an MQTT server using the provided connect options. + * @param options a set of connection parameters that override the + * defaults. + * @return token used to track and wait for the connect to complete. The + * token will be passed to any callback that has been set. + * @throw exception for non security related problems + * @throw security_exception for security related problems + */ + virtual token_ptr connect(connect_options options) =0; + /** + * Connects to an MQTT server using the specified options. + * + * @param options a set of connection parameters that override the + * defaults. + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param cb callback listener that will be notified when the connect + * completes. + * @return token used to track and wait for the connect to complete. The + * token will be passed to any callback that has been set. + * @throw exception for non security related problems + * @throw security_exception for security related problems + */ + virtual token_ptr connect(connect_options options, void* userContext, + iaction_listener& cb) =0; + /** + * + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param cb listener that will be notified when the connect completes. + * @return token used to track and wait for the connect to complete. The + * token will be passed to any callback that has been set. + * @throw exception for non security related problems + * @throw security_exception for security related problems + */ + virtual token_ptr connect(void* userContext, iaction_listener& cb) =0; + /** + * Reconnects the client using options from the previous connect. + * The client must have previously called connect() for this to work. + * @return token used to track the progress of the reconnect. + */ + virtual token_ptr reconnect() =0; + /** + * Disconnects from the server. + * @return token used to track and wait for the disconnect to complete. + * The token will be passed to any callback that has been set. + * @throw exception for problems encountered while disconnecting + */ + virtual token_ptr disconnect() =0; + /** + * Disconnects from the server. + * @param opts Options for disconnecting. + * @return token used to track and wait for the disconnect to complete. + * The token will be passed to any callback that has been set. + * @throw exception for problems encountered while disconnecting + */ + virtual token_ptr disconnect(disconnect_options opts) =0; + /** + * Disconnects from the server. + * @param timeout the amount of time in milliseconds to allow for + * existing work to finish before disconnecting. A value + * of zero or less means the client will not quiesce. + * @return token used to track and wait for the disconnect to complete. + * The token will be passed to any callback that has been set. + * @throw exception for problems encountered while disconnecting + */ + virtual token_ptr disconnect(int timeout) =0; + /** + * Disconnects from the server. + * @param timeout the amount of time in milliseconds to allow for + * existing work to finish before disconnecting. A value + * of zero or less means the client will not quiesce. + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param cb listener that will be notified when the disconnect + * completes. + * @return token used to track and wait for the disconnect to complete. + * The token will be passed to any callback that has been set. + * @throw exception for problems encountered while disconnecting + */ + virtual token_ptr disconnect(int timeout, void* userContext, iaction_listener& cb) =0; + /** + * Disconnects from the server. + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param cb listener that will be notified when the disconnect + * completes. + * @return token used to track and wait for the disconnect to complete. + * The token will be passed to any callback that has been set. + * @throw exception for problems encountered while disconnecting + */ + virtual token_ptr disconnect(void* userContext, iaction_listener& cb) =0; + /** + * Returns the delivery token for the specified message ID. + * @return delivery_token + */ + virtual delivery_token_ptr get_pending_delivery_token(int msgID) const =0; + /** + * Returns the delivery tokens for any outstanding publish operations. + * @return delivery_token[] + */ + virtual std::vector get_pending_delivery_tokens() const =0; + /** + * Returns the client ID used by this client. + * @return string + */ + virtual string get_client_id() const =0; + /** + * Returns the address of the server used by this client. + */ + virtual string get_server_uri() const =0; + /** + * Determines if this client is currently connected to the server. + */ + virtual bool is_connected() const =0; + /** + * Publishes a message to a topic on the server + * @param topic The topic to deliver the message to + * @param payload the bytes to use as the message payload + * @param n the number of bytes in the payload + * @param qos the Quality of Service to deliver the message at. Valid + * values are 0, 1 or 2. + * @param retained whether or not this message should be retained by the + * server. + * @return token used to track and wait for the publish to complete. The + * token will be passed to callback methods if set. + */ + virtual delivery_token_ptr publish(string_ref topic, + const void* payload, size_t n, + int qos, bool retained) =0; + /** + * Publishes a message to a topic on the server + * @param topic The topic to deliver the message to + * @param payload the bytes to use as the message payload + * @param n the number of bytes in the payload + * @return token used to track and wait for the publish to complete. The + * token will be passed to callback methods if set. + */ + virtual delivery_token_ptr publish(string_ref topic, + const void* payload, size_t n) =0; + /** + * Publishes a message to a topic on the server + * @param topic The topic to deliver the message to + * @param payload the bytes to use as the message payload + * @param n the number of bytes in the payload + * @param qos the Quality of Service to deliver the message at. Valid + * values are 0, 1 or 2. + * @param retained whether or not this message should be retained by the + * server. + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param cb + * @return token used to track and wait for the publish to complete. The + * token will be passed to callback methods if set. + */ + virtual delivery_token_ptr publish(string_ref topic, + const void* payload, size_t n, + int qos, bool retained, + void* userContext, iaction_listener& cb) =0; + /** + * Publishes a message to a topic on the server + * @param topic The topic to deliver the message to + * @param payload the bytes to use as the message payload + * @param qos the Quality of Service to deliver the message at. Valid + * values are 0, 1 or 2. + * @param retained whether or not this message should be retained by the + * server. + * @return token used to track and wait for the publish to complete. The + * token will be passed to callback methods if set. + */ + virtual delivery_token_ptr publish(string_ref topic, binary_ref payload, + int qos, bool retained) =0; + /** + * Publishes a message to a topic on the server. + * @param topic The topic to deliver the message to + * @param payload the bytes to use as the message payload + * @return token used to track and wait for the publish to complete. The + * token will be passed to callback methods if set. + */ + virtual delivery_token_ptr publish(string_ref topic, binary_ref payload) =0; + /** + * Publishes a message to a topic on the server Takes an Message + * message and delivers it to the server at the requested quality of + * service. + * @param msg the message to deliver to the server + * @return token used to track and wait for the publish to complete. The + * token will be passed to callback methods if set. + */ + virtual delivery_token_ptr publish(const_message_ptr msg) =0; + /** + * Publishes a message to a topic on the server. + * @param msg the message to deliver to the server + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param cb optional listener that will be notified when message + * delivery has completed to the requested quality of service + * @return token used to track and wait for the publish to complete. The + * token will be passed to callback methods if set. + */ + virtual delivery_token_ptr publish(const_message_ptr msg, + void* userContext, iaction_listener& cb) =0; + /** + * Sets a callback listener to use for events that happen + * asynchronously. + * @param cb callback which will be invoked for certain asynchronous + * events + */ + virtual void set_callback(callback& cb) =0; + /** + * Stops the callbacks. + */ + virtual void disable_callbacks() =0; + /** + * Subscribe to a topic, which may include wildcards. + * @param topicFilter the topic to subscribe to, which can include + * wildcards. + * @param qos the maximum quality of service at which to subscribe. + * Messages published at a lower quality of service will be + * received at the published QoS. Messages published at a + * higher quality of service will be received using the QoS + * specified on the subscribe. + * @param opts The options for the subscription. + * @return token used to track and wait for the subscribe to complete. + * The token will be passed to callback methods if set. + */ + virtual token_ptr subscribe(const string& topicFilter, int qos, + const subscribe_options& opts=subscribe_options()) =0; + /** + * Subscribe to a topic, which may include wildcards. + * @param topicFilter the topic to subscribe to, which can include + * wildcards. + * @param qos the maximum quality of service at which to subscribe. + * Messages published at a lower quality of service will be + * received at the published QoS. Messages published at a + * higher quality of service will be received using the QoS + * specified on the subscribe. + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param callback listener that will be notified when subscribe has + * completed + * @param opts The options for the subscription. + * @return token used to track and wait for the subscribe to complete. + * The token will be passed to callback methods if set. + */ + virtual token_ptr subscribe(const string& topicFilter, int qos, + void* userContext, iaction_listener& callback, + const subscribe_options& opts=subscribe_options()) =0; + /** + * Subscribe to multiple topics, each of which may include wildcards. + * Provides an optimized way to subscribe to multiple topics compared to + * subscribing to each one individually. + * @param topicFilters one or more topics to subscribe to, which can + * include wildcards + * @param qos the maximum quality of service at which to subscribe. + * Messages published at a lower quality of service will be + * received at the published QoS. Messages published at a + * higher quality of service will be received using the QoS + * specified on the subscribe. + * @param opts A collection of subscription optsions (one for each + * topic) + * @return token used to track and wait for the subscribe to complete. + * The token will be passed to callback methods if set. + */ + virtual token_ptr subscribe(const_string_collection_ptr topicFilters, + const qos_collection& qos, + const std::vector& opts=std::vector()) =0; + /** + * Subscribes to multiple topics, each of which may include wildcards. + * @param topicFilters one or more topics to subscribe to, which can + * include wildcards + * @param qos the maximum quality of service at which to subscribe. + * Messages published at a lower quality of service will be + * received at the published QoS. Messages published at a + * higher quality of service will be received using the QoS + * specified on the subscribe. + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param callback listener that will be notified when subscribe has + * completed + * @param opts A collection of subscription optsions (one for each + * topic) + * @return token used to track and wait for the subscribe to complete. + * The token will be passed to callback methods if set. + */ + virtual token_ptr subscribe(const_string_collection_ptr topicFilters, + const qos_collection& qos, + void* userContext, iaction_listener& callback, + const std::vector& opts=std::vector()) =0; + /** + * Requests the server unsubscribe the client from a topic. + * @param topicFilter the topic to unsubscribe from. It must match a + * topicFilter specified on an earlier subscribe. + * @return token used to track and wait for the unsubscribe to complete. + * The token will be passed to callback methods if set. + */ + virtual token_ptr unsubscribe(const string& topicFilter) =0; + /** + * Requests the server unsubscribe the client from one or more topics. + * @param topicFilters one or more topics to unsubscribe from. Each + * topicFilter must match one specified on an + * earlier subscribe. + * @return token used to track and wait for the unsubscribe to complete. + * The token will be passed to callback methods if set. + */ + virtual token_ptr unsubscribe(const_string_collection_ptr topicFilters) =0; + /** + * Requests the server unsubscribe the client from one or more topics. + * @param topicFilters one or more topics to unsubscribe from. Each + * topicFilter must match one specified on an + * earlier subscribe. + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param cb listener that will be notified when unsubscribe has + * completed + * @return token used to track and wait for the unsubscribe to complete. + * The token will be passed to callback methods if set. + */ + virtual token_ptr unsubscribe(const_string_collection_ptr topicFilters, + void* userContext, iaction_listener& cb) =0; + /** + * Requests the server unsubscribe the client from a topics. + * @param topicFilter the topic to unsubscribe from. It must match a + * topicFilter specified on an earlier subscribe. + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param cb listener that will be notified when unsubscribe has + * completed + * @return token used to track and wait for the unsubscribe to complete. + * The token will be passed to callback methods if set. + */ + virtual token_ptr unsubscribe(const string& topicFilter, + void* userContext, iaction_listener& cb) =0; +}; + +///////////////////////////////////////////////////////////////////////////// +// end namespace 'mqtt' +} + +#endif // __mqtt_iasync_client_h + diff --git a/src/mqtt/src/mqtt/iclient_persistence.h b/src/mqtt/src/mqtt/iclient_persistence.h new file mode 100644 index 00000000..852ec8ef --- /dev/null +++ b/src/mqtt/src/mqtt/iclient_persistence.h @@ -0,0 +1,135 @@ +///////////////////////////////////////////////////////////////////////////// +/// @file iclient_persistence.h +/// Declaration of MQTT iclient_persistence interface +/// @date May 1, 2013 +/// @author Frank Pagliughi +///////////////////////////////////////////////////////////////////////////// + +/******************************************************************************* + * Copyright (c) 2013-2016 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + *******************************************************************************/ + +#ifndef __mqtt_iclient_persistence_h +#define __mqtt_iclient_persistence_h + +#include "MQTTAsync.h" +#include "mqtt/types.h" +#include "mqtt/buffer_view.h" +#include "mqtt/string_collection.h" +#include + +namespace mqtt { + +///////////////////////////////////////////////////////////////////////////// + +/** + * Represents a persistent data store, used to store outbound and inbound + * messages while they are in flight, enabling delivery to the QoS + * specified. You can specify an implementation of this interface using + * client::client(string, string, iclient_persistence), which the + * client will use to persist QoS 1 and 2 messages. + * + * If the methods defined throw the MqttPersistenceException then the state + * of the data persisted should remain as prior to the method being called. + * For example, if put(string, persistable) throws an exception at any + * point then the data will be assumed to not be in the persistent store. + * Similarly if remove(string) throws an exception then the data will be + * assumed to still be held in the persistent store. + * + * It is up to the persistence interface to log any exceptions or error + * information which may be required when diagnosing a persistence failure. + */ +class iclient_persistence +{ + friend class async_client; + friend class iclient_persistence_test; + + /** Callbacks from the C library */ + static int persistence_open(void** handle, const char* clientID, const char* serverURI, void* context); + static int persistence_close(void* handle); + static int persistence_put(void* handle, char* key, int bufcount, char* buffers[], int buflens[]); + static int persistence_get(void* handle, char* key, char** buffer, int* buflen); + static int persistence_remove(void* handle, char* key); + static int persistence_keys(void* handle, char*** keys, int* nkeys); + static int persistence_clear(void* handle); + static int persistence_containskey(void* handle, char* key); + +public: + /** Smart/shared pointer to an object of this class. */ + using ptr_t = std::shared_ptr; + /** Smart/shared pointer to a const object of this class. */ + using const_ptr_t = std::shared_ptr; + + /** + * Virtual destructor. + */ + virtual ~iclient_persistence() {} + /** + * Initialize the persistent store. + * This uses the client ID and server name to create a unique location + * for the data store. + * @param clientId The identifier string for the client. + * @param serverURI The server to which the client is connected. + */ + virtual void open(const string& clientId, const string& serverURI) =0; + /** + * Close the persistent store that was previously opened. + */ + virtual void close() =0; + /** + * Clears persistence, so that it no longer contains any persisted data. + */ + virtual void clear() =0; + /** + * Returns whether or not data is persisted using the specified key. + * @param key The key to find + * @return @em true if the key exists, @em false if not. + */ + virtual bool contains_key(const string& key) =0; + /** + * Returns a collection of keys in this persistent data store. + * @return A collection of strings representing the keys in the store. + */ + virtual const string_collection& keys() const =0; + /** + * Puts the specified data into the persistent store. + * @param key The key. + * @param bufs The data to store + */ + virtual void put(const string& key, const std::vector& bufs) =0; + /** + * Gets the specified data out of the persistent store. + * @param key The key + * @return A const view of the data associated with the key. + */ + virtual string_view get(const string& key) const =0; + /** + * Remove the data for the specified key. + * @param key The key + */ + virtual void remove(const string& key) =0; +}; + +/** Smart/shared pointer to a persistence client */ +using iclient_persistence_ptr = iclient_persistence::ptr_t; + +/** Smart/shared pointer to a persistence client */ +using const_iclient_persistence_ptr = iclient_persistence::const_ptr_t; + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + +#endif // __mqtt_iclient_persistence_h diff --git a/src/mqtt/src/mqtt/message.h b/src/mqtt/src/mqtt/message.h new file mode 100644 index 00000000..5d64e8d0 --- /dev/null +++ b/src/mqtt/src/mqtt/message.h @@ -0,0 +1,409 @@ +///////////////////////////////////////////////////////////////////////////// +/// @file message.h +/// Declaration of MQTT message class +/// @date May 1, 2013 +/// @author Frank Pagliughi +///////////////////////////////////////////////////////////////////////////// + +/******************************************************************************* + * Copyright (c) 2013-2019 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + * Frank Pagliughi - MQTT v5 support (properties) + *******************************************************************************/ + +#ifndef __mqtt_message_h +#define __mqtt_message_h + +#include "MQTTAsync.h" +#include "mqtt/buffer_ref.h" +#include "mqtt/properties.h" +#include "mqtt/exception.h" +#include + +namespace mqtt { + +///////////////////////////////////////////////////////////////////////////// + +/** + * An MQTT message holds everything required for an MQTT PUBLISH message. + * This holds the binary message payload, topic string, and all the + * additional meta-data for an MQTT message. + * + * The topic and payload buffers are kept as references to const data, so + * they can be reassigned as needed, but the buffers can not be updated + * in-place. Normally they would be created externally then copied or moved + * into the message. The library to transport the messages never touchec the + * payloads or topics. + * + * This also means that message objects are farily cheap to copy, since they + * don't copy the payloads. They simply copy the reference to the buffers. + * It is safe to pass these buffer references across threads since all + * references promise not to update the contents of the buffer. + */ +class message +{ +public: + /** The default QoS for a message */ + static constexpr int DFLT_QOS = 0; + /** The default retained flag */ + static constexpr bool DFLT_RETAINED = false; + +private: + /** Initializer for the C struct (from the C library) */ + static const MQTTAsync_message DFLT_C_STRUCT; + + /** The underlying C message struct */ + MQTTAsync_message msg_; + /** The topic that the message was (or should be) sent on. */ + string_ref topic_; + /** The message payload - an arbitrary binary blob. */ + binary_ref payload_; + /** The properties for the message */ + properties props_; + + /** The client has special access. */ + friend class async_client; + friend class message_test; + + /** + * Set the dup flag in the underlying message + * @param dup + */ + void set_duplicate(bool dup) { msg_.dup = to_int(dup); } + +public: + /** Smart/shared pointer to this class. */ + using ptr_t = std::shared_ptr; + /** Smart/shared pointer to this class. */ + using const_ptr_t = std::shared_ptr; + + /** + * Constructs a message with an empty payload, and all other values set + * to defaults. + */ + message(); + /** + * Constructs a message with the specified array as a payload, and all + * other values set to defaults. + * @param topic The message topic + * @param payload the bytes to use as the message payload + * @param len the number of bytes in the payload + * @param qos The quality of service for the message. + * @param retained Whether the message should be retained by the broker. + */ + message(string_ref topic, const void* payload, size_t len, + int qos, bool retained); + /** + * Constructs a message with the specified array as a payload, and all + * other values set to defaults. + * @param topic The message topic + * @param payload the bytes to use as the message payload + * @param len the number of bytes in the payload + */ + message(string_ref topic, const void* payload, size_t len) + : message(std::move(topic), payload, len, DFLT_QOS, DFLT_RETAINED) {} + /** + * Constructs a message from a byte buffer. + * Note that the payload accepts copy or move semantics. + * @param topic The message topic + * @param payload A byte buffer to use as the message payload. + * @param qos The quality of service for the message. + * @param retained Whether the message should be retained by the broker. + */ + message(string_ref topic, binary_ref payload, int qos, bool retained); + /** + * Constructs a message from a byte buffer. + * Note that the payload accepts copy or move semantics. + * @param topic The message topic + * @param payload A byte buffer to use as the message payload. + */ + message(string_ref topic, binary_ref payload) + : message(std::move(topic), std::move(payload), DFLT_QOS, DFLT_RETAINED) {} + /** + * Constructs a message as a copy of the message structure. + * @param topic The message topic + * @param msg A "C" MQTTAsync_message structure. + */ + message(string_ref topic, const MQTTAsync_message& msg); + /** + * Constructs a message as a copy of the other message. + * @param other The message to copy into this one. + */ + message(const message& other); + /** + * Moves the other message to this one. + * @param other The message to move into this one. + */ + message(message&& other); + /** + * Destroys a message and frees all associated resources. + */ + ~message() {} + + /** + * Constructs a message with the specified array as a payload, and all + * other values set to defaults. + * @param topic The message topic + * @param payload the bytes to use as the message payload + * @param len the number of bytes in the payload + * @param qos The quality of service for the message. + * @param retained Whether the message should be retained by the broker. + */ + static ptr_t create(string_ref topic, const void* payload, size_t len, + int qos, bool retained) { + return std::make_shared(std::move(topic), payload, len, + qos, retained); + } + /** + * Constructs a message with the specified array as a payload, and all + * other values set to defaults. + * @param topic The message topic + * @param payload the bytes to use as the message payload + * @param len the number of bytes in the payload + */ + static ptr_t create(string_ref topic, const void* payload, size_t len) { + return std::make_shared(std::move(topic), payload, len, + DFLT_QOS, DFLT_RETAINED); + } + /** + * Constructs a message from a byte buffer. + * Note that the payload accepts copy or move semantics. + * @param topic The message topic + * @param payload A byte buffer to use as the message payload. + * @param qos The quality of service for the message. + * @param retained Whether the message should be retained by the broker. + */ + static ptr_t create(string_ref topic, binary_ref payload, int qos, bool retained) { + return std::make_shared(std::move(topic), std::move(payload), + qos, retained); + } + /** + * Constructs a message from a byte buffer. + * Note that the payload accepts copy or move semantics. + * @param topic The message topic + * @param payload A byte buffer to use as the message payload. + */ + static ptr_t create(string_ref topic, binary_ref payload) { + return std::make_shared(std::move(topic), std::move(payload), + DFLT_QOS, DFLT_RETAINED); + } + /** + * Constructs a message as a copy of the C message struct. + * @param topic The message topic + * @param msg A "C" MQTTAsync_message structure. + */ + static ptr_t create(string_ref topic, const MQTTAsync_message& msg) { + return std::make_shared(std::move(topic), msg); + } + /** + * Copies another message to this one. + * @param rhs The other message. + * @return A reference to this message. + */ + message& operator=(const message& rhs); + /** + * Moves another message to this one. + * @param rhs The other message. + * @return A reference to this message. + */ + message& operator=(message&& rhs); + /** + * Sets the topic string. + * @param topic The topic on which the message is published. + */ + void set_topic(string_ref topic) { + topic_ = topic_ ? std::move(topic) : string_ref(string()); + } + /** + * Gets the topic reference for the message. + * @return The topic reference for the message. + */ + const string_ref& get_topic_ref() const { return topic_; } + /** + * Gets the topic for the message. + * @return The topic string for the message. + */ + const string& get_topic() const { + static const string EMPTY_STR; + return topic_ ? topic_.str() : EMPTY_STR; + } + /** + * Clears the payload, resetting it to be empty. + */ + void clear_payload(); + /** + * Gets the payload reference. + */ + const binary_ref& get_payload_ref() const { return payload_; } + /** + * Gets the payload + */ + const binary& get_payload() const { + static const binary EMPTY_BIN; + return payload_ ? payload_.str() : EMPTY_BIN; + } + /** + * Gets the payload as a string + */ + const string& get_payload_str() const { + static const string EMPTY_STR; + return payload_ ? payload_.str() : EMPTY_STR; + } + /** + * Returns the quality of service for this message. + * @return The quality of service for this message. + */ + int get_qos() const { return msg_.qos; } + /** + * Returns whether or not this message might be a duplicate of one which + * has already been received. + * @return true this message might be a duplicate of one which + * has already been received, false otherwise + */ + bool is_duplicate() const { return to_bool(msg_.dup); } + /** + * Returns whether or not this message should be/was retained by the + * server. + * @return true if this message should be/was retained by the + * server, false otherwise. + */ + bool is_retained() const { return to_bool(msg_.retained); } + /** + * Sets the payload of this message to be the specified buffer. + * Note that this accepts copy or move operations: + * set_payload(buf); + * set_payload(std::move(buf)); + * @param payload A buffer to use as the message payload. + */ + void set_payload(binary_ref payload); + /** + * Sets the payload of this message to be the specified byte array. + * @param payload the bytes to use as the message payload + * @param n the number of bytes in the payload + */ + void set_payload(const void* payload, size_t n) { + set_payload(binary_ref(static_cast(payload), n)); + } + /** + * Sets the quality of service for this message. + * @param qos The integer Quality of Service for the message + */ + void set_qos(int qos) { + validate_qos(qos); + msg_.qos = qos; + } + /** + * Determines if the QOS value is a valid one. + * @param qos The QOS value. + * @throw std::invalid_argument If the qos value is invalid. + */ + static void validate_qos(int qos) { + if (qos < 0 || qos > 2) + throw exception(MQTTASYNC_BAD_QOS, "Bad QoS"); + } + /** + * Whether or not the publish message should be retained by the broker. + * @param retained @em true if the message should be retained by the + * broker, @em false if not. + */ + void set_retained(bool retained) { msg_.retained = to_int(retained); } + /** + * Gets the properties in the message. + * @return A const reference to the properties in the message. + */ + const properties& get_properties() const { + return props_; + } + /** + * Sets the properties in the message. + * @param props The properties to place into the message. + */ + void set_properties(const properties& props) { + props_ = props; + msg_.properties = props_.c_struct(); + } + /** + * Moves the properties into the message. + * @param props The properties to move into the message. + */ + void set_properties(properties&& props) { + props_ = props; + msg_.properties = props_.c_struct(); + } + /** + * Returns a string representation of this messages payload. + * @return A string representation of this messages payload. + */ + string to_string() const { return get_payload_str(); } +}; + +/** Smart/shared pointer to a message */ +using message_ptr = message::ptr_t; + +/** Smart/shared pointer to a const message */ +using const_message_ptr = message::const_ptr_t; + +/** + * Constructs a message with the specified array as a payload, and all + * other values set to defaults. + * @param topic The message topic + * @param payload the bytes to use as the message payload + * @param len the number of bytes in the payload + */ +inline message_ptr make_message(string_ref topic, const void* payload, size_t len) { + return mqtt::message::create(std::move(topic), payload, len); +} + +/** + * Constructs a message with the specified array as a payload, and all + * other values set to defaults. + * @param topic The message topic + * @param payload the bytes to use as the message payload + * @param len the number of bytes in the payload + * @param qos The quality of service for the message. + * @param retained Whether the message should be retained by the broker. + */ +inline message_ptr make_message(string_ref topic, const void* payload, size_t len, + int qos, bool retained) { + return mqtt::message::create(std::move(topic), payload, len, qos, retained); +} + +/** + * Constructs a message with the specified buffer as a payload, and + * all other values set to defaults. + * @param topic The message topic + * @param payload A string to use as the message payload. + */ +inline message_ptr make_message(string_ref topic, binary_ref payload) { + return mqtt::message::create(std::move(topic), std::move(payload)); +} + +/** + * Constructs a message with the specified values. + * @param topic The message topic + * @param payload A buffer to use as the message payload. + * @param qos The quality of service for the message. + * @param retained Whether the message should be retained by the broker. + */ +inline message_ptr make_message(string_ref topic, binary_ref payload, + int qos, bool retained) { + return mqtt::message::create(std::move(topic), std::move(payload), qos, retained); +} + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + +#endif // __mqtt_message_h + diff --git a/src/mqtt/src/mqtt/properties.h b/src/mqtt/src/mqtt/properties.h new file mode 100644 index 00000000..bbe977c8 --- /dev/null +++ b/src/mqtt/src/mqtt/properties.h @@ -0,0 +1,389 @@ +///////////////////////////////////////////////////////////////////////////// +/// @file properties.h +/// Declaration of MQTT properties class +/// @date July 7, 2019 +/// @author Frank Pagliughi +///////////////////////////////////////////////////////////////////////////// + +/******************************************************************************* + * Copyright (c) 2019 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + *******************************************************************************/ + +#ifndef __mqtt_properties_h +#define __mqtt_properties_h + +extern "C" { + #include "MQTTProperties.h" +} + +#include "mqtt/types.h" +#include "mqtt/buffer_ref.h" +#include "mqtt/exception.h" +#include +#include + +#include + +namespace mqtt { + +using string_pair = std::tuple; + +///////////////////////////////////////////////////////////////////////////// + +/** + * A single MQTT v5 property. + */ +class property +{ + /** The underlying Paho C property struct. */ + MQTTProperty prop_; + + // Make a deep copy of the property struct into this one. + // For string properties, this allocates memory and copied the string(s) + void copy(const MQTTProperty& other); + +public: + enum code { + PAYLOAD_FORMAT_INDICATOR = 1, + MESSAGE_EXPIRY_INTERVAL = 2, + CONTENT_TYPE = 3, + RESPONSE_TOPIC = 8, + CORRELATION_DATA = 9, + SUBSCRIPTION_IDENTIFIER = 11, + SESSION_EXPIRY_INTERVAL = 17, + ASSIGNED_CLIENT_IDENTIFER = 18, + SERVER_KEEP_ALIVE = 19, + AUTHENTICATION_METHOD = 21, + AUTHENTICATION_DATA = 22, + REQUEST_PROBLEM_INFORMATION = 23, + WILL_DELAY_INTERVAL = 24, + REQUEST_RESPONSE_INFORMATION = 25, + RESPONSE_INFORMATION = 26, + SERVER_REFERENCE = 28, + REASON_STRING = 31, + RECEIVE_MAXIMUM = 33, + TOPIC_ALIAS_MAXIMUM = 34, + TOPIC_ALIAS = 35, + MAXIMUM_QOS = 36, + RETAIN_AVAILABLE = 37, + USER_PROPERTY = 38, + MAXIMUM_PACKET_SIZE = 39, + WILDCARD_SUBSCRIPTION_AVAILABLE = 40, + SUBSCRIPTION_IDENTIFIERS_AVAILABLE = 41, + SHARED_SUBSCRIPTION_AVAILABLE = 42 + }; + + /** + * Create a numeric property. + * This can be a byte, or 2-byte, 4-byte, or variable byte integer. + * @param c The property code + * @param val The integer value for the property + */ + property(code c, int32_t val); + /** + * Create a string or binary property. + * @param c The property code + * @param val The value for the property + */ + property(code c, string_ref val); + /** + * Create a string pair property. + * @param c The property code + * @param name The string name for the property + * @param val The string value for the property + */ + property(code c, string_ref name, string_ref val); + /** + * Creates a property list from an C struct. + * @param cprop A C struct for a property list. + */ + explicit property(const MQTTProperty& cprop); + /** + * Moves a C struct into this property list. + * This takes ownership of any memory that the C struct is holding. + * @param cprop A C struct for a property list. + */ + explicit property(MQTTProperty&& cprop); + /** + * Copy constructor + * @param other The other property to copy into this one. + */ + property(const property& other); + /** + * Move constructor. + * @param other The other property that is moved into this one. + */ + property(property&& other); + /** + * Destructor + */ + ~property(); + /** + * Copy assignment. + * @param rhs Another property list to copy into this one. + * @return A reference to this object. + */ + property& operator=(const property& rhs); + /** + * Move assignment. + * @param rhs Another property list to move into this one. + * @return A reference to this object. + */ + property& operator=(property&& rhs); + /** + * Gets the underlying C property struct. + * @return A const reference to the underlying C property + * struct. + */ + const MQTTProperty& c_struct() const { return prop_; } + /** + * Gets the property type (identifier). + * @return The code for the property type. + */ + code type() const { return code(prop_.identifier); } + /** + * Gets a printable name for the property type. + * @return A printable name for the property type. + */ + const char* type_name() const { + return ::MQTTPropertyName(prop_.identifier); + } +}; + +/** + * Extracts the value from the property as the specitied type. + * @return The value from the property as the specitied type. + */ +template +inline T get(const property&) { throw bad_cast(); } + +template <> +inline uint8_t get(const property& prop) { + return (uint8_t) prop.c_struct().value.byte; +} + +template <> +inline uint16_t get(const property& prop) { + return (uint16_t) prop.c_struct().value.integer2; +} + +template <> +inline int16_t get(const property& prop) { + return (int16_t) prop.c_struct().value.integer2; +} + +template <> +inline uint32_t get(const property& prop) { + return (uint32_t) prop.c_struct().value.integer4; +} + +template <> +inline int32_t get(const property& prop) { + return (int32_t) prop.c_struct().value.integer4; +} + +template <> +inline string get(const property& prop) { + return (!prop.c_struct().value.data.data) ? string() + : string(prop.c_struct().value.data.data, prop.c_struct().value.data.len); +} + +template <> +inline string_pair get(const property& prop) { + string name = (!prop.c_struct().value.data.data) ? string() + : string(prop.c_struct().value.data.data, prop.c_struct().value.data.len); + + string value = (!prop.c_struct().value.value.data) ? string() + : string(prop.c_struct().value.value.data, prop.c_struct().value.value.len); + + return std::make_tuple(std::move(name), std::move(value)); +} + +///////////////////////////////////////////////////////////////////////////// + +/** + * MQTT v5 property list. + * + * A collection of properties that can be added to outgoing packets or + * retrieved from incoming packets. + */ +class properties +{ + /** The underlying C properties struct */ + MQTTProperties props_; + + template + friend T get(const properties& props, property::code propid, size_t idx); + + template + friend T get(const properties& props, property::code propid); + +public: + /** + * Default constructor. + * Creates an empty properties list. + */ + properties() { + std::memset(&props_, 0, sizeof(MQTTProperties)); + } + /** + * Copy constructor. + * @param other The property list to copy. + */ + properties(const properties& other) + : props_(::MQTTProperties_copy(&other.props_)) {} + /** + * Move constructor. + * @param other The property list to move to this one. + */ + properties(properties&& other) : props_(other.props_) { + std::memset(&other.props_, 0, sizeof(MQTTProperties)); + } + /** + * Creates a list of properties from a C struct. + * @param cprops The c struct of properties + */ + properties(const MQTTProperties& cprops) { + props_ = ::MQTTProperties_copy(&cprops); + } + /** + * Constructs from a list of property objects. + * @param An initializer list of property objects. + */ + properties(std::initializer_list props); + /** + * Destructor. + */ + ~properties() { ::MQTTProperties_free(&props_); } + /** + * Gets a reference to the underlying C properties structure. + * @return A const reference to the underlying C properties structure. + */ + const MQTTProperties& c_struct() const { return props_; } + /** + * Copy assignment. + * @param rhs The other property list to copy into this one + * @return A reference to this object. + */ + properties& operator=(const properties& rhs) { + if (&rhs != this) { + ::MQTTProperties_free(&props_); + props_ = ::MQTTProperties_copy(&rhs.props_); + } + return *this; + } + /** + * Move assignment + * @param rht The property list to move to this one. + * @return A reference to this object. + */ + properties& operator=(properties&& rhs) { + if (&rhs != this) { + ::MQTTProperties_free(&props_); + props_ = rhs.props_; + std::memset(&rhs.props_, 0, sizeof(MQTTProperties)); + } + return *this; + } + /** + * Determines if the property list is empty. + * @return @em true if there are no properties in the list, @em false if + * the list contains any items. + */ + bool empty() const { return props_.count == 0; } + /** + * Gets the numbers of property items in the list. + * @return The number of property items in the list. + */ + size_t size() const { return size_t(props_.count); } + /** + * Gets the number of bytes required for the serialized + * structure on the wire. + * @return The number of bytes required for the serialized + * struct. + */ + size_t byte_length() const { + return (size_t) ::MQTTProperties_len(const_cast(&props_)); + } + /** + * Adds a property to the list. + * @param prop The property to add to the list. + */ + void add(const property& prop) { + ::MQTTProperties_add(&props_, &prop.c_struct()); + } + /** + * Removes all the items from the property list. + */ + void clear(); + /** + * Determines if the list contains a specific property. + * @param propid The property ID (code). + * @return @em true if the list contains the property, @em false if not. + */ + bool contains(property::code propid) const { + return ::MQTTProperties_hasProperty(const_cast(&props_), + MQTTPropertyCodes(propid)) != 0; + } + /** + * Get the number of properties in the list with the specified property + * ID. + * + * Most properties can exist only once. User properties and subscription + * ID's can exist more than once. + * + * @param propid The property ID (code). + * @return The number of properties in the list with the specified ID. + */ + size_t count(property::code propid) { + return size_t(::MQTTProperties_propertyCount(&props_, MQTTPropertyCodes(propid))); + } + /** + * Gets the property with the specified ID. + * + * @param propid The property ID (code). + * @param idx Which instance of the property to retrieve, if there are + * more than one. + * @return The requested property + */ + property get(property::code propid, size_t idx=0); +}; + +// -------------------------------------------------------------------------- + +template +inline T get(const properties& props, property::code propid, size_t idx) +{ + MQTTProperty* prop = MQTTProperties_getPropertyAt( + const_cast(&props.c_struct()), + MQTTPropertyCodes(propid), int(idx)); + if (!prop) + throw bad_cast(); + + return get(property(*prop)); +} + +template +inline T get(const properties& props, property::code propid) +{ + return get(props, propid, 0); +} + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + +#endif // __mqtt_properties_h + diff --git a/src/mqtt/src/mqtt/response_options.h b/src/mqtt/src/mqtt/response_options.h new file mode 100644 index 00000000..48f78314 --- /dev/null +++ b/src/mqtt/src/mqtt/response_options.h @@ -0,0 +1,121 @@ +///////////////////////////////////////////////////////////////////////////// +/// @file response_options.h +/// Implementation of the class 'response_options' +/// @date 26-Aug-2016 +///////////////////////////////////////////////////////////////////////////// + +#ifndef __mqtt_response_options_h +#define __mqtt_response_options_h + +#include "MQTTAsync.h" +#include "mqtt/token.h" +#include "mqtt/delivery_token.h" +#include "subscribe_options.h" + +namespace mqtt { + +class response_options_test; +class delivery_response_options_test; +class token_test; + +///////////////////////////////////////////////////////////////////////////// +// response_options +///////////////////////////////////////////////////////////////////////////// + +/** + * The response options for various asynchronous calls. + * + * This is an internal data structure, only used within the library. + Therefor it is not totally fleshed out, but rather only exposes the + functionality currently required by the library. + */ +class response_options +{ + /** The underlying C structure */ + MQTTAsync_responseOptions opts_; + + /** The token to which we are connected */ + token::weak_ptr_t tok_; + + /** A list of subscription options for subscribe-many */ + std::vector subOpts_; + + /** The client has special access */ + friend class async_client; + friend class response_options_test; + friend class token_test; + +public: + /** + * Create an empty response object. + */ + explicit response_options(int mqttVersion=MQTTVERSION_DEFAULT); + /** + * Creates a response object with the specified callbacks. + * @param tok A token to be used as the context. + */ + response_options(const token_ptr& tok, int mqttVersion=MQTTVERSION_DEFAULT); + /** + * Sets the callback context to a generic token. + * @param tok The token to be used as the callback context. + */ + void set_token(const token_ptr& tok); + /** + * Sets the options for a single topic subscription. + * @param opts The subscribe options. + */ + void set_subscribe_options(const subscribe_options& opts); + /** + * Sets the options for a multi-topic subscription. + * @param opts A vector of the subscribe options. + */ + void set_subscribe_options(const std::vector& opts); +}; + +///////////////////////////////////////////////////////////////////////////// +// delivery_response_options +///////////////////////////////////////////////////////////////////////////// + +/** + * The response options for asynchronous calls targeted at delivery. + * Each of these objects is tied to a specific delivery_token. + */ +class delivery_response_options +{ + /** The underlying C structure */ + MQTTAsync_responseOptions opts_; + + /** The delivery token to which we are connected */ + delivery_token::weak_ptr_t dtok_; + + /** The client has special access */ + friend class async_client; + friend class delivery_response_options_test; + +public: + /** + * Create an empty delivery response object. + */ + delivery_response_options(int mqttVersion=MQTTVERSION_DEFAULT); + /** + * Creates a response object tied to the specific delivery token. + * @param dtok A delivery token to be used as the context. + */ + delivery_response_options(const delivery_token_ptr& dtok, + int mqttVersion=MQTTVERSION_DEFAULT); + /** + * Sets the callback context to a delivery token. + * @param dtok The delivery token to be used as the callback context. + */ + void set_token(const delivery_token_ptr& dtok) { + dtok_ = dtok; + opts_.context = dtok.get(); + } +}; + +///////////////////////////////////////////////////////////////////////////// +// end namespace 'mqtt' +} + +#endif // __mqtt_response_options_h + diff --git a/src/mqtt/src/mqtt/server_response.h b/src/mqtt/src/mqtt/server_response.h new file mode 100644 index 00000000..7f0fe722 --- /dev/null +++ b/src/mqtt/src/mqtt/server_response.h @@ -0,0 +1,202 @@ +///////////////////////////////////////////////////////////////////////////// +/// @file server_response.h +/// Declaration of MQTT server response classes. +/// @date July 26, 2019 +/// @author Frank Pagliughi +///////////////////////////////////////////////////////////////////////////// + +/******************************************************************************* + * Copyright (c) 2019 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + *******************************************************************************/ + +#ifndef __mqtt_server_response_h +#define __mqtt_server_response_h + +#include "MQTTAsync.h" +#include "mqtt/types.h" +#include "mqtt/properties.h" + +namespace mqtt { + +/** + * Base class for responses from the server. + */ +class server_response +{ + /** The properties from the acknowledge */ + properties props_; + +public: + /** + * Creates a response with empty property list. + */ + server_response() {} + /** + * Creates a server response with the specified properties. + * @param props The properties in the response. + */ + server_response(const properties& props) + : props_(props) {} + /** + * Creates a server response with the specified properties. + * @param props The properties in the response. + */ + server_response(properties&& props) + : props_(std::move(props)) {} + /** + * Virtual destructor. + */ + virtual ~server_response() {} + /** + * Gets the properties from the response. + * @return The properties from the response. + */ + const properties& get_properties() const { return props_; } +}; + +/** + * Response for a connect request + */ +class connect_response : public server_response +{ + /** The connection string of the server */ + string serverURI_; + /** The version of MQTT being used */ + int mqttVersion_; + /** The session present flag returned from the server */ + bool sessionPresent_; + + friend class token; + + connect_response(const MQTTAsync_successData5* rsp) : + server_response(properties(rsp->properties)), + serverURI_(string(rsp->alt.connect.serverURI)), + mqttVersion_(rsp->alt.connect.MQTTVersion), + sessionPresent_(to_bool(rsp->alt.connect.sessionPresent)) { + } + + connect_response(const MQTTAsync_successData* rsp) : + serverURI_(string(rsp->alt.connect.serverURI)), + mqttVersion_(rsp->alt.connect.MQTTVersion), + sessionPresent_(to_bool(rsp->alt.connect.sessionPresent)) { + } + +public: + /** + * Gets the URI of the broker to which we connected. + * @return The URI of the broker. + */ + string get_server_uri() const { return serverURI_; } + /** + * Gets the MQTT version for the connection. + * @return The MQTT version for the connection. + */ + int get_mqtt_version() const { return mqttVersion_; } + /** + * Determines whether a session already existed for this client on the + * server. + * This tells whether the server has a persistent session stored for the + * client, given the ClientID specified in the connect message. + * @return Whether a session already existed for this client on the server. + */ + bool is_session_present() const { return sessionPresent_; } +}; + +/** + * Response for subscribe messages + */ +struct subscribe_response : public server_response +{ + /** The reason/result code for each topic request. */ + std::vector reasonCodes_; + + friend class token; + + subscribe_response(MQTTAsync_successData5* rsp) + : server_response(properties(rsp->properties)) { + if (rsp->alt.sub.reasonCodeCount == 1) + reasonCodes_.push_back(ReasonCode(rsp->reasonCode)); + else { + for (int i=0; ialt.sub.reasonCodeCount; ++i) + reasonCodes_.push_back(ReasonCode(rsp->alt.sub.reasonCodes[i])); + } + } + + subscribe_response(size_t n, MQTTAsync_successData* rsp) { + if (n == 0) + reasonCodes_.push_back(ReasonCode(rsp->alt.qos)); + else + for (size_t i=0; ialt.qosList[i])); + } + +public: + /** + * Gets the reason codes from the server response. + * On a subscribe ack there is a reason code for each topic that + * was sent in the subscribe packet. Each tells the granted QoS + * for the corresponding topic. + * @return A collection of return codes corresponding to + * subscribing each topic. On success, this is the + * granted QoS for each topic. On failure it is the + * reason for the failure. + */ + const std::vector& get_reason_codes() const { + return reasonCodes_; + } +}; + +/** + * Response for unsubscribe messages. + */ +class unsubscribe_response : public server_response +{ + /** The reason/result code for each topic request. */ + std::vector reasonCodes_; + + friend class token; + + unsubscribe_response(MQTTAsync_successData5* rsp) + : server_response(properties(rsp->properties)) { + if (rsp->alt.unsub.reasonCodeCount == 1) + reasonCodes_.push_back(ReasonCode(rsp->reasonCode)); + else { + for (int i=0; ialt.unsub.reasonCodeCount; ++i) + reasonCodes_.push_back(ReasonCode(rsp->alt.unsub.reasonCodes[i])); + } + } + + unsubscribe_response(MQTTAsync_successData* rsp) {} + +public: + /** + * Gets the reason codes from the server response. + * On an unsubscribe ack there is a reason code for each topic + * that was sent in the unsubscribe packet. Each tells the + * result of unsubscribing to the corresponding topic. + * @return A collection of return codes corresponding to + * unsubscribing each topic. + */ + const std::vector& get_reason_codes() const { + return reasonCodes_; + } +}; + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + +#endif // __mqtt_server_response_h + diff --git a/src/mqtt/src/mqtt/ssl_options.h b/src/mqtt/src/mqtt/ssl_options.h new file mode 100644 index 00000000..f44607c8 --- /dev/null +++ b/src/mqtt/src/mqtt/ssl_options.h @@ -0,0 +1,287 @@ +///////////////////////////////////////////////////////////////////////////// +/// @file ssl_options.h +/// Declaration of MQTT ssl_options class +/// @date Jul 7, 2016 +/// @author Guilherme Ferreira +///////////////////////////////////////////////////////////////////////////// + +/******************************************************************************* + * Copyright (c) 2016 Guilherme Ferreira + * Copyright (c) 2016-2019 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Guilherme Ferreira - initial implementation and documentation + * Frank Pagliughi - added copy & move operations + * Frank Pagliughi - upgraded compatibility to Paho C 1.3 + *******************************************************************************/ + +#ifndef __mqtt_ssl_options_h +#define __mqtt_ssl_options_h + +#include "MQTTAsync.h" +#include "mqtt/message.h" +#include "mqtt/topic.h" +#include "mqtt/types.h" +#include + +namespace mqtt { + +///////////////////////////////////////////////////////////////////////////// + +/** + * Holds the set of SSL options for connection. + */ +class ssl_options +{ + /** The default C struct */ + static const MQTTAsync_SSLOptions DFLT_C_STRUCT ; + + /** The underlying C SSL options */ + MQTTAsync_SSLOptions opts_; + + /** + * The file containing the public digital certificates trusted by + * the client. + */ + string trustStore_; + + /** The file containing the public certificate chain of the client. */ + string keyStore_; + + /** The file containing the client's private key. */ + string privateKey_; + + /** The password to load the client's privateKey if encrypted. */ + string privateKeyPassword_; + + /** Path to a directory containing CA certificates in PEM format */ + string caPath_; + + /** + * The list of cipher suites that the client will present to the + * server during the SSL handshake. + */ + string enabledCipherSuites_; + + /** The connect options has special access */ + friend class connect_options; + friend class connect_options_test; + friend class ssl_options_test; + + /** + * Gets a pointer to the C-language NUL-terminated strings for the + * struct. + * @note In the SSL options, by default, the Paho C treats nullptr char + * arrays as unset values, so we keep that semantic and only set those + * char arrays if the string is non-empty. + * @param str The C++ string object. + * @return Pointer to a NUL terminated string. This is only valid until + * the next time the string is updated. + */ + const char* c_str(const string& str) { + return str.empty() ? nullptr : str.c_str(); + } + /** + * Updates the underlying C structure to match our strings. + */ + void update_c_struct(); + +public: + /** Smart/shared pointer to an object of this class. */ + using ptr_t = std::shared_ptr; + /** Smart/shared pointer to a const object of this class. */ + using const_ptr_t = std::shared_ptr; + + /** + * Constructs a new MqttConnectOptions object using the default values. + */ + ssl_options(); + /** + * Argument constructor. + * @param trustStore The file containing the public digital certificates + * trusted by the client. + * @param keyStore The file containing the public certificate chain of the + * client. + * @param privateKey The file containing the client's private key. + * @param privateKeyPassword The password to load the client's privateKey + * if encrypted. + * @param enabledCipherSuites The list of cipher suites that the client + * will present to the server during the SSL handshake. + * @param enableServerCertAuth True/False option to enable verification of + * the server certificate + */ + ssl_options(const string& trustStore, const string& keyStore, + const string& privateKey, const string& privateKeyPassword, + const string& enabledCipherSuites, bool enableServerCertAuth); + /** + * Copy constructor. + * @param opt The other options to copy. + */ + ssl_options(const ssl_options& opt); + /** + * Move constructor. + * @param opt The other options to move to this one. + */ + ssl_options(ssl_options&& opt); + /** + * Copy assignment. + * @param opt The other options to copy. + * @return A reference to this object. + */ + ssl_options& operator=(const ssl_options& opt); + /** + * Move assignment. + * @param opt The other options to move to this one. + * @return A reference to this object. + */ + ssl_options& operator=(ssl_options&& opt); + /** + * Returns the file containing the public digital certificates trusted by + * the client. + * @return string + */ + string get_trust_store() const { return trustStore_; } + /** + * Returns the file containing the public certificate chain of the client. + * @return string + */ + string get_key_store() const { return keyStore_; } + /** + * Returns the file containing the client's private key. + * @return string + */ + string get_private_key() const { return privateKey_; } + /** + * Returns the password to load the client's privateKey if encrypted. + * @return string + */ + string get_private_key_password() const { return privateKeyPassword_; } + /** + * Returns the list of cipher suites that the client will present to the + * server during the SSL handshake. + * @return string + */ + string get_enabled_cipher_suites() const { return enabledCipherSuites_; } + /** + * Returns the true/false to enable verification of the server certificate . + * @return bool + */ + bool get_enable_server_cert_auth() const { + return to_bool(opts_.enableServerCertAuth); + } + /** + * Sets the file containing the public digital certificates trusted by + * the client. + * @param trustStore The file in PEM format containing the public + * digital certificates trusted by the client. + */ + void set_trust_store(const string& trustStore); + /** + * Sets the file containing the public certificate chain of the client. + * @param keyStore The file in PEM format containing the public + * certificate chain of the client. It may also include + * the client's private key. + */ + void set_key_store(const string& keyStore); + /** + * Sets the file containing the client's private key. + * @param privateKey If not included in the sslKeyStore, this is the + * file in PEM format containing the client's private + * key. + */ + void set_private_key(const string& privateKey); + /** + * Sets the password to load the client's privateKey if encrypted. + * @param privateKeyPassword The password to load the privateKey if + * encrypted. + */ + void set_private_key_password(const string& privateKeyPassword); + /** + * Sets the list of cipher suites that the client will present to the server + * during the SSL handshake. + * @param enabledCipherSuites The list of cipher suites that the client + * will present to the server during the SSL + * handshake. For a full explanation of the + * cipher list format, please see the OpenSSL + * on-line documentation: + * http://www.openssl.org/docs/apps/ciphers.html#CIPHER_LIST_FORMAT + * If this setting is ommitted, its default + * value will be "ALL", that is, all the + * cipher suites -excluding those offering no + * encryption- will be considered. This + * setting can be used to set an SSL + * anonymous connection (empty string value, + * for instance). + */ + void set_enabled_cipher_suites(const string& enabledCipherSuites); + /** + * Enables or disables verification of the server certificate. + * @param enablServerCertAuth enable/disable verification of the server + * certificate + */ + void set_enable_server_cert_auth(bool enablServerCertAuth); + /** + * Gets the requested SSL/TLS version. + * @return The requested SSL/TLS version. + */ + int get_ssl_version() const { return opts_.sslVersion; } + /** + * Set the SSL/TLS version to use. + * + * @param ver The desired SSL/TLS version. Specify one of: + * @li MQTT_SSL_VERSION_DEFAULT (0) + * @li MQTT_SSL_VERSION_TLS_1_0 (1) + * @li MQTT_SSL_VERSION_TLS_1_1 (2) + * @li MQTT_SSL_VERSION_TLS_1_2 (3) + */ + void set_ssl_version(int ver) { opts_.sslVersion = ver; } + /** + * Determines whether it will carry out post-connect checks, including + * that a certificate matches the given host name. + * @return Whether it will carry out post-connect checks. + */ + bool get_verify() const { return to_bool(opts_.verify); } + /** + * Sets whether it should carry out post-connect checks, including that + * a certificate matches the given host name. + * @param v Whether it should carry out post-connect checks. + */ + void set_verify(bool v) { opts_.verify = to_int(v); } + /** + * Gets the path to a directory containing CA certificates in PEM + * format. + * + * @return Path to a directory containing CA certificates in PEM format, + * if set. If this isn't set, returns an empty string. + */ + string ca_path() const { return caPath_; } + /** + * Sets the path to a directory containing CA certificates in PEM + * format. + * + * @param path Path to a directory containing CA certificates in PEM + * format. + */ + void ca_path(const string& path); +}; + +/** + * Shared pointer to the ssl options class. + */ +using ssl_options_ptr = ssl_options::ptr_t; + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + +#endif // __mqtt_ssl_options_h + diff --git a/src/mqtt/src/mqtt/string_collection.h b/src/mqtt/src/mqtt/string_collection.h new file mode 100644 index 00000000..548edc13 --- /dev/null +++ b/src/mqtt/src/mqtt/string_collection.h @@ -0,0 +1,241 @@ +///////////////////////////////////////////////////////////////////////////// +/// @file string_collection.h +/// Definition of the string_collection class for the Paho MQTT C++ library. +/// @date April 23, 2017 +/// @author Frank Pagliughi +///////////////////////////////////////////////////////////////////////////// + +/******************************************************************************* + * Copyright (c) 2017 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + *******************************************************************************/ + +#ifndef __mqtt_string_collection_h +#define __mqtt_string_collection_h + +#include "mqtt/types.h" +#include +#include + +namespace mqtt { + +///////////////////////////////////////////////////////////////////////////// + +/** + * Type for a collection of topics. + * This acts like a collection of strings but carries an array of pointers + * to the C strings for easy interactions with the Paho C library. + */ +class string_collection +{ + /** The type for the collection of strings */ + using collection_type = std::vector; + /** The type for the array of C pointers */ + using c_arr_type = std::vector; + + /** + * The collection of strings for the topics. + */ + collection_type coll_; + /** + * A colleciton of pointers to NUL-terminated C strings for the topics. + * This is what is required by the Paho C library, and thus the lifetime + * of the pointers will remain consistent with the lifetime of the + * object. The value is kept consistent with the current topics and + * updated whenever topics are added or removed. + */ + c_arr_type cArr_; + /** + * Updated the cArr_ object to agree with the values in coll_ + * This should be called any time the coll_ variable is modified + * in any way. + */ + void update_c_arr(); + +public: + /** Smart/shared pointer to an object of this type */ + using ptr_t = std::shared_ptr; + /** Smart/shared pointer to a const object of this type */ + using const_ptr_t = std::shared_ptr; + + /** + * Construct an empty string collection. + */ + string_collection() =default; + /** + * Construct a collection initially containing a single string. + * @param str The string + */ + string_collection(const string& str); + /** + * Construct a collection initially containing a single string. + * @param str The string + */ + string_collection(string&& str); + /** + * Constructs a string collection by copying a vector of strings. + * @param vec A vector of strings. + */ + string_collection(const collection_type& vec); + /** + * Constructs a string collection by moving a vector of strings. + * @param vec A vector of strings. + */ + string_collection(collection_type&& vec); + /** + * Copy constructor. + * @param coll An existing string collection. + */ + string_collection(const string_collection& coll); + /** + * Move constructor. + * @param coll An existing string collection. + */ + string_collection(string_collection&& coll) = default; + /** + * Construct a string collection from an initialization list of strings. + * @param sl An initialization list of strings. + */ + string_collection(std::initializer_list sl); + /** + * Construct a string collection from an initialization list of C string + * pointers. + * @param sl An initialization list of C character arrays. + */ + string_collection(std::initializer_list sl); + /** + * Create an empty string collection on the heap. + * @return A smart/shared pointer to a string collection. + */ + static ptr_t create(const string& str) { + return std::make_shared(str); + } + /** + * Create a string collection on the heap, initially containing a single + * string. + * @param str The string + * @return A smart/shared pointer to a string collection. + */ + static ptr_t create(string&& str) { + return std::make_shared(str); + } + /** + * Creates a string collection on the heap by copying a vector of + * strings. + * @param vec A vector of strings. + */ + static ptr_t create(const collection_type& vec) { + return std::make_shared(vec); + } + /** + * Creates a string collection on the heap by copying a vector of + * strings. + * @param vec A vector of strings. + * @return A smart/shared pointer to a string collection. + */ + static ptr_t create(collection_type&& vec) { + return std::make_shared(vec); + } + /** + * Create a string collection on the heap from an initialization list of + * strings. + * @param sl An initialization list of strings. + * @return A smart/shared pointer to a string collection. + */ + static ptr_t create(std::initializer_list sl) { + return std::make_shared(sl); + } + /** + * Create a string collection on the heap from an initialization list of + * C string pointers. + * @param sl An initialization list of C character arrays. + * @return A smart/shared pointer to a string collection. + */ + static ptr_t create(std::initializer_list sl) { + return std::make_shared(sl); + } + /** + * Copy assignment. + * Copy another string collection to this one. + * @param coll A string collection + * @return A reference to this collection. + */ + string_collection& operator=(const string_collection& coll); + /** + * Move assignment. + * Move another string collection to this one. + * @param coll A string collection + * @return A reference to this collection. + */ + string_collection& operator=(string_collection&& coll) = default; + /** + * Determines if the collection is empty. + * @return @em true if the collection is empty, @em false if not. + */ + bool empty() const { return coll_.empty(); } + /** + * Gets the number of strings in the collection. + * @return The number of strings in the collection. + */ + size_t size() const { return coll_.size(); } + /** + * Copies a string to the back of the collection. + * @param str A string. + */ + void push_back(const string& str); + /** + * Moves a string to the back of the collection. + * @param str A string. + */ + void push_back(string&& str); + /** + * Removes all the strings from the collection. + */ + void clear(); + /** + * Gets the n'th string in the collection. + * @param i Index to the desired string. + * @return A const reference to the string. + */ + const string& operator[](size_t i) const { return coll_[i]; } + + /** + * Gets a pointer to an array of NUL-terminated C string pointers. + * This is the collection type supported by the underlying Paho C + * library. The returned pointer is guaranteed valid so long as the + * object is not updated. The return value may change if the object is + * modified, so the application should not cache the return value, but + * rather request the value when needed. + * @return pointer to an array of NUL-terminated C string pointers of + * the topics in the object. + * + */ + char* const* c_arr() const { return (char* const *) cArr_.data(); } + //const char* const* c_arr() const { return cArr_.data(); } +}; + +///////////////////////////////////////////////////////////////////////////// + +/** Smart/shared pointer to a topic collection */ +using string_collection_ptr = string_collection::ptr_t; + +/** Smart/shared pointer to a const string_collection */ +using const_string_collection_ptr = string_collection::const_ptr_t; + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + +#endif // __mqtt_string_collection_h + diff --git a/src/mqtt/src/mqtt/subscribe_options.h b/src/mqtt/src/mqtt/subscribe_options.h new file mode 100644 index 00000000..753c811b --- /dev/null +++ b/src/mqtt/src/mqtt/subscribe_options.h @@ -0,0 +1,170 @@ +///////////////////////////////////////////////////////////////////////////// +/// @file subscribe_options.h +/// Declaration of MQTT subscribe_options class +/// @date Aug 1, 2019 @ +/// @author Frank Pagliughi +///////////////////////////////////////////////////////////////////////////// + +/******************************************************************************* + * Copyright (c) 2019 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + *******************************************************************************/ + +#ifndef __mqtt_subscribe_options_h +#define __mqtt_subscribe_options_h + +#include "MQTTAsync.h" +#include "MQTTSubscribeOpts.h" +#include "mqtt/types.h" + +namespace mqtt { + +///////////////////////////////////////////////////////////////////////////// + +/** + * The MQTT v5 subscription options. + * These are defined in section 3.8.3.1 of the MQTT v5 spec. + * The defaults use the behavior that was present in MQTT v3.1.1. + */ +class subscribe_options +{ + /** The underlying C structure */ + MQTTSubscribe_options opts_; + + friend class async_client; + friend class response_options; + friend class subscribe_options_test; + +public: + /** Smart/shared pointer to an object of this class. */ + using ptr_t = std::shared_ptr; + /** Smart/shared pointer to a const object of this class. */ + using const_ptr_t = std::shared_ptr; + + /** Don't receive our own publications */ + static constexpr bool SUBSCRIBE_NO_LOCAL = true; + /** Receive our own publications */ + static constexpr bool SUBSCRIBE_LOCAL = false; + + /** + * Retain flag is only set on publications sent by a broker if in + * response to a subscribe request + */ + static constexpr bool NO_RETAIN_AS_PUBLISHED = false; + /** Keep the retain flag as on the original publish message */ + static constexpr bool RETAIN_AS_PUBLISHED = true; + + /** The options for subscription retain handling */ + enum RetainHandling { + /** Send retained messages at the time of the subscribe */ + SEND_RETAINED_ON_SUBSCRIBE = 0, + /** Send retained messages on subscribe only if subscription is new */ + SEND_RETAINED_ON_NEW = 1, + /** Do not send retained messages at all */ + DONT_SEND_RETAINED = 2 + }; + + /** + * Create default subscription options. + * These are the default options corresponding to the original MQTT (v3) + * behaviors. + */ + subscribe_options() + : opts_(MQTTSubscribe_options_initializer) {} + /** + * Creates a set of subscription options. + * + * @param noLocal Whether the server should send back our own + * publications, if subscribed. + * @param retainAsPublished Whether to keep the retained flag as in the + * original published message (true). + * @param retainHandling When to send retained messages: + * @li (SEND_RETAINED_ON_SUBSCRIBE, 0) At the time of the subscribe + * @li (SEND_RETAINED_ON_NEW, 1) Only at the time of a new + * subscribe + * @li (DONT_SEND_RETAINED, 2) Not at the time of subscribe + */ + explicit subscribe_options(bool noLocal, byte retainAsPublished=false, + RetainHandling retainHandling=SEND_RETAINED_ON_SUBSCRIBE) + : opts_(MQTTSubscribe_options_initializer) { + opts_.noLocal = noLocal ? 1 : 0; + opts_.retainAsPublished = retainAsPublished ? 1 : 0; + opts_.retainHandling = (unsigned char) retainHandling; + } + /** + * Gets the value of the "no local" flag. + * @return Whether the server should send back our own publications, if + * subscribed. + */ + bool get_no_local() const { + return to_bool(opts_.noLocal); + } + /** + * Sets the "no local" flag on or off. + * @param on Whether the server should send back our own publications, + * if subscribed. + */ + void set_no_local(bool on=true) { + opts_.noLocal = on ? 1 : 0; + } + /** + * Gets the "retain as published" flag. + * @return Whether to keep the retained flag as in the original + * published message. + */ + bool get_retain_as_published() const { + return to_bool(opts_.retainAsPublished); + } + /** + * Sets the "retain as published" flag on or off. + * @param on Whether to keep the retained flag as in the original + * published message. + */ + void set_retain_as_published(bool on) { + opts_.retainAsPublished = on ? 1 : 0; + } + /** + * Gets the "retasin handling" option. + * @return When to send retained messages: + * @li (SEND_RETAINED_ON_SUBSCRIBE, 0) At the time of the subscribe + * @li (SEND_RETAINED_ON_NEW, 1) Only at the time of a new + * subscribe + * @li (DONT_SEND_RETAINED, 2) Not at the time of subscribe + */ + auto get_retain_handling() const -> RetainHandling { + return RetainHandling(opts_.retainHandling); + } + /** + * Sets the "retain handling" option. + * @param retainHandling When to send retained messages: + * @li (SEND_RETAINED_ON_SUBSCRIBE, 0) At the time of the subscribe + * @li (SEND_RETAINED_ON_NEW, 1) Only at the time of a new + * subscribe + * @li (DONT_SEND_RETAINED, 2) Not at the time of subscribe + */ + void set_retain_handling(RetainHandling retainHandling) { + opts_.retainHandling = (unsigned char) retainHandling; + } +}; + + +/** Smart/shared pointer to a subscribe options object. */ +using subscribe_options_ptr = subscribe_options::ptr_t; + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + +#endif // __mqtt_subscribe_options_h + diff --git a/src/mqtt/src/mqtt/thread_queue.h b/src/mqtt/src/mqtt/thread_queue.h new file mode 100644 index 00000000..5dafe6eb --- /dev/null +++ b/src/mqtt/src/mqtt/thread_queue.h @@ -0,0 +1,332 @@ +///////////////////////////////////////////////////////////////////////////// +/// @file thread_queue.h +/// Implementation of the template class 'thread_queue', a thread-safe, +/// blocking queue for passing data between threads, safe for use with smart +/// pointers. +/// @date 09-Jan-2017 +///////////////////////////////////////////////////////////////////////////// + +/******************************************************************************* + * Copyright (c) 2017 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + *******************************************************************************/ + +#ifndef __mqtt_thread_queue_h +#define __mqtt_thread_queue_h + +#include +#include +#include +#include +#include +#include +#include + +namespace mqtt { + +///////////////////////////////////////////////////////////////////////////// + +/** + * A thread-safe queue for inter-thread communication. + * This is a lockinq queue with blocking operations. The get() operations + * can always block on an empty queue, but have variations for non-blocking + * (try_get) and bounded-time blocking (try_get_for, try_get_until). + * @par + * The default queue has a capacity that is unbounded in the practical + * sense, limited by available memory. In this mode the object will not + * block when placing values into the queue. A capacity can bet set with the + * construtor or, at any time later by calling the @ref capacity(size_type) + * method. Using this latter method, the capacity can be set to an amount + * smaller than the current size of the queue. In that case all put's to the + * queue will block until the number of items are removed from the queue to + * bring the size below the new capacity. + * @par + * Note that the queue uses move semantics to place items into the queue and + * remove items from the queue. This means that the type, T, of the data + * held by the queue only needs to follow move semantics; not copy + * semantics. In addition, this means that copies of the value will @em not + * be left in the queue. This is especially useful when creating queues of + * shared pointers, as the "dead" part of the queue will not hold onto a + * reference count after the item has been removed from the queue. + * + * @param T The type of the items to be held in the queue. + * @param Container The type of the underlying container to use. It must + * support back(), front(), push_back(), pop_front(). + */ +template > +class thread_queue +{ +public: + /** The underlying container type to use for the queue. */ + using container_type = Container; + /** The type of items to be held in the queue. */ + using value_type = T; + /** The type used to specify number of items in the container. */ + using size_type = typename Container::size_type; + + /** The maximum capacity of the queue. */ + static constexpr size_type MAX_CAPACITY = std::numeric_limits::max(); + +private: + /** Object lock */ + mutable std::mutex lock_; + /** Condition get signaled when item added to empty queue */ + std::condition_variable notEmptyCond_; + /** Condition gets signaled then item removed from full queue */ + std::condition_variable notFullCond_; + /** The capacity of the queue */ + size_type cap_; + /** The actual STL container to hold data */ + std::queue que_; + + /** Simple, scope-based lock guard */ + using guard = std::lock_guard; + /** General purpose guard */ + using unique_guard = std::unique_lock; + +public: + /** + * Constructs a queue with the maximum capacity. + */ + thread_queue() : cap_(MAX_CAPACITY) {} + /** + * Constructs a queue with the specified capacity. + * @param cap The maximum number of items that can be placed in the + * queue. The minimum capacity is 1. + */ + explicit thread_queue(size_t cap) : cap_(std::max(cap, 1)) {} + /** + * Determine if the queue is empty. + * @return @em true if there are no elements in the queue, @em false if + * there are any items in the queue. + */ + bool empty() const { + guard g(lock_); + return que_.empty(); + } + /** + * Gets the capacity of the queue. + * @return The maximum number of elements before the queue is full. + */ + size_type capacity() const { + guard g(lock_); + return cap_; + } + /** + * Sets the capacity of the queue. + * Note that the capacity can be set to a value smaller than the current + * size of the queue. In that event, all calls to put() will block until + * a suffucuent number + */ + void capacity(size_type cap) { + guard g(lock_); + cap_ = cap; + } + /** + * Gets the number of items in the queue. + * @return The number of items in the queue. + */ + size_type size() const { + guard g(lock_); + return que_.size(); + } + /** + * Put an item into the queue. + * If the queue is full, this will block the caller until items are + * removed bringing the size less than the capacity. + * @param val The value to add to the queue. + */ + void put(value_type val) { + unique_guard g(lock_); + if (que_.size() >= cap_) + notFullCond_.wait(g, [=]{return que_.size() < cap_;}); + bool wasEmpty = que_.empty(); + que_.emplace(std::move(val)); + if (wasEmpty) { + g.unlock(); + notEmptyCond_.notify_one(); + } + } + /** + * Non-blocking attempt to place an item into the queue. + * @param val The value to add to the queue. + * @return @em true if the item was added to the queue, @em false if the + * item was not added because the queue is currently full. + */ + bool try_put(value_type val) { + unique_guard g(lock_); + size_type n = que_.size(); + if (n >= cap_) + return false; + que_.emplace(std::move(val)); + if (n == 0) { + g.unlock(); + notEmptyCond_.notify_one(); + } + return true; + } + /** + * Attempt to place an item in the queue with a bounded wait. + * This will attempt to place the value in the queue, but if it is full, + * it will wait up to the specified time duration before timing out. + * @param val The value to add to the queue. + * @param relTime The amount of time to wait until timing out. + * @return @em true if the value was added to the queue, @em false if a + * timeout occurred. + */ + template + bool try_put_for(value_type* val, const std::chrono::duration& relTime) { + unique_guard g(lock_); + if (que_.size() >= cap_ && !notFullCond_.wait_for(g, relTime, [=]{return que_.size() < cap_;})) + return false; + bool wasEmpty = que_.empty(); + que_.emplace(std::move(val)); + if (wasEmpty) { + g.unlock(); + notEmptyCond_.notify_one(); + } + return true; + } + /** + * Attempt to place an item in the queue with a bounded wait to an + * absolute time point. + * This will attempt to place the value in the queue, but if it is full, + * it will wait up until the specified time before timing out. + * @param val The value to add to the queue. + * @param absTime The absolute time to wait to before timing out. + * @return @em true if the value was added to the queue, @em false if a + * timeout occurred. + */ + template + bool try_put_until(value_type* val, const std::chrono::time_point& absTime) { + unique_guard g(lock_); + if (que_.size() >= cap_ && !notFullCond_.wait_until(g, absTime, [=]{return que_.size() < cap_;})) + return false; + bool wasEmpty = que_.empty(); + que_.emplace(std::move(val)); + if (wasEmpty) { + g.unlock(); + notEmptyCond_.notify_one(); + } + return true; + } + /** + * Retrieve a value from the queue. + * If the queue is empty, this will block indefinitely until a value is + * added to the queue by another thread, + * @param val Pointer to a variable to receive the value. + */ + void get(value_type* val) { + unique_guard g(lock_); + if (que_.empty()) + notEmptyCond_.wait(g, [=]{return !que_.empty();}); + *val = std::move(que_.front()); + que_.pop(); + if (que_.size() == cap_-1) { + g.unlock(); + notFullCond_.notify_one(); + } + } + /** + * Retrieve a value from the queue. + * If the queue is empty, this will block indefinitely until a value is + * added to the queue by another thread, + * @return The value removed from the queue + */ + value_type get() { + unique_guard g(lock_); + if (que_.empty()) + notEmptyCond_.wait(g, [=]{return !que_.empty();}); + value_type val = std::move(que_.front()); + que_.pop(); + if (que_.size() == cap_-1) { + g.unlock(); + notFullCond_.notify_one(); + } + return val; + } + /** + * Attempts to remove a value from the queue without blocking. + * If the queue is currently empty, this will return immediately with a + * failure, otherwise it will get the next value and return it. + * @param val Pointer to a variable to receive the value. + * @return @em true if a value was removed from the queue, @em false if + * the queue is empty. + */ + bool try_get(value_type* val) { + unique_guard g(lock_); + if (que_.empty()) + return false; + *val = std::move(que_.front()); + que_.pop(); + if (que_.size() == cap_-1) { + g.unlock(); + notFullCond_.notify_one(); + } + return true; + } + /** + * Attempt to remove an item from the queue for a bounded amout of time. + * This will retrieve the next item from the queue. If the queue is + * empty, it will wait the specified amout of time for an item to arive + * before timing out. + * @param val Pointer to a variable to receive the value. + * @param relTime The amount of time to wait until timing out. + * @return @em true if the value was removed the queue, @em false if a + * timeout occurred. + */ + template + bool try_get_for(value_type* val, const std::chrono::duration& relTime) { + unique_guard g(lock_); + if (que_.empty() && !notEmptyCond_.wait_for(g, relTime, [=]{return !que_.empty();})) + return false; + *val = std::move(que_.front()); + que_.pop(); + if (que_.size() == cap_-1) { + g.unlock(); + notFullCond_.notify_one(); + } + return true; + } + /** + * Attempt to remove an item from the queue for a bounded amout of time. + * This will retrieve the next item from the queue. If the queue is + * empty, it will wait until the specified time for an item to arive + * before timing out. + * @param val Pointer to a variable to receive the value. + * @param absTime The absolute time to wait to before timing out. + * @return @em true if the value was removed from the queue, @em false + * if a timeout occurred. + */ + template + bool try_get_until(value_type* val, const std::chrono::time_point& absTime) { + unique_guard g(lock_); + if (que_.empty() && !notEmptyCond_.wait_until(g, absTime, [=]{return !que_.empty();})) + return false; + *val = std::move(que_.front()); + que_.pop(); + if (que_.size() == cap_-1) { + g.unlock(); + notFullCond_.notify_one(); + } + return true; + } +}; + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + +#endif // __mqtt_thread_queue_h + diff --git a/src/mqtt/src/mqtt/token.h b/src/mqtt/src/mqtt/token.h new file mode 100644 index 00000000..efb1659b --- /dev/null +++ b/src/mqtt/src/mqtt/token.h @@ -0,0 +1,504 @@ +///////////////////////////////////////////////////////////////////////////// +/// @file token.h +/// Declaration of MQTT token class +/// @date May 1, 2013 +/// @author Frank Pagliughi +///////////////////////////////////////////////////////////////////////////// + +/******************************************************************************* + * Copyright (c) 2013-2019 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + * Frank Pagliughi - MQTT v5 support & server responses + *******************************************************************************/ + +#ifndef __mqtt_token_h +#define __mqtt_token_h + +#include "MQTTAsync.h" +#include "mqtt/iaction_listener.h" +#include "mqtt/exception.h" +#include "mqtt/types.h" +#include "mqtt/properties.h" +#include "mqtt/buffer_ref.h" +#include "mqtt/string_collection.h" +#include "mqtt/server_response.h" +#include +#include +#include +#include +#include + +namespace mqtt { + +class iasync_client; + +///////////////////////////////////////////////////////////////////////////// + +/** + * Provides a mechanism for tracking the completion of an asynchronous + * action. + */ +class token +{ +public: + /** Smart/shared pointer to an object of this class */ + using ptr_t = std::shared_ptr; + /** Smart/shared pointer to an object of this class */ + using const_ptr_t = std::shared_ptr; + /** Weak pointer to an object of this class */ + using weak_ptr_t = std::weak_ptr; + + /** The type of request that the token is tracking */ + enum Type { + CONNECT, + SUBSCRIBE, + PUBLISH, + UNSUBSCRIBE, + DISCONNECT + }; + +private: + /** Lock guard type for this class. */ + using guard = std::lock_guard; + /** Unique type for this class. */ + using unique_lock = std::unique_lock; + + /** Object monitor mutex. */ + mutable std::mutex lock_; + /** Condition variable signals when the action completes */ + mutable std::condition_variable cond_; + + /** The type of request that the token is tracking */ + Type type_; + /** The MQTT client that is processing this action */ + iasync_client* cli_; + /** The action success/failure code */ + int rc_; + /** MQTT v5 reason code */ + ReasonCode reasonCode_; + /** Error message from the C lib (if any) */ + string errMsg_; + /** The underlying C token. Note that this is just an integer */ + MQTTAsync_token msgId_; + /** The topic string(s) for the action being tracked by this token */ + const_string_collection_ptr topics_; + /** User supplied context */ + void* userContext_; + + /** + * User supplied listener. + * Note that the user listener fires after the action is marked + * complete, but before the token is signaled. + */ + iaction_listener* listener_; + /** The number of expected responses */ + size_t nExpected_; + /** Whether the action has yet to complete */ + bool complete_; + + /** MQTT v5 propeties */ + //properties props_; + /** Connection response (null if not available) */ + std::unique_ptr connRsp_; + /** Subscribe response (null if not available) */ + std::unique_ptr subRsp_; + /** Unsubscribe response (null if not available) */ + std::unique_ptr unsubRsp_; + + /** Client and token-related options have special access */ + friend class async_client; + friend class token_test; + + friend class connect_options; + friend class response_options; + friend class delivery_response_options; + friend class disconnect_options; + + /** + * Resets the token back to a non-signaled state. + */ + void reset(); + /** + * Sets the ID for the message. + * This is a guaranteed atomic operation. + * @param msgId The ID of the message. + */ + void set_message_id(MQTTAsync_token msgId) { + guard g(lock_); + msgId_ = msgId; + } + /** + * C-style callback for success. + * This simply passes the call on to the proper token object for + * processing. + * @param tokObj The token object to process the call. Note that this is + * @em not the user-supplied context pointer. That is + * kept in the object itself. + * @param rsp The success response. + */ + static void on_success(void* tokObj, MQTTAsync_successData* rsp); + static void on_success5(void* tokObj, MQTTAsync_successData5* rsp); + /** + * C-style callback for failure. + * This simply passes the call on to the proper token object for + * processing. + * @param tokObj The token object to process the call. Note that this is + * @em not the user-supplied context pointer. That is + * kept in the object itself. + * @param rsp The failure response. + */ + static void on_failure(void* tokObj, MQTTAsync_failureData* rsp); + static void on_failure5(void* tokObj, MQTTAsync_failureData5* rsp); + /** + * C-style callback for client (re)connection. + * This is normally only used to process a reconnect completion message. + * The initial connect() is processed via on_success/failure. + * @param tokObj Pointer to the token object used to process the call. + */ + static void on_connected(void* tokObj, char* /*cause*/); + /** + * Internal handler for the success callback. + * @param rsp The success response. + */ + void on_success(MQTTAsync_successData* rsp); + void on_success5(MQTTAsync_successData5* rsp); + /** + * Internal handler for the failure callback. + * @param rsp The failure response. + */ + void on_failure(MQTTAsync_failureData* rsp); + void on_failure5(MQTTAsync_failureData5* rsp); + + /** + * Check the current return code and throw an exception if it is not a + * success code. + */ + void check_ret() const { + if (rc_ != MQTTASYNC_SUCCESS || reasonCode_ > ReasonCode::GRANTED_QOS_2) + throw exception(rc_, reasonCode_, errMsg_); + } + +public: + /** + * Constructs a token object. + * @param cli The client that created the token. + */ + token(Type typ, iasync_client& cli) + : token(typ, cli, MQTTAsync_token(0)) {} + /** + * Constructs a token object. + * @param cli The client that created the token. + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param cb callback listener that will be notified when subscribe has + * completed + */ + token(Type typ, iasync_client& cli, void* userContext, iaction_listener& cb) + : token(typ, cli, const_string_collection_ptr(), userContext, cb) {} + + /** + * Constructs a token object. + * @param cli The client that created the token. + * @param topic The topic assiciated with the token + */ + token(Type typ, iasync_client& cli, const string& topic) + : token(typ, cli, string_collection::create(topic)) {} + /** + * Constructs a token object. + * @param cli The client that created the token. + * @param topic The topic assiciated with the token + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param cb callback listener that will be notified when subscribe has + * completed + */ + token(Type typ, iasync_client& cli, const string& topic, + void* userContext, iaction_listener& cb) + : token(typ, cli, string_collection::create(topic), userContext, cb) {} + + /** + * Constructs a token object. + * @param cli The client that created the token. + * @param topics The topics associated with the token + */ + token(Type typ, iasync_client& cli, const_string_collection_ptr topics); + /** + * Constructs a token object. + * @param cli The client that created the token. + * @param topics The topics associated with the token + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param cb callback listener that will be notified when subscribe has + * completed + */ + token(Type typ, iasync_client& cli, const_string_collection_ptr topics, + void* userContext, iaction_listener& cb); + /** + * Constructs a token object. + * @param cli The client that created the token. + * @param tok The message ID + */ + token(Type typ, iasync_client& cli, MQTTAsync_token tok); + /** + * Virtual destructor. + */ + virtual ~token() {} + /** + * Constructs a token object. + * @param cli The client that created the token. + * @return A smart/shared pointer to a token. + */ + static ptr_t create(Type typ, iasync_client& cli) { + return std::make_shared(typ, cli); + } + /** + * Constructs a token object. + * @param cli The client that created the token. + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param cb callback listener that will be notified when subscribe has + * completed + */ + static ptr_t create(Type typ, iasync_client& cli, void* userContext, + iaction_listener& cb) { + return std::make_shared(typ, cli, userContext, cb); + } + /** + * Constructs a token object. + * @param cli The client that created the token. + * @param topic The topic assiciated with the token + */ + static ptr_t create(Type typ, iasync_client& cli, const string& topic) { + return std::make_shared(typ, cli, topic); + } + /** + * Constructs a token object. + * @param cli The client that created the token. + * @param topic The topic assiciated with the token + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param cb callback listener that will be notified when subscribe has + * completed + */ + static ptr_t create(Type typ, iasync_client& cli, const string& topic, + void* userContext, iaction_listener& cb) { + return std::make_shared(typ, cli, topic, userContext, cb); + } + /** + * Constructs a token object. + * @param cli The client that created the token. + * @param topics The topics associated with the token + */ + static ptr_t create(Type typ, iasync_client& cli, const_string_collection_ptr topics) { + return std::make_shared(typ, cli, topics); + } + /** + * Constructs a token object. + * @param cli The client that created the token. + * @param topics The topics associated with the token + * + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + * @param cb callback listener that will be notified when subscribe has + */ + static ptr_t create(Type typ, iasync_client& cli, const_string_collection_ptr topics, + void* userContext, iaction_listener& cb) { + return std::make_shared(typ, cli, topics, userContext, cb); + } + /** + * Gets the type of action the token is tracking, like CONNECT, PUBLISH, + * etc. + * @return The type of action the token is tracking. + */ + Type get_type() const { return type_; } + /** + * Gets the action listener for this token. + * @return The action listener for this token. + */ + virtual iaction_listener* get_action_callback() const { + guard g(lock_); + return listener_; + } + /** + * Returns the MQTT client that is responsible for processing the + * asynchronous action. + * @return The client to which this token is connected. + */ + virtual iasync_client* get_client() const { return cli_; } + /** + * Returns the ID of the message that is associated with the token. + * @return The message ID of the transaction being tracked. + */ + virtual int get_message_id() const { + static_assert(sizeof(msgId_) <= sizeof(int), "MQTTAsync_token must fit into int"); + return int(msgId_); + } + /** + * Gets the topic string(s) for the action being tracked by this + * token. + * @return A const pointer to the collection of topics being tracked by + * the token. + */ + virtual const_string_collection_ptr get_topics() const { + return topics_; + } + /** + * Retrieve the context associated with an action. + * @return The context associated with an action. + */ + virtual void* get_user_context() const { + guard g(lock_); + return userContext_; + } + /** + * Returns whether or not the action has finished. + * @return @em true if the transaction has completed, @em false if not. + */ + virtual bool is_complete() const { return complete_; } + /** + * Gets the return code from the action. + * This is only valid after the action has completed (i.e. if @ref + * is_complete() returns @em true). + * @return The return code from the action. + */ + virtual int get_return_code() const { return rc_; } + /** + * Register a listener to be notified when an action completes. + * @param listener The callback to be notified when actions complete. + */ + virtual void set_action_callback(iaction_listener& listener) { + guard g(lock_); + listener_ = &listener; + } + /** + * Store some context associated with an action. + * @param userContext optional object used to pass context to the + * callback. Use @em nullptr if not required. + */ + virtual void set_user_context(void* userContext) { + guard g(lock_); + userContext_ = userContext; + } + /** + * Sets the number of results expected. + * This is only required for subecribe_many() with < MQTTv5 + * @param n The number of results expected. + */ + void set_num_expected(size_t n) { nExpected_ = n; } + + /** + * Gets the properties for the operation. + * @return A const reference to the properties for the operation + */ + //const properties& get_properties() const { return props_; } + /** + * Gets the reason code for the operation. + * @return The reason code for the operation. + */ + ReasonCode get_reason_code() const { return reasonCode_; } + /** + * Blocks the current thread until the action this token is associated + * with has completed. + */ + virtual void wait(); + /** + * Non-blocking check to see if the action has completed. + * @return @em true if the wait finished successfully, @em false if the + * action has not completed yet. + */ + virtual bool try_wait() { + guard g(lock_); + if (complete_) + check_ret(); + return complete_; + } + /** + * Blocks the current thread until the action this token is associated + * with has completed. + * @param timeout The timeout (in milliseconds) + * @return @em true if the wait finished successfully, @em false if a + * timeout occurred. + */ + virtual bool wait_for(long timeout) { + return wait_for(std::chrono::milliseconds(timeout)); + } + /** + * Waits a relative amount of time for the action to complete. + * @param relTime The amount of time to wait for the event. + * @return @em true if the event gets signaled in the specified time, + * @em false on a timeout. + */ + template + bool wait_for(const std::chrono::duration& relTime) { + unique_lock g(lock_); + if (!cond_.wait_for(g, std::chrono::milliseconds(relTime), + [this]{return complete_;})) + return false; + check_ret(); + return true; + } + /** + * Waits until an absolute time for the action to complete. + * @param absTime The absolute time to wait for the event. + * @return @em true if the event gets signaled in the specified time, + * @em false on a timeout. + */ + template + bool wait_until( const std::chrono::time_point& absTime) { + unique_lock g(lock_); + if (!cond_.wait_until(g, absTime, [this]{return complete_;})) + return false; + check_ret(); + return true; + } + + /** + * Gets the response from a connect operation. + * This returns the result of the completed operation. If the + * operaton is not yet complete this will block until the result + * is available. + * @return The result of the operation. + */ + connect_response get_connect_response() const; + /** + * Gets the response from a connect operation. + * This returns the result of the completed operation. If the + * operaton is not yet complete this will block until the result + * is available. + * @return The result of the operation. + */ + subscribe_response get_subscribe_response() const; + /** + * Gets the response from a connect operation. + * This returns the result of the completed operation. If the + * operaton is not yet complete this will block until the result + * is available. + * @return The result of the operation. + */ + unsubscribe_response get_unsubscribe_response() const; +}; + +/** Smart/shared pointer to a token object */ +using token_ptr = token::ptr_t; + +/** Smart/shared pointer to a const token object */ +using const_token_ptr = token::const_ptr_t; + + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + +#endif // __mqtt_token_h + diff --git a/src/mqtt/src/mqtt/topic.h b/src/mqtt/src/mqtt/topic.h new file mode 100644 index 00000000..618eb85e --- /dev/null +++ b/src/mqtt/src/mqtt/topic.h @@ -0,0 +1,180 @@ +///////////////////////////////////////////////////////////////////////////// +/// @file topic.h +/// Declaration of MQTT topic class +/// @date May 1, 2013 +/// @author Frank Pagliughi +///////////////////////////////////////////////////////////////////////////// + +/******************************************************************************* + * Copyright (c) 2013-2016 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + *******************************************************************************/ + +#ifndef __mqtt_topic_h +#define __mqtt_topic_h + +#include "MQTTAsync.h" +#include "mqtt/delivery_token.h" +#include "mqtt/subscribe_options.h" +#include "mqtt/message.h" +#include "mqtt/types.h" +#include + +namespace mqtt { + +class iasync_client; + +///////////////////////////////////////////////////////////////////////////// + +/** + * Represents a topic destination, used for publish/subscribe messaging. + */ +class topic +{ + /** The client to which this topic is connected */ + iasync_client& cli_; + /** The topic name */ + string name_; + /** The default QoS */ + int qos_; + /** The default retined flag */ + bool retained_; + +public: + /** A smart/shared pointer to this class. */ + using ptr_t = std::shared_ptr; + /** A smart/shared pointer to this class. */ + using const_ptr_t = std::shared_ptr; + + /** + * Construct an MQTT topic destination for messages. + * @param cli Client to which the topic is attached + * @param name The topic string + * @param qos The default QoS for publishing. + * @param retained The default retained flag for the topic. + */ + topic(iasync_client& cli, const string& name, + int qos=message::DFLT_QOS, bool retained=message::DFLT_RETAINED) + : cli_(cli), name_(name), qos_(qos), retained_(retained) {} + /** + * Creates a new topic + * @param cli Client to which the topic is attached + * @param name The topic string + * @param qos The default QoS for publishing. + * @param retained The default retained flag for the topic. + * @return A shared pointer to the topic. + */ + static ptr_t create(iasync_client& cli, const string& name, + int qos=message::DFLT_QOS, + bool retained=message::DFLT_RETAINED) { + return std::make_shared(cli, name, qos, retained); + } + /** + * Gets a reference to the MQTT client used by this topic + * @return The MQTT client used by this topic + */ + iasync_client& get_client() { return cli_; } + /** + * Gets the name of the topic. + * @return The name of the topic. + */ + const string& get_name() const { return name_; } + /** + * Gets the default quality of service for this topic. + * @return The default quality of service for this topic. + */ + int get_qos() const { return qos_; } + /** + * Gets the default retained flag used for this topic. + * @return The default retained flag used for this topic. + */ + bool get_retained() const { return retained_; } + /** + * Sets the default quality of service for this topic. + * @param qos The default quality of service for this topic. + */ + void set_qos(int qos) { + message::validate_qos(qos); + qos_ = qos; + } + /** + * Sets the default retained flag used for this topic. + * @param retained The default retained flag used for this topic. + */ + void set_retained(bool retained) { retained_ = retained; } + /** + * Publishes a message on the topic using the default QoS and retained + * flag. + * @param payload the bytes to use as the message payload + * @param n the number of bytes in the payload + * @return The delivery token used to track and wait for the publish to + * complete. + */ + delivery_token_ptr publish(const void* payload, size_t n); + /** + * Publishes a message on the topic. + * @param payload the bytes to use as the message payload + * @param n the number of bytes in the payload + * @param qos the Quality of Service to deliver the message at. Valid + * values are 0, 1 or 2. + * @param retained whether or not this message should be retained by the + * server. + * @return The delivery token used to track and wait for the publish to + * complete. + */ + delivery_token_ptr publish(const void* payload, size_t n, + int qos, bool retained); + /** + * Publishes a message on the topic using the default QoS and retained + * flag. + * @param payload the bytes to use as the message payload + * @return The delivery token used to track and wait for the publish to + * complete. + */ + delivery_token_ptr publish(binary_ref payload); + /** + * Publishes a message on the topic. + * @param payload the bytes to use as the message payload + * @param qos the Quality of Service to deliver the message at. Valid + * values are 0, 1 or 2. + * @param retained whether or not this message should be retained by the + * server. + * @return The delivery token used to track and wait for the publish to + * complete. + */ + delivery_token_ptr publish(binary_ref payload, int qos, bool retained); + /** + * Subscribe to the topic. + * @return A token used to track the progress of the operation. + */ + token_ptr subscribe(const subscribe_options& opts=subscribe_options()); + /** + * Returns a string representation of this topic. + * @return The name of the topic + */ + string to_string() const { return name_; } +}; + +/** A smart/shared pointer to a topic object. */ +using topic_ptr = topic::ptr_t ; + +/** A smart/shared pointer to a const topic object. */ +using const_topic_ptr = topic::const_ptr_t ; + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + +#endif // __mqtt_topic_h + diff --git a/src/mqtt/src/mqtt/types.h b/src/mqtt/src/mqtt/types.h new file mode 100644 index 00000000..9c212343 --- /dev/null +++ b/src/mqtt/src/mqtt/types.h @@ -0,0 +1,184 @@ +///////////////////////////////////////////////////////////////////////////// +/// @file types.h +/// Basic types and type conversions for the Paho MQTT C++ library. +/// @date May 17, 2015 @author Frank Pagliughi +///////////////////////////////////////////////////////////////////////////// + +/******************************************************************************* + * Copyright (c) 2015-2017 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + *******************************************************************************/ + +#ifndef __mqtt_types_h +#define __mqtt_types_h + +#include +#include +#include +#include + +namespace mqtt { + +///////////////////////////////////////////////////////////////////////////// +// Basic data types + +/** A 'byte' is an 8-bit, unsigned int */ +using byte = uint8_t; + +/** An mqtt string is just a std::string */ +using string = std::string; +/** A binary blob of data is, umm, just a string too! */ +using binary = std::string; //std::basic_string; + +/** Smart/shared pointer to a const string */ +using string_ptr = std::shared_ptr; +/** Smart/shared pointer to a const binary blob */ +using binary_ptr = std::shared_ptr; + +///////////////////////////////////////////////////////////////////////////// +// General protocol enumerations + +/** + * The MQTT v5 Reason Codes. + * + * These map to the Paho C MQTTReasonCodes + */ +enum ReasonCode { + SUCCESS = 0, + NORMAL_DISCONNECTION = 0, + GRANTED_QOS_0 = 0, + GRANTED_QOS_1 = 1, + GRANTED_QOS_2 = 2, + DISCONNECT_WITH_WILL_MESSAGE = 4, + NO_MATCHING_SUBSCRIBERS = 16, + NO_SUBSCRIPTION_FOUND = 17, + CONTINUE_AUTHENTICATION = 24, + RE_AUTHENTICATE = 25, + UNSPECIFIED_ERROR = 128, + MALFORMED_PACKET = 129, + PROTOCOL_ERROR = 130, + IMPLEMENTATION_SPECIFIC_ERROR = 131, + UNSUPPORTED_PROTOCOL_VERSION = 132, + CLIENT_IDENTIFIER_NOT_VALID = 133, + BAD_USER_NAME_OR_PASSWORD = 134, + NOT_AUTHORIZED = 135, + SERVER_UNAVAILABLE = 136, + SERVER_BUSY = 137, + BANNED = 138, + SERVER_SHUTTING_DOWN = 139, + BAD_AUTHENTICATION_METHOD = 140, + KEEP_ALIVE_TIMEOUT = 141, + SESSION_TAKEN_OVER = 142, + TOPIC_FILTER_INVALID = 143, + TOPIC_NAME_INVALID = 144, + PACKET_IDENTIFIER_IN_USE = 145, + PACKET_IDENTIFIER_NOT_FOUND = 146, + RECEIVE_MAXIMUM_EXCEEDED = 147, + TOPIC_ALIAS_INVALID = 148, + PACKET_TOO_LARGE = 149, + MESSAGE_RATE_TOO_HIGH = 150, + QUOTA_EXCEEDED = 151, + ADMINISTRATIVE_ACTION = 152, + PAYLOAD_FORMAT_INVALID = 153, + RETAIN_NOT_SUPPORTED = 154, + QOS_NOT_SUPPORTED = 155, + USE_ANOTHER_SERVER = 156, + SERVER_MOVED = 157, + SHARED_SUBSCRIPTIONS_NOT_SUPPORTED = 158, + CONNECTION_RATE_EXCEEDED = 159, + MAXIMUM_CONNECT_TIME = 160, + SUBSCRIPTION_IDENTIFIERS_NOT_SUPPORTED = 161, + WILDCARD_SUBSCRIPTIONS_NOT_SUPPORTED = 162, + MQTTPP_V3_CODE = 255 // This is not a protocol code; used internally by the library +}; + +///////////////////////////////////////////////////////////////////////////// +// Time functions + +/** + * Convert a chrono duration to seconds. + * This casts away precision to get integer seconds. + * @param dur A chrono duration type + * @return The duration as a chrono seconds value + */ +template +std::chrono::seconds to_seconds(const std::chrono::duration& dur) { + return std::chrono::duration_cast(dur); +} + +/** + * Convert a chrono duration to a number of seconds. + * This casts away precision to get integer seconds. + * @param dur A chrono duration type + * @return The duration as a number of seconds + */ +template +long to_seconds_count(const std::chrono::duration& dur) { + return (long) to_seconds(dur).count(); +} + +/** + * Convert a chrono duration to milliseconds. + * This casts away precision to get integer milliseconds. + * @param dur A chrono duration type + * @return The duration as a chrono milliseconds value + */ +template +std::chrono::milliseconds to_milliseconds(const std::chrono::duration& dur) { + return std::chrono::duration_cast(dur); +} + +/** + * Convert a chrono duration to a number of milliseconds. + * This casts away precision to get integer milliseconds. + * @param dur A chrono duration type + * @return The duration as a number of milliseconds + */ +template +long to_milliseconds_count(const std::chrono::duration& dur) { + return (long) to_milliseconds(dur).count(); +} + +///////////////////////////////////////////////////////////////////////////// +// Misc + +/** + * Converts an into to a bool. + * @param n An integer. + * @return @em true if n not equal to zero, @em false otherwise + */ +inline bool to_bool(int n) { return n != 0; } +/** + * Converts the boolean into a C integer true/false value. + * @param b A boolean + * @return Zero if b is false, non-zero if b is true. + */ +inline int to_int(bool b) { return b ? (!0) : 0; } + +/** + * Gets a valid string for the char pointer. + * @param cstr A C-string pointer + * @return A string copy of the C array. If `cstr` is NULL, this returns an + * empty string. + */ +inline string to_string(const char* cstr) { + return cstr ? string(cstr) : string(); +} + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + +#endif // __mqtt_types_h + diff --git a/src/mqtt/src/mqtt/will_options.h b/src/mqtt/src/mqtt/will_options.h new file mode 100644 index 00000000..a685a966 --- /dev/null +++ b/src/mqtt/src/mqtt/will_options.h @@ -0,0 +1,258 @@ +///////////////////////////////////////////////////////////////////////////// +/// @file will_options.h +/// Declaration of MQTT will_options class +/// @date Jul 7, 2016 +/// @author Guilherme M. Ferreira +///////////////////////////////////////////////////////////////////////////// + +/******************************************************************************* + * Copyright (c) 2016 Guilherme M. Ferreira + * Copyright (c) 2016-2019 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Guilherme M. Ferreira - initial implementation and documentation + * Frank Pagliughi - added copy & move operations + *******************************************************************************/ + +#ifndef __mqtt_will_options_h +#define __mqtt_will_options_h + +#include "MQTTAsync.h" +#include "mqtt/types.h" +#include "mqtt/message.h" +#include "mqtt/topic.h" + +namespace mqtt { + +class connect_options; + +///////////////////////////////////////////////////////////////////////////// + +/** + * Holds the set of options that govern the Last Will and Testament feature. + * + * @note + * This wraps struct v1 of the C library's MQTTAsync_willOptions structure. + * It sets the LWT binary payload, via the 'payload' struct field, and + leaves the 'message' field as a nullptr. + */ +class will_options +{ + /** The default QoS for the LWT, if unspecified */ + static constexpr int DFLT_QOS = 0; + /** The defalut retained flag for LWT, if unspecified */ + static constexpr bool DFLT_RETAINED = false; + /** A default C struct to support re-initializing variables */ + static const MQTTAsync_willOptions DFLT_C_STRUCT; + + /** The underlying C LWT options */ + MQTTAsync_willOptions opts_; + + /** LWT message topic **/ + string_ref topic_; + + /** LWT message text */ + binary_ref payload_; + + /** + * The properties for the LWT message. + * Strangely, in the C lib, the will properties are not in the + * willOptions struct, but are rather in the connectOptions. + */ + properties props_; + + /** The connect options has special access */ + friend class connect_options; + friend class connect_options_test; + friend class will_options_test; + + /** + * Gets a pointer to the C-language NUL-terminated strings for the + * struct. + * Some structs, such as this one, require valid pointers at all times, + * while others expect NULL pointers for missing parameters. + * So we always return a pointer to a valid C char array, even when + * empty. + * @param str The C++ string object. + * @return Pointer to a NUL terminated string. This is only valid until + * the next time the string is updated. This is never nullptr. + */ + const char* c_str(const string_ref& sr) { + return sr ? sr.to_string().c_str() : nullptr; + } + +public: + /** Smart/shared pointer to this class. */ + using ptr_t = std::shared_ptr; + /** Smart/shared pointer to a const object of this class. */ + using const_ptr_t = std::shared_ptr; + + /** + * Constructs a new object using the default values. + */ + will_options(); + /** + * Sets the "Last Will and Testament" (LWT) for the connection. + * @param top The LWT message is published to the this topic. + * @param payload The message that is published to the Will Topic. + * @param payload_len The message size in bytes + * @param qos The message Quality of Service. + * @param retained Tell the broker to keep the LWT message after send to + * subscribers. + */ + will_options(string_ref top, const void *payload, size_t payload_len, + int qos=DFLT_QOS, bool retained=DFLT_RETAINED); + /** + * Sets the "Last Will and Testament" (LWT) for the connection. + * @param top The LWT message is published to the this topic. + * @param payload The message that is published to the Will Topic. + * @param payload_len The message size in bytes. + * @param qos The message Quality of Service. + * @param retained Tell the broker to keep the LWT message after send to + * subscribers. + */ + will_options(const topic& top, const void *payload, size_t payload_len, + int qos=DFLT_QOS, bool retained=DFLT_RETAINED); + /** + * Sets the "Last Will and Testament" (LWT) for the connection. + * @param top The LWT message is published to the this topic. + * @param payload The message payload that is published to the Will + * Topic. + * @param qos The message Quality of Service. + * @param retained Tell the broker to keep the LWT message after send to + * subscribers. + */ + will_options(string_ref top, binary_ref payload, + int qos=DFLT_QOS, bool retained=DFLT_RETAINED); + /** + * Sets the "Last Will and Testament" (LWT) for the connection. + * @param top The LWT message is published to the this topic. + * @param payload The message payload that is published to the Will + * Topic, as a string. + * @param qos The message Quality of Service. + * @param retained Tell the broker to keep the LWT message after send to + * subscribers. + */ + will_options(string_ref top, const string& payload, + int qos=DFLT_QOS, bool retained=DFLT_QOS); + /** + * Sets the "Last Will and Testament" (LWT) for the connection. + * @param msg The message that is published to the Will Topic. + */ + will_options(const message& msg); + /** + * Copy constructor for the LWT options. + * @param opt The other options. + */ + will_options(const will_options& opt); + /** + * Move constructor for the LWT options. + * @param opt The other options. + */ + will_options(will_options&& opt); + /** + * Copy assignment for the LWT options. + * @param opt The other options. + */ + will_options& operator=(const will_options& opt); + /** + * Move assignment for the LWT options. + * @param opt The other options. + */ + will_options& operator=(will_options&& opt); + /** + * Gets the LWT message topic name. + * @return The LWT message topic name. + */ + string get_topic() const { return topic_ ? topic_.to_string() : string(); } + /** + * Gets the LWT message payload. + * @return The LWT message payload. + */ + const binary_ref& get_payload() const { return payload_; } + /** + * Gets the LWT message payload as a string. + * @return The LWT message payload as a string. + */ + string get_payload_str() const { return payload_ ? payload_.to_string() : string(); } + /** + * Gets the QoS value for the LWT message. + * @return The QoS value for the LWT message. + */ + int get_qos() const { return opts_.qos; } + /** + * Gets the 'retained' flag for the LWT message. + * @return The 'retained' flag for the LWT message. + */ + bool is_retained() const { return opts_.retained != 0; } + /** + * Gets the LWT message as a message object. + * @return A pointer to a copy of the LWT message. + */ + const_message_ptr get_message() const { + return message::create(topic_, payload_, opts_.qos, to_bool(opts_.retained)); + } + /** + * Sets the LWT message topic name. + * @param top The topic where to sent the message + */ + void set_topic(string_ref top); + /** + * Sets the LWT message text. + * @param msg The LWT message + */ + void set_payload(binary_ref msg); + /** + * Sets the LWT message text. + * @param msg The LWT message + */ + void set_payload(string msg) { set_payload(binary_ref(std::move(msg))); } + /** + * Sets the QoS value. + * @param qos The LWT message QoS + */ + void set_qos(const int qos) { opts_.qos = qos; } + /** + * Sets the retained flag. + * @param retained Tell the broker to keep the LWT message after send to + * subscribers. + */ + void set_retained(bool retained) { opts_.retained = to_int(retained); } + /** + * Gets the connect properties. + * @return A const reference to the properties for the connect. + */ + const properties& get_properties() const { return props_; } + /** + * Sets the properties for the connect. + * @param props The properties to place into the message. + */ + void set_properties(const properties& props) { props_ = props; } + /** + * Moves the properties for the connect. + * @param props The properties to move into the connect object. + */ + void set_properties(properties&& props) { props_ = props; } +}; + +/** Shared pointer to a will options object. */ +using will_options_ptr = will_options::ptr_t; + +/** Shared pointer to a const will options object. */ +using const_will_options_ptr = will_options::const_ptr_t; + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + +#endif // __mqtt_will_options_h + diff --git a/src/mqtt/src/properties.cpp b/src/mqtt/src/properties.cpp new file mode 100644 index 00000000..33d2adb7 --- /dev/null +++ b/src/mqtt/src/properties.cpp @@ -0,0 +1,183 @@ +// properties.cpp + +/******************************************************************************* + * Copyright (c) 2019 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + *******************************************************************************/ + +#include "mqtt/properties.h" + +namespace mqtt { + +///////////////////////////////////////////////////////////////////////////// + +property::property(code c, int32_t val) +{ + prop_.identifier = ::MQTTPropertyCodes(c); + + switch (::MQTTProperty_getType(prop_.identifier)) { + case MQTTPROPERTY_TYPE_BYTE: + prop_.value.byte = uint8_t(val); + break; + case MQTTPROPERTY_TYPE_TWO_BYTE_INTEGER: + prop_.value.integer2 = int16_t(val); + break; + case MQTTPROPERTY_TYPE_FOUR_BYTE_INTEGER: + case MQTTPROPERTY_TYPE_VARIABLE_BYTE_INTEGER: + prop_.value.integer4 = val; + break; + default: + // TODO: Throw an exception + break; + } +} + +property::property(code c, string_ref val) +{ + prop_.identifier = ::MQTTPropertyCodes(c); + + size_t n = val.size(); + prop_.value.data.len = int(n); + prop_.value.data.data = (char*) malloc(n); + std::memcpy(prop_.value.data.data, val.data(), n); +} + +property::property(code c, string_ref name, string_ref val) +{ + prop_.identifier = MQTTPropertyCodes(c); + + size_t n = name.size(); + prop_.value.data.len = int(n); + prop_.value.data.data = (char*) malloc(n); + std::memcpy(prop_.value.data.data, name.data(), n); + + n = val.size(); + prop_.value.value.len = int(n); + prop_.value.value.data = (char*) malloc(n); + std::memcpy(prop_.value.value.data, val.data(), n); +} + +property::property(const MQTTProperty& cprop) +{ + copy(cprop); +} + +property::property(MQTTProperty&& cprop) + :prop_(cprop) +{ + memset(&cprop, 0, sizeof(MQTTProperty)); +} + + +property::property(const property& other) +{ + copy(other.prop_); +} + +property::property(property&& other) +{ + std::memcpy(&prop_, &other.prop_, sizeof(MQTTProperty)); + memset(&other.prop_, 0, sizeof(MQTTProperty)); +} + +property::~property() +{ + switch (::MQTTProperty_getType(prop_.identifier)) { + case MQTTPROPERTY_TYPE_UTF_8_STRING_PAIR: + free(prop_.value.value.data); + // Fall-through + + case MQTTPROPERTY_TYPE_BINARY_DATA: + case MQTTPROPERTY_TYPE_UTF_8_ENCODED_STRING: + free(prop_.value.data.data); + break; + + default: + // Nothing necessary + break; + } +} + +void property::copy(const MQTTProperty& cprop) +{ + size_t n; + + std::memcpy(&prop_, &cprop, sizeof(MQTTProperty)); + + switch (::MQTTProperty_getType(prop_.identifier)) { + case MQTTPROPERTY_TYPE_UTF_8_STRING_PAIR: + n = prop_.value.value.len; + prop_.value.value.data = (char*) malloc(n); + memcpy(prop_.value.value.data, cprop.value.value.data, n); + // Fall-through + + case MQTTPROPERTY_TYPE_BINARY_DATA: + case MQTTPROPERTY_TYPE_UTF_8_ENCODED_STRING: + n = prop_.value.data.len; + prop_.value.data.data = (char*) malloc(n); + memcpy(prop_.value.data.data, cprop.value.data.data, n); + break; + + default: + // Nothing necessary + break; + } +} + +property& property::operator=(const property& rhs) +{ + if (&rhs != this) + copy(rhs.prop_); + + return *this; +} + +property& property::operator=(property&& rhs) +{ + if (&rhs != this) { + std::memcpy(&prop_, &rhs.prop_, sizeof(MQTTProperty)); + memset(&rhs.prop_, 0, sizeof(MQTTProperty)); + } + return *this; +} + +///////////////////////////////////////////////////////////////////////////// + +properties::properties(std::initializer_list props) +{ + std::memset(&props_, 0, sizeof(properties)); + for (const auto& prop : props) + ::MQTTProperties_add(&props_, &prop.c_struct()); +} + +void properties::clear() +{ + ::MQTTProperties_free(&props_); + memset(&props_, 0, sizeof(MQTTProperties)); +} + +property properties::get(property::code propid, size_t idx /*=0*/) +{ + MQTTProperty* prop = MQTTProperties_getPropertyAt(&props_, + MQTTPropertyCodes(propid), int(idx)); + if (!prop) + throw bad_cast(); + + return property(*prop); +} + +///////////////////////////////////////////////////////////////////////////// +// end namespace 'mqtt' +} + diff --git a/src/mqtt/src/response_options.cpp b/src/mqtt/src/response_options.cpp new file mode 100644 index 00000000..443de927 --- /dev/null +++ b/src/mqtt/src/response_options.cpp @@ -0,0 +1,91 @@ +// response_options.cpp + +/******************************************************************************* + * Copyright (c) 2019 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + *******************************************************************************/ + +#include "mqtt/response_options.h" + +namespace mqtt { + +///////////////////////////////////////////////////////////////////////////// + +response_options::response_options(int mqttVersion /*=MQTTVERSION_DEFAULT*/) + : opts_(MQTTAsync_responseOptions_initializer) +{ + if (mqttVersion < MQTTVERSION_5) { + opts_.onSuccess = &token::on_success; + opts_.onFailure = &token::on_failure; + } + else { + opts_.onSuccess5 = &token::on_success5; + opts_.onFailure5 = &token::on_failure5; + } +} + +response_options::response_options(const token_ptr& tok, + int mqttVersion /*=MQTTVERSION_DEFAULT*/) + : response_options(mqttVersion) +{ + set_token(tok); +} + +void response_options::set_token(const token_ptr& tok) +{ + tok_ = tok; + opts_.context = tok.get(); +} + +void response_options::set_subscribe_options(const subscribe_options& opts) +{ + opts_.subscribeOptions = opts.opts_; +} + +void response_options::set_subscribe_options(const std::vector& opts) +{ + subOpts_.clear(); + for (const auto& opt : opts) + subOpts_.push_back(opt.opts_); + + opts_.subscribeOptionsCount = int(opts.size()); + opts_.subscribeOptionsList = const_cast(subOpts_.data()); +} + +///////////////////////////////////////////////////////////////////////////// + +delivery_response_options::delivery_response_options(int mqttVersion /*=MQTTVERSION_DEFAULT*/) + : opts_(MQTTAsync_responseOptions_initializer) +{ + if (mqttVersion < MQTTVERSION_5) { + opts_.onSuccess = &delivery_token::on_success; + opts_.onFailure = &delivery_token::on_failure; + } + else { + opts_.onSuccess5 = &delivery_token::on_success5; + opts_.onFailure5 = &delivery_token::on_failure5; + } +} + +delivery_response_options::delivery_response_options(const delivery_token_ptr& tok, + int mqttVersion /*=MQTTVERSION_DEFAULT*/) + : delivery_response_options(mqttVersion) +{ + set_token(tok); +} + +///////////////////////////////////////////////////////////////////////////// +// end namespace 'mqtt' +} + diff --git a/src/mqtt/src/ssl_options.cpp b/src/mqtt/src/ssl_options.cpp new file mode 100644 index 00000000..aaaff781 --- /dev/null +++ b/src/mqtt/src/ssl_options.cpp @@ -0,0 +1,146 @@ +/******************************************************************************* + * Copyright (c) 2016 Guilherme Ferreira + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Guilherme Ferreira - initial implementation and documentation + * Frank Pagliughi - added copy & move operations + *******************************************************************************/ + +#include "mqtt/ssl_options.h" +#include +#include + +namespace mqtt { + +///////////////////////////////////////////////////////////////////////////// + +const MQTTAsync_SSLOptions ssl_options::DFLT_C_STRUCT = MQTTAsync_SSLOptions_initializer; + +ssl_options::ssl_options() : opts_(DFLT_C_STRUCT) +{ +} + +ssl_options::ssl_options(const string& trustStore, const string& keyStore, + const string& privateKey, const string& privateKeyPassword, + const string& enabledCipherSuites, bool enableServerCertAuth) + : opts_(DFLT_C_STRUCT), trustStore_(trustStore), keyStore_(keyStore), + privateKey_(privateKey), privateKeyPassword_(privateKeyPassword), + enabledCipherSuites_(enabledCipherSuites) +{ + update_c_struct(); + opts_.enableServerCertAuth = enableServerCertAuth; +} + +ssl_options::ssl_options(const ssl_options& opt) + : opts_(opt.opts_), trustStore_(opt.trustStore_), keyStore_(opt.keyStore_), + privateKey_(opt.privateKey_), privateKeyPassword_(opt.privateKeyPassword_), + enabledCipherSuites_(opt.enabledCipherSuites_) +{ + update_c_struct(); +} + +ssl_options::ssl_options(ssl_options&& opt) + : opts_(opt.opts_), trustStore_(std::move(opt.trustStore_)), + keyStore_(std::move(opt.keyStore_)), privateKey_(std::move(opt.privateKey_)), + privateKeyPassword_(std::move(opt.privateKeyPassword_)), + enabledCipherSuites_(std::move(opt.enabledCipherSuites_)) +{ + update_c_struct(); +} + +void ssl_options::update_c_struct() +{ + opts_.trustStore = c_str(trustStore_); + opts_.keyStore = c_str(keyStore_); + opts_.privateKey = c_str(privateKey_); + opts_.privateKeyPassword = c_str(privateKeyPassword_); + opts_.enabledCipherSuites = c_str(enabledCipherSuites_); +} + +ssl_options& ssl_options::operator=(const ssl_options& rhs) +{ + if (&rhs == this) + return *this; + + opts_ = rhs.opts_; + + trustStore_ = rhs.trustStore_; + keyStore_ = rhs.keyStore_; + privateKey_ = rhs.privateKey_; + privateKeyPassword_ = rhs.privateKeyPassword_; + enabledCipherSuites_ = rhs.enabledCipherSuites_; + + update_c_struct(); + return *this; +} + +ssl_options& ssl_options::operator=(ssl_options&& rhs) +{ + if (&rhs == this) + return *this; + + opts_ = rhs.opts_; + + trustStore_ = std::move(rhs.trustStore_); + keyStore_ = std::move(rhs.keyStore_); + privateKey_ = std::move(rhs.privateKey_); + privateKeyPassword_ = std::move(rhs.privateKeyPassword_); + enabledCipherSuites_ = std::move(rhs.enabledCipherSuites_); + + update_c_struct(); + return *this; +} + +void ssl_options::set_trust_store(const string& trustStore) +{ + trustStore_ = trustStore; + opts_.trustStore = c_str(trustStore_); +} + +void ssl_options::set_key_store(const string& keyStore) +{ + keyStore_ = keyStore; + opts_.keyStore = c_str(keyStore_); +} + +void ssl_options::set_private_key(const string& privateKey) +{ + privateKey_ = privateKey; + opts_.privateKey = c_str(privateKey_); +} + +void ssl_options::set_private_key_password(const string& privateKeyPassword) +{ + privateKeyPassword_ = privateKeyPassword; + opts_.privateKeyPassword = c_str(privateKeyPassword_); +} + +void ssl_options::set_enabled_cipher_suites(const string& enabledCipherSuites) +{ + enabledCipherSuites_ = enabledCipherSuites; + opts_.enabledCipherSuites = c_str(enabledCipherSuites_); +} + +void ssl_options::set_enable_server_cert_auth(bool enableServerCertAuth) +{ + opts_.enableServerCertAuth = to_int(enableServerCertAuth); +} + +void ssl_options::ca_path(const string& path) +{ + caPath_ = path; + opts_.CApath = c_str(caPath_); +} + +///////////////////////////////////////////////////////////////////////////// +} // end namespace mqtt + diff --git a/src/mqtt/src/string_collection.cpp b/src/mqtt/src/string_collection.cpp new file mode 100644 index 00000000..f9c9a9ec --- /dev/null +++ b/src/mqtt/src/string_collection.cpp @@ -0,0 +1,103 @@ +// string_collection.cpp + +/******************************************************************************* + * Copyright (c) 2017 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + *******************************************************************************/ + + +#include "mqtt/string_collection.h" + +namespace mqtt { + +///////////////////////////////////////////////////////////////////////////// + +string_collection::string_collection(const string& str) : coll_{ str } +{ + update_c_arr(); +} + +string_collection::string_collection(string&& str) : coll_{ std::move(str) } +{ + update_c_arr(); +} + +string_collection::string_collection(const collection_type& vec) : coll_{ vec } +{ + update_c_arr(); +} + +string_collection::string_collection(collection_type&& vec) : coll_{ std::move(vec) } +{ + update_c_arr(); +} + +string_collection::string_collection(const string_collection& coll) : coll_ { coll.coll_ } +{ + update_c_arr(); +} + +string_collection::string_collection(std::initializer_list sl) +{ + for (const auto& s : sl) + coll_.push_back(s); + update_c_arr(); +} + +string_collection::string_collection(std::initializer_list sl) +{ + for (const auto& s : sl) + coll_.push_back(string(s)); + update_c_arr(); +} + +void string_collection::update_c_arr() +{ + cArr_.clear(); + cArr_.reserve(coll_.size()); + for (const auto& s : coll_) + cArr_.push_back(s.c_str()); +} + +string_collection& string_collection::operator=(const string_collection& coll) +{ + coll_ = coll.coll_; + update_c_arr(); + return *this; +} + +void string_collection::push_back(const string& str) +{ + coll_.push_back(str); + update_c_arr(); +} + +void string_collection::push_back(string&& str) +{ + coll_.push_back(str); + update_c_arr(); +} + +void string_collection::clear() +{ + coll_.clear(); + cArr_.clear(); +} + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + + + diff --git a/src/mqtt/src/subscribe_options.cpp b/src/mqtt/src/subscribe_options.cpp new file mode 100644 index 00000000..efa01f81 --- /dev/null +++ b/src/mqtt/src/subscribe_options.cpp @@ -0,0 +1,36 @@ +// subscribe_options.cpp + +/******************************************************************************* + * Copyright (c) 2019 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + *******************************************************************************/ + +#include "mqtt/subscribe_options.h" + +namespace mqtt { + +///////////////////////////////////////////////////////////////////////////// + +#if __cplusplus < 201703L + constexpr bool subscribe_options::SUBSCRIBE_NO_LOCAL; + constexpr bool subscribe_options::SUBSCRIBE_LOCAL; + + constexpr bool subscribe_options::NO_RETAIN_AS_PUBLISHED; + constexpr bool subscribe_options::RETAIN_AS_PUBLISHED; +#endif + +///////////////////////////////////////////////////////////////////////////// +// end namespace 'mqtt' +} + diff --git a/src/mqtt/src/token.cpp b/src/mqtt/src/token.cpp new file mode 100644 index 00000000..3853cf2a --- /dev/null +++ b/src/mqtt/src/token.cpp @@ -0,0 +1,297 @@ +// token.cpp + +/******************************************************************************* + * Copyright (c) 2013-2019 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + * Frank Pagliughi - MQTT v5 support + *******************************************************************************/ + +#include "mqtt/token.h" +#include "mqtt/async_client.h" +#include + +#include + +namespace mqtt { + +// -------------------------------------------------------------------------- +// Constructors + +token::token(Type typ, iasync_client& cli, const_string_collection_ptr topics) + : type_(typ), cli_(&cli), rc_(0), reasonCode_(ReasonCode::SUCCESS), + msgId_(MQTTAsync_token(0)), topics_(topics), + userContext_(nullptr), listener_(nullptr), nExpected_(0), + complete_(false) +{ +} + +token::token(Type typ, iasync_client& cli, const_string_collection_ptr topics, + void* userContext, iaction_listener& cb) + : type_(typ), cli_(&cli), rc_(0), reasonCode_(ReasonCode::SUCCESS), + msgId_(MQTTAsync_token(0)), topics_(topics), + userContext_(userContext), listener_(&cb), nExpected_(0), + complete_(false) +{ +} + +token::token(Type typ, iasync_client& cli, MQTTAsync_token tok) + : type_(typ), cli_(&cli), rc_(0), reasonCode_(ReasonCode::SUCCESS), + msgId_(tok), userContext_(nullptr), + listener_(nullptr), nExpected_(0), complete_(false) +{ +} + +// -------------------------------------------------------------------------- +// Class static callbacks. +// These are the callbacks directly from the C library. +// The 'context' is a raw pointer to the token object. + +void token::on_success(void* context, MQTTAsync_successData* rsp) +{ + if (context) + static_cast(context)->on_success(rsp); +} + +void token::on_success5(void* context, MQTTAsync_successData5* rsp) +{ + if (context) + static_cast(context)->on_success5(rsp); +} + +void token::on_failure(void* context, MQTTAsync_failureData* rsp) +{ + if (context) + static_cast(context)->on_failure(rsp); +} + +void token::on_failure5(void* context, MQTTAsync_failureData5* rsp) +{ + if (context) + static_cast(context)->on_failure5(rsp); +} + +// -------------------------------------------------------------------------- +// Object callbacks + +// +// The success callback for MQTT v3 connections +// +void token::on_success(MQTTAsync_successData* rsp) +{ + unique_lock g(lock_); + iaction_listener* listener = listener_; + + if (rsp) { + msgId_ = rsp->token; + + switch (type_) { + case Type::CONNECT: + connRsp_.reset(new connect_response(rsp)); + break; + + case Type::SUBSCRIBE: + subRsp_.reset(new subscribe_response(nExpected_, rsp)); + break; + + case Type::UNSUBSCRIBE: + unsubRsp_.reset(new unsubscribe_response(rsp)); + break; + + default: + // The others don't have responses + break; + } + } + + rc_ = MQTTASYNC_SUCCESS; + complete_ = true; + g.unlock(); + + // Note: callback always completes before the object is signaled. + if (listener) + listener->on_success(*this); + cond_.notify_all(); + + cli_->remove_token(this); +} + +// +// The success callback for MQTT v5 connections +// +void token::on_success5(MQTTAsync_successData5* rsp) +{ + unique_lock g(lock_); + iaction_listener* listener = listener_; + if (rsp) { + msgId_ = rsp->token; + reasonCode_ = ReasonCode(rsp->reasonCode); + + switch (type_) { + case Type::CONNECT: + connRsp_.reset(new connect_response(rsp)); + break; + + case Type::SUBSCRIBE: + subRsp_.reset(new subscribe_response(rsp)); + break; + + case Type::UNSUBSCRIBE: + unsubRsp_.reset(new unsubscribe_response(rsp)); + break; + + default: + // The others don't have responses + break; + } + } + rc_ = MQTTASYNC_SUCCESS; + complete_ = true; + g.unlock(); + + // Note: callback always completes before the object is signaled. + if (listener) + listener->on_success(*this); + cond_.notify_all(); + + cli_->remove_token(this); +} + +// +// The failure callback for MQTT v3 connections +// +void token::on_failure(MQTTAsync_failureData* rsp) +{ + unique_lock g(lock_); + iaction_listener* listener = listener_; + if (rsp) { + msgId_ = rsp->token; + rc_ = rsp->code; + + // HACK: For backward compatability with v3 connections + reasonCode_ = ReasonCode(MQTTPP_V3_CODE); + + if (rsp->message) + errMsg_ = string(rsp->message); + } + else { + rc_ = -1; + } + complete_ = true; + g.unlock(); + + // Note: callback always completes before the obect is signaled. + if (listener) + listener->on_failure(*this); + cond_.notify_all(); + + cli_->remove_token(this); +} + +// +// The failure callback for MQTT v5 connections +// +void token::on_failure5(MQTTAsync_failureData5* rsp) +{ + unique_lock g(lock_); + iaction_listener* listener = listener_; + if (rsp) { + msgId_ = rsp->token; + reasonCode_ = ReasonCode(rsp->reasonCode); + //props_ = properties(rsp->properties); + rc_ = rsp->code; + if (rsp->message) + errMsg_ = string(rsp->message); + } + else { + rc_ = -1; + } + complete_ = true; + g.unlock(); + + // Note: callback always completes before the obect is signaled. + if (listener) + listener->on_failure(*this); + cond_.notify_all(); + + cli_->remove_token(this); +} + +// -------------------------------------------------------------------------- +// API + +void token::reset() +{ + guard g(lock_); + complete_ = false; + rc_ = MQTTASYNC_SUCCESS; + reasonCode_ = ReasonCode::SUCCESS; + errMsg_.clear(); +} + +void token::wait() +{ + unique_lock g(lock_); + cond_.wait(g, [this]{return complete_;}); + check_ret(); +} + +connect_response token::get_connect_response() const +{ + if (type_ != Type::CONNECT) + throw bad_cast(); + + unique_lock g(lock_); + cond_.wait(g, [this]{return complete_;}); + check_ret(); + + if (!connRsp_) + throw missing_response("connect"); + + return *connRsp_; +} + +subscribe_response token::get_subscribe_response() const +{ + if (type_ != Type::SUBSCRIBE) + throw bad_cast(); + + unique_lock g(lock_); + cond_.wait(g, [this]{return complete_;}); + check_ret(); + + if (!subRsp_) + throw missing_response("subscribe"); + + return *subRsp_; +} + +unsubscribe_response token::get_unsubscribe_response() const +{ + if (type_ != Type::UNSUBSCRIBE) + throw bad_cast(); + + unique_lock g(lock_); + cond_.wait(g, [this]{return complete_;}); + check_ret(); + + if (!unsubRsp_) + throw missing_response("unsubscribe"); + + return *unsubRsp_; +} + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + diff --git a/src/mqtt/src/topic.cpp b/src/mqtt/src/topic.cpp new file mode 100644 index 00000000..8e486a9e --- /dev/null +++ b/src/mqtt/src/topic.cpp @@ -0,0 +1,57 @@ +// topic.cpp + +/******************************************************************************* + * Copyright (c) 2013-2016 Frank Pagliughi + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Frank Pagliughi - initial implementation and documentation + *******************************************************************************/ + +#include "mqtt/topic.h" +#include "mqtt/async_client.h" + +namespace mqtt { + +///////////////////////////////////////////////////////////////////////////// + +delivery_token_ptr topic::publish(const void* payload, size_t n) +{ + return cli_.publish(name_, payload, n, qos_, retained_); +} + +delivery_token_ptr topic::publish(const void* payload, size_t n, + int qos, bool retained) +{ + return cli_.publish(name_, payload, n, qos, retained); +} + +delivery_token_ptr topic::publish(binary_ref payload) +{ + return cli_.publish(name_, std::move(payload), qos_, retained_); +} + +delivery_token_ptr topic::publish(binary_ref payload, int qos, bool retained) +{ + return cli_.publish(name_, std::move(payload), qos, retained); +} + +token_ptr topic::subscribe(const subscribe_options& opts) +{ + return cli_.subscribe(name_, qos_, opts); +} + +///////////////////////////////////////////////////////////////////////////// +// end namespace mqtt +} + + + diff --git a/src/mqtt/src/will_options.cpp b/src/mqtt/src/will_options.cpp new file mode 100644 index 00000000..457d0603 --- /dev/null +++ b/src/mqtt/src/will_options.cpp @@ -0,0 +1,133 @@ +/******************************************************************************* + * Copyright (c) 2016 Guilherme M. Ferreira + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * and Eclipse Distribution License v1.0 which accompany this distribution. + * + * The Eclipse Public License is available at + * http://www.eclipse.org/legal/epl-v10.html + * and the Eclipse Distribution License is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * Contributors: + * Guilherme M. Ferreira - initial implementation and documentation + * Frank Pagliughi - added copy & move operations + *******************************************************************************/ + +#include "mqtt/will_options.h" +#include +#include + +namespace mqtt { + +#if __cplusplus < 201703L + constexpr int will_options::DFLT_QOS; + constexpr bool will_options::DFLT_RETAINED; +#endif + +const MQTTAsync_willOptions will_options::DFLT_C_STRUCT = MQTTAsync_willOptions_initializer; + +///////////////////////////////////////////////////////////////////////////// + +will_options::will_options() : opts_(DFLT_C_STRUCT) +{ + set_topic(string()); +// set_payload(binary_ref()); +} + +will_options::will_options(string_ref top, + const void *payload, size_t payloadlen, + int qos, bool retained) + : opts_(DFLT_C_STRUCT) +{ + opts_.qos = qos; + opts_.retained = retained; + set_topic(std::move(top)); + set_payload(binary_ref(static_cast(payload), payloadlen)); +} + +will_options::will_options(const topic& top, + const void *payload, size_t payloadlen, + int qos, bool retained) + : will_options(top.get_name(), payload, payloadlen, qos, retained) +{ +} + + +will_options::will_options(string_ref top, binary_ref payload, + int qos, bool retained) + : opts_(DFLT_C_STRUCT) +{ + opts_.qos = qos; + opts_.retained = retained; + set_topic(std::move(top)); + set_payload(std::move(payload)); +} + +will_options::will_options(string_ref top, const string& payload, + int qos, bool retained) + : opts_(DFLT_C_STRUCT) +{ + opts_.qos = qos; + opts_.retained = retained; + set_topic(std::move(top)); + set_payload(payload); +} + +will_options::will_options(const message& msg) + : will_options(msg.get_topic(), msg.get_payload(), msg.get_qos(), msg.is_retained()) +{ +} + +will_options::will_options(const will_options& opt) : opts_(opt.opts_) +{ + set_topic(opt.topic_); + set_payload(opt.payload_); +} + +will_options::will_options(will_options&& other) : opts_(other.opts_) +{ + set_topic(std::move(other.topic_)); + set_payload(std::move(other.payload_)); +} + +will_options& will_options::operator=(const will_options& rhs) +{ + if (&rhs != this) { + opts_ = rhs.opts_; + set_topic(rhs.topic_); + set_payload(rhs.payload_); + } + return *this; +} + +will_options& will_options::operator=(will_options&& rhs) +{ + if (&rhs != this) { + opts_ = rhs.opts_; + set_topic(std::move(rhs.topic_)); + set_payload(std::move(rhs.payload_)); + } + return *this; +} + +void will_options::set_topic(string_ref top) +{ + topic_ = top ? std::move(top) : string_ref(string()); + opts_.topicName = topic_.c_str(); +} + +void will_options::set_payload(binary_ref msg) +{ + // The C struct payload must not be nullptr for will options + payload_ = msg ? std::move(msg) : binary_ref(binary()); + + opts_.payload.len = (int) payload_.size(); + opts_.payload.data = payload_.data(); +} + +///////////////////////////////////////////////////////////////////////////// + +} // end namespace mqtt + From 24ee1d9bb896effcb1e0d3e89a20112652190e3c Mon Sep 17 00:00:00 2001 From: OuYun Date: Wed, 4 Mar 2020 18:13:47 +0800 Subject: [PATCH 004/181] update cmake file to integrate mqtt to bigbang --- src/bigbang/CMakeLists.txt | 1 + src/mqtt/CMakeLists.txt | 41 +++++++++- src/mqtt/src/CMakeLists.txt | 146 ------------------------------------ 3 files changed, 38 insertions(+), 150 deletions(-) delete mode 100644 src/mqtt/src/CMakeLists.txt diff --git a/src/bigbang/CMakeLists.txt b/src/bigbang/CMakeLists.txt index 606a3fac..bc502cdc 100755 --- a/src/bigbang/CMakeLists.txt +++ b/src/bigbang/CMakeLists.txt @@ -104,6 +104,7 @@ target_link_libraries(bigbang # delegate jsonrpc common + mqtt ) # create symbol link diff --git a/src/mqtt/CMakeLists.txt b/src/mqtt/CMakeLists.txt index 7d115f5d..9ba79461 100644 --- a/src/mqtt/CMakeLists.txt +++ b/src/mqtt/CMakeLists.txt @@ -75,17 +75,48 @@ endif() # For the paho_mqtt_c module list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) -add_subdirectory(src) # --- Default library --- if(PAHO_BUILD_SHARED) - set(PAHO_CPP_LIB paho-mqttpp3) + set(PAHO_CPP_LIB mqtt-shared) message(STATUS "shared......" ${PAHO_BUILD_SHARED}) #set(PAHO_CPP_LIB paho-mqttpp3-static) else() - set(PAHO_CPP_LIB paho-mqttpp3-static) message(STATUS "static......") + set(PAHO_CPP_LIB mqtt) + ## --- Library dependencies --- + find_package(PahoMqttC REQUIRED) + set (THREADS_PREFER_PTHREAD_FLAG ON) + find_package(Threads REQUIRED) + set(sources + src/async_client.cpp + src/client.cpp + src/disconnect_options.cpp + src/iclient_persistence.cpp + src/message.cpp + src/properties.cpp + src/response_options.cpp + src/ssl_options.cpp + src/string_collection.cpp + src/subscribe_options.cpp + src/token.cpp + src/topic.cpp + src/connect_options.cpp + src/will_options.cpp + ) + add_library(mqtt ${sources}) + include_directories(./src/mqtt ${PAHO_MQTT_C_INCLUDE_DIRS}) + target_link_libraries(mqtt + PRIVATE ${LIBS_SYSTEM} + PUBLIC PahoMqttC::PahoMqttC Threads::Threads + ${PAHO_MQTT_C_LIBRARIES} + ) + + target_include_directories(mqtt PUBLIC + $ + $ + ) endif() ## --- Packaging settings --- @@ -97,5 +128,7 @@ elseif(UNIX) endif() include(CPack) -add_subdirectory(cmake) +#add_subdirectory(cmake) + + diff --git a/src/mqtt/src/CMakeLists.txt b/src/mqtt/src/CMakeLists.txt deleted file mode 100644 index 850bc86e..00000000 --- a/src/mqtt/src/CMakeLists.txt +++ /dev/null @@ -1,146 +0,0 @@ -# CMakeLists.txt -# -# CMake file for the Paho C++ core library. -# -# This is part of the Paho MQTT C++ client library. -# - -#******************************************************************************* -# Copyright (c) 2016-2017, Guilherme Maciel Ferreira -# Copyright (c) 2017-2018, Frank Pagliughi -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Eclipse Public License v1.0 -# and Eclipse Distribution License v1.0 which accompany this distribution. -# -# The Eclipse Public License is available at -# http://www.eclipse.org/legal/epl-v10.html -# and the Eclipse Distribution License is available at -# http://www.eclipse.org/org/documents/edl-v10.php. -# -# Contributors: -# Guilherme Maciel Ferreira - initial version -# Frank Pagliughi - made the shared library optional -#*******************************************************************************/ - -find_package(PahoMqttC REQUIRED) - -# --- The headers --- - -#add_subdirectory(mqtt) - -## --- Library dependencies --- - -set (THREADS_PREFER_PTHREAD_FLAG ON) -find_package(Threads REQUIRED) - -#set(LIBS_SYSTEM Threads::Threads) -#if(WIN32) -# string(APPEND LIBS_SYSTEM " ws2_32") -#endif() - -## --- Use object library to optimize compilation --- - -add_library(paho-cpp-objs OBJECT - async_client.cpp - client.cpp - disconnect_options.cpp - iclient_persistence.cpp - message.cpp - properties.cpp - response_options.cpp - ssl_options.cpp - string_collection.cpp - subscribe_options.cpp - token.cpp - topic.cpp - connect_options.cpp - will_options.cpp -) - -## install the shared library -#install(TARGETS paho-cpp-objs EXPORT PahoMqttCpp -# OBJECTS DESTINATION ${CMAKE_INSTALL_LIBDIR} -#) - -# Object libraries can't use target_link_libraries in order to take advantage -# of transitive usage requirements until CMake 3.12. This is a workaround: -#target_include_directories(OBJS PRIVATE ${PAHO_MQTT_C_INCLUDE_DIRS}) - -target_include_directories(paho-cpp-objs - PUBLIC - $ - $ - PRIVATE - ${PAHO_MQTT_C_INCLUDE_DIRS} - src -) - - -## --- Build the shared library, if requested --- -message(STATUS "???" " " "PAHO_BUILD_STATIC:" ${PAHO_BUILD_STATIC} " " "PAHO_BUILD_SHARED:" ${PAHO_BUILD_SHARED}) - -if(PAHO_BUILD_SHARED) - ## create the shared library - add_library(paho-mqttpp3 SHARED $) - - ## add dependencies to the shared library - target_link_libraries(paho-mqttpp3 - PRIVATE ${LIBS_SYSTEM} - PUBLIC PahoMqttC::PahoMqttC Threads::Threads) - - # It would be nice to exort the include paths from the obj lib, but we - # get an export error. Perhaps in a future version? - # - # CMake Error: install(EXPORT "PahoMqttCpp" ...) includes target "paho-mqttpp3" - # which requires target "paho-cpp-objs" that is not in the export set. - - #target_include_directories(paho-mqttpp3 PUBLIC - # $ - #) - - target_include_directories(paho-mqttpp3 PUBLIC - $ - $ - ) - - ## set the shared library soname - set_target_properties(paho-mqttpp3 PROPERTIES - VERSION ${PROJECT_VERSION} - SOVERSION ${PROJECT_VERSION_MAJOR}) - - ## install the shared library - install(TARGETS paho-mqttpp3 EXPORT PahoMqttCpp - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) -endif() - -## --- Build static version of the library, if requested --- - -if(PAHO_BUILD_STATIC) - ## create the static library - add_library(paho-mqttpp3-static STATIC $) - - ## add dependencies to the shared library - target_link_libraries(paho-mqttpp3-static - PRIVATE ${LIBS_SYSTEM} - PUBLIC PahoMqttC::PahoMqttC Threads::Threads) - - target_include_directories(paho-mqttpp3-static PUBLIC - $ - $ - ) - - ## install the static library - install(TARGETS paho-mqttpp3-static EXPORT PahoMqttCpp - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) - - ## Let the archive use the same name as the shared lib on *nix systems - if(UNIX) - set_target_properties(paho-mqttpp3-static PROPERTIES OUTPUT_NAME paho-mqttpp3) - endif() -endif() - - From a467775b07b42262c33a9074090bdac31ab39425 Mon Sep 17 00:00:00 2001 From: OuYun Date: Thu, 5 Mar 2020 18:24:03 +0800 Subject: [PATCH 005/181] make mqtt functions in bigbangcore available --- src/bigbang/CMakeLists.txt | 5 +- src/bigbang/cmqcluster.cpp | 11 -- src/bigbang/cmqcluster.h | 17 --- src/bigbang/core.cpp | 2 +- src/bigbang/entry.cpp | 9 ++ src/bigbang/mode/mode.h | 1 + src/bigbang/mode/module_type.h | 1 + src/bigbang/mqcluster.cpp | 228 +++++++++++++++++++++++++++++++++ src/bigbang/mqcluster.h | 63 +++++++++ src/bigbang/mqevent.h | 46 +++++++ src/xengine/util.cpp | 10 +- 11 files changed, 357 insertions(+), 36 deletions(-) delete mode 100644 src/bigbang/cmqcluster.cpp delete mode 100644 src/bigbang/cmqcluster.h create mode 100644 src/bigbang/mqcluster.cpp create mode 100644 src/bigbang/mqcluster.h create mode 100644 src/bigbang/mqevent.h diff --git a/src/bigbang/CMakeLists.txt b/src/bigbang/CMakeLists.txt index bc502cdc..8759627b 100755 --- a/src/bigbang/CMakeLists.txt +++ b/src/bigbang/CMakeLists.txt @@ -47,7 +47,7 @@ set(src wallet.cpp wallet.h blockchain.cpp blockchain.h forkmanager.cpp forkmanager.h - cmqcluster.cpp cmqcluster.h + mqcluster.cpp mqcluster.h datastat.cpp datastat.h checkrepair.cpp checkrepair.h event.h @@ -56,6 +56,7 @@ set(src param.h storage.h version.h + mqevent.h ) set(mode_src mode/basic_config.cpp mode/basic_config.h @@ -80,7 +81,7 @@ set(sources add_executable(bigbang ${sources}) -include_directories(../xengine ../crypto ../common ../storage ../network ../delegate ../mpvss ../jsonrpc ./) +include_directories(../xengine ../crypto ../common ../storage ../network ../delegate ../mpvss ../jsonrpc ../mqtt/src/mqtt ./) include_directories(${CMAKE_CURRENT_BINARY_DIR}) include_directories(${CMAKE_BINARY_DIR}/src/jsonrpc) include_directories(${Protobuf_INCLUDE_DIRS}) diff --git a/src/bigbang/cmqcluster.cpp b/src/bigbang/cmqcluster.cpp deleted file mode 100644 index b80a4894..00000000 --- a/src/bigbang/cmqcluster.cpp +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2019-2020 The Bigbang developers -// Distributed under the MIT/X11 software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include "cmqcluster.h" - -namespace bigbang -{ - - -} // namespace bigbang diff --git a/src/bigbang/cmqcluster.h b/src/bigbang/cmqcluster.h deleted file mode 100644 index d1bb0d6a..00000000 --- a/src/bigbang/cmqcluster.h +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) 2019-2020 The Bigbang developers -// Distributed under the MIT/X11 software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BIGBANG_CMQCLUSTER_H -#define BIGBANG_CMQCLUSTER_H - -namespace bigbang -{ - -class CMQCluster -{ -}; - -} // namespace bigbang - -#endif //BIGBANG_CMQCLUSTER_H diff --git a/src/bigbang/core.cpp b/src/bigbang/core.cpp index 1cc3818c..c5e3d282 100644 --- a/src/bigbang/core.cpp +++ b/src/bigbang/core.cpp @@ -19,7 +19,7 @@ static const int64 MAX_CLOCK_DRIFT = 10 * 60; static const int PROOF_OF_WORK_BITS_LOWER_LIMIT = 8; static const int PROOF_OF_WORK_BITS_UPPER_LIMIT = 200; -static const int PROOF_OF_WORK_BITS_INIT_MAINNET = 32; +static const int PROOF_OF_WORK_BITS_INIT_MAINNET = 8; //32 static const int PROOF_OF_WORK_BITS_INIT_TESTNET = 10; static const int PROOF_OF_WORK_ADJUST_COUNT = 8; static const int PROOF_OF_WORK_ADJUST_DEBOUNCE = 15; diff --git a/src/bigbang/entry.cpp b/src/bigbang/entry.cpp index b05e4592..c711c834 100644 --- a/src/bigbang/entry.cpp +++ b/src/bigbang/entry.cpp @@ -27,6 +27,7 @@ #include "txpool.h" #include "version.h" #include "wallet.h" +#include "mqcluster.h" #ifdef WIN32 #ifdef _MSC_VER @@ -405,6 +406,14 @@ bool CBbEntry::InitializeModules(const EModeType& mode) } break; } + case EModuleType::MQCLUSTER: + { + if (!AttachModule(new CMQCluster())) + { + return false; + } + break; + } default: cerr << "Unknown module:%d" << CMode::IntValue(m) << endl; break; diff --git a/src/bigbang/mode/mode.h b/src/bigbang/mode/mode.h index 8ce5ef36..9e6d9f68 100644 --- a/src/bigbang/mode/mode.h +++ b/src/bigbang/mode/mode.h @@ -171,6 +171,7 @@ class CMode EModuleType::HTTPSERVER, EModuleType::RPCMODE, EModuleType::BLOCKMAKER, + EModuleType::MQCLUSTER, EModuleType::DATASTAT } }, { EModeType::CONSOLE, { EModuleType::HTTPGET, diff --git a/src/bigbang/mode/module_type.h b/src/bigbang/mode/module_type.h index 6f5e04a8..d3ead603 100644 --- a/src/bigbang/mode/module_type.h +++ b/src/bigbang/mode/module_type.h @@ -29,6 +29,7 @@ enum class EModuleType CONSENSUS, // CConsensus FORKMANAGER, // CForkManager DATASTAT, // CDataStat + MQCLUSTER, // CMQCluster }; } // namespace bigbang diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp new file mode 100644 index 00000000..9e99e03a --- /dev/null +++ b/src/bigbang/mqcluster.cpp @@ -0,0 +1,228 @@ +// Copyright (c) 2019-2020 The Bigbang developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "mqcluster.h" + +#include +#include +#include +#include // For sleep +#include +#include +#include +#include "async_client.h" + +using namespace std; + +namespace bigbang +{ + +CMQCluster::CMQCluster() + : thrMqttClient("mqttcli", boost::bind(&CMQCluster::MqttThreadFunc, this)), + pCoreProtocol(nullptr), pBlockChain(nullptr), pService(nullptr), fAuth(false), fAbort(false) +{ + +} + +bool CMQCluster::IsAuthenticated() +{ + return fAuth; +} + +bool CMQCluster::HandleInitialize() +{ + if (!GetObject("coreprotocol", pCoreProtocol)) + { + Error("Failed to request coreprotocol"); + return false; + } + + if (!GetObject("blockchain", pBlockChain)) + { + Error("Failed to request blockchain"); + return false; + } + + if (!GetObject("service", pService)) + { + Error("Failed to request service"); + return false; + } + + Log("CMQCluster::HandleInitialize() successfully"); + return true; +} + +void CMQCluster::HandleDeinitialize() +{ + pCoreProtocol = nullptr; + pBlockChain = nullptr; + pService = nullptr; +} + +bool CMQCluster::HandleInvoke() +{ + if (!ThreadDelayStart(thrMqttClient)) + { + return false; + } + return IIOModule::HandleInvoke(); +} + +void CMQCluster::HandleHalt() +{ + IIOModule::HandleHalt(); + + condMQ.notify_all(); + + if (thrMqttClient.IsRunning()) + { + thrMqttClient.Interrupt(); + } + ThreadExit(thrMqttClient); +} + +bool CMQCluster::HandleEvent(CEventMQSyncBlock& eventMqSyncBlock) +{ + return true; +} + +bool CMQCluster::HandleEvent(CEventMQUpdateBlock& eventMqUpdateBlock) +{ + return true; +} + +bool CMQCluster::HandleEvent(CEventMQAgreement& eventMqAgreement) +{ + return true; +} + +class callback : public virtual mqtt::callback +{ +public: + void connection_lost(const string& cause) override { + cout << "\nConnection lost" << endl; + if (!cause.empty()) + cout << "\tcause: " << cause << endl; + } + + void delivery_complete(mqtt::delivery_token_ptr tok) override { + cout << "\tDelivery complete for token: " + << (tok ? tok->get_message_id() : -1) << endl; + } +}; + +class action_listener : public virtual mqtt::iaction_listener +{ +protected: + void on_failure(const mqtt::token& tok) override { + cout << "\tListener failure for token: " + << tok.get_message_id() << endl; + } + + void on_success(const mqtt::token& tok) override { + cout << "\tListener success for token: " + << tok.get_message_id() << endl; + } +}; + +class delivery_action_listener : public action_listener +{ + atomic done_; + + void on_failure(const mqtt::token& tok) override { + action_listener::on_failure(tok); + done_ = true; + } + + void on_success(const mqtt::token& tok) override { + action_listener::on_success(tok); + done_ = true; + } + +public: + delivery_action_listener() : done_(false) {} + bool is_done() const { return done_; } +}; + +void CMQCluster::Publish() +{ + const std::string SERVER_ADDRESS { "tcp://localhost:1883" }; + const std::string CLIENT_ID { "DPOS-NODE1" }; + const string TOPIC { "Cluster01/dpos/SyncBlockReq" }; + const char* PAYLOAD = "Request for secure main block chain for BBC."; + const char* LWT_PAYLOAD = "Last will and testament."; + + string address = SERVER_ADDRESS; + clientID = CLIENT_ID; + + cout << "Initializing for server '" << address << "'..." << endl; + mqtt::async_client client(address, clientID); + + callback cb; + client.set_callback(cb); + + mqtt::connect_options conopts; + mqtt::message willmsg(TOPIC, LWT_PAYLOAD, 1, true); + mqtt::will_options will(willmsg); + conopts.set_will(will); + + cout << " ...OK" << endl; + + try { + cout << "\nConnecting..." << endl; + mqtt::token_ptr conntok = client.connect(conopts); + cout << "Waiting for the connection..." << endl; + conntok->wait(); + cout << " ...OK" << endl; + + cout << "\nSending message..." << endl; + delivery_action_listener deliveryListener; + mqtt::message_ptr pubmsg = mqtt::make_message(TOPIC, PAYLOAD); + client.publish(pubmsg, nullptr, deliveryListener); + + while (!deliveryListener.is_done()) { + this_thread::sleep_for(std::chrono::milliseconds(100)); + } + cout << "OK" << endl; + + // Double check that there are no pending tokens + + auto toks = client.get_pending_delivery_tokens(); + if (!toks.empty()) + cout << "Error: There are pending delivery tokens!" << endl; + + // Disconnect + cout << "\nDisconnecting..." << endl; + conntok = client.disconnect(); + conntok->wait(); + cout << " ...OK" << endl; + } + catch (const mqtt::exception& exc) { + cerr << exc.what() << endl; + } +} + +void CMQCluster::MqttThreadFunc() +{ + Log("entering thread function of MQTT"); + while(!fAbort) + { + boost::system_time const timeout = boost::get_system_time() + boost::posix_time::seconds(5); + { + boost::unique_lock lock(mutex); + while (!fAbort) + { + if (!condMQ.timed_wait(lock, timeout)) + { + break; + } + } + } + Publish(); + Log("thread function of MQTT: go through an iteration"); + } +} + +} // namespace bigbang diff --git a/src/bigbang/mqcluster.h b/src/bigbang/mqcluster.h new file mode 100644 index 00000000..5a5e7a17 --- /dev/null +++ b/src/bigbang/mqcluster.h @@ -0,0 +1,63 @@ +// Copyright (c) 2019-2020 The Bigbang developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BIGBANG_CMQCLUSTER_H +#define BIGBANG_CMQCLUSTER_H + +#include "base.h" +#include "mqevent.h" +#include "xengine.h" + +namespace bigbang +{ + +class IMQCluster : public xengine::IIOModule, virtual public CMQEventListener +{ +public: + IMQCluster() + : IIOModule("mqcluster") {} + virtual bool IsAuthenticated() = 0; +}; + +class CMQCluster : public IMQCluster +{ + enum NODE_TYPE {BBCNODE, FORKNODE, DPOSNODE}; +public: + CMQCluster(); + ~CMQCluster() = default; + +protected: + bool HandleInitialize() override; + void HandleDeinitialize() override; + bool HandleInvoke() override; + void HandleHalt() override; + + bool HandleEvent(CEventMQSyncBlock& eventMqSyncBlock) override; + bool HandleEvent(CEventMQUpdateBlock& eventMqUpdateBlock) override; + bool HandleEvent(CEventMQAgreement& eventMqAgreement) override; + + bool IsAuthenticated() override; + +protected: + mutable boost::shared_mutex rwAccess; + ICoreProtocol* pCoreProtocol; + IBlockChain* pBlockChain; + IService* pService; + + boost::mutex mutex; + boost::condition_variable condMQ; + xengine::CThread thrMqttClient; + +private: + void Publish(); + //bool Publish(); + void MqttThreadFunc(); + bool fAuth; + bool fAbort; + string clientID; +}; + +} // namespace bigbang + +#endif //BIGBANG_CMQCLUSTER_H diff --git a/src/bigbang/mqevent.h b/src/bigbang/mqevent.h new file mode 100644 index 00000000..6feaf914 --- /dev/null +++ b/src/bigbang/mqevent.h @@ -0,0 +1,46 @@ +// Copyright (c) 2019-2020 The Bigbang developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BIGBANG_MQEVENT_H +#define BIGBANG_MQEVENT_H + +#include "xengine.h" +#include "struct.h" + +using namespace xengine; + +namespace bigbang +{ + +enum +{ + EVENT_MQ_BASE = EVENT_USER_BASE, + EVENT_MQ_SYNCBLOCK, + EVENT_MQ_UPDATEBLOCK, + EVENT_MQ_AGREEMENT +}; + +class CMQEventListener; + +#define TYPE_MQEVENT(type, body, res) \ + CEventCategory + + + +typedef TYPE_MQEVENT(EVENT_MQ_SYNCBLOCK, int, std::string) CEventMQSyncBlock; +typedef TYPE_MQEVENT(EVENT_MQ_UPDATEBLOCK, CDelegateAgreement, int) CEventMQUpdateBlock; +typedef TYPE_MQEVENT(EVENT_MQ_AGREEMENT, CDelegateAgreement, int) CEventMQAgreement; + +class CMQEventListener : virtual public CEventListener +{ +public: + virtual ~CMQEventListener() {} + DECLARE_EVENTHANDLER(CEventMQSyncBlock); + DECLARE_EVENTHANDLER(CEventMQUpdateBlock); + DECLARE_EVENTHANDLER(CEventMQAgreement); +}; + +} // namespace bigbang + +#endif //BIGBANG_MQEVENT_H diff --git a/src/xengine/util.cpp b/src/xengine/util.cpp index 28572950..c31dea36 100644 --- a/src/xengine/util.cpp +++ b/src/xengine/util.cpp @@ -223,7 +223,7 @@ void StdTrace(const char* pszName, const char* pszFormat, ...) ss << arg_buffer; std::string str = ss.str(); - BOOST_LOG_SCOPED_THREAD_TAG("ThreadName", GetThreadName().c_str()); + BOOST_LOG_SCOPED_THREAD_TAG("ThreadName", GetThreadName().c_str()) BOOST_LOG_CHANNEL_SEV(lg::get(), pszName, debug) << str; } } @@ -241,7 +241,7 @@ void StdDebug(const char* pszName, const char* pszFormat, ...) ss << arg_buffer; std::string str = ss.str(); - BOOST_LOG_SCOPED_THREAD_TAG("ThreadName", GetThreadName().c_str()); + BOOST_LOG_SCOPED_THREAD_TAG("ThreadName", GetThreadName().c_str()) BOOST_LOG_CHANNEL_SEV(lg::get(), pszName, debug) << str; } } @@ -259,7 +259,7 @@ void StdLog(const char* pszName, const char* pszFormat, ...) ss << arg_buffer; std::string str = ss.str(); - BOOST_LOG_SCOPED_THREAD_TAG("ThreadName", GetThreadName().c_str()); + BOOST_LOG_SCOPED_THREAD_TAG("ThreadName", GetThreadName().c_str()) BOOST_LOG_CHANNEL_SEV(lg::get(), pszName, info) << str; } } @@ -277,7 +277,7 @@ void StdWarn(const char* pszName, const char* pszFormat, ...) ss << arg_buffer; std::string str = ss.str(); - BOOST_LOG_SCOPED_THREAD_TAG("ThreadName", GetThreadName().c_str()); + BOOST_LOG_SCOPED_THREAD_TAG("ThreadName", GetThreadName().c_str()) BOOST_LOG_CHANNEL_SEV(lg::get(), pszName, warn) << str; } } @@ -295,7 +295,7 @@ void StdError(const char* pszName, const char* pszFormat, ...) ss << arg_buffer; std::string str = ss.str(); - BOOST_LOG_SCOPED_THREAD_TAG("ThreadName", GetThreadName().c_str()); + BOOST_LOG_SCOPED_THREAD_TAG("ThreadName", GetThreadName().c_str()) BOOST_LOG_CHANNEL_SEV(lg::get(), pszName, error) << str; } } From b7a146f5332ca1c8868263d354cf22bee9d3ae44 Mon Sep 17 00:00:00 2001 From: OuYun Date: Fri, 6 Mar 2020 10:35:09 +0800 Subject: [PATCH 006/181] correct some typos --- doc/auto_rpc.md | 4 ++-- script/auto_protocol.py | 6 +++--- src/bigbang/mode/mode.h | 2 +- src/bigbang/mode/mode_impl.h | 8 ++++---- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/doc/auto_rpc.md b/doc/auto_rpc.md index b4d66762..0e7f77b7 100755 --- a/doc/auto_rpc.md +++ b/doc/auto_rpc.md @@ -30,11 +30,11 @@ CBasicConfig* CreateConfig(const std::string& cmd) { if (cmd == "help") { - return new mode_impl::CCombinConfig; + return new mode_impl::CCombineConfig; } else if (cmd == "stop") { - return new mode_impl::CCombinConfig; + return new mode_impl::CCombineConfig; } ... } diff --git a/script/auto_protocol.py b/script/auto_protocol.py index 12e5f39b..d0f2be60 100755 --- a/script/auto_protocol.py +++ b/script/auto_protocol.py @@ -1074,20 +1074,20 @@ def CreateConfig_h(w): indent = brace_begin(w) if len(config_class) == 0: - w.write(indent + 'return new mode_impl::CCombinConfig();\n') + w.write(indent + 'return new mode_impl::CCombineConfig();\n') else: w.write(indent) for cmd, name in config_class.items(): w.write('if (cmd == "' + cmd + '")\n') indent = brace_begin(w, indent) - w.write(indent + 'return new mode_impl::CCombinConfig<' + + w.write(indent + 'return new mode_impl::CCombineConfig<' + name + ', T...>;\n') indent = brace_end(w, indent) w.write(indent + 'else ') empty_line(w) indent = brace_begin(w, indent) - w.write(indent + 'return new mode_impl::CCombinConfig();\n') + w.write(indent + 'return new mode_impl::CCombineConfig();\n') indent = brace_end(w, indent) brace_end(w, indent) diff --git a/src/bigbang/mode/mode.h b/src/bigbang/mode/mode.h index 9e6d9f68..9e42fd26 100644 --- a/src/bigbang/mode/mode.h +++ b/src/bigbang/mode/mode.h @@ -103,7 +103,7 @@ class CMode } else { - return new mode_impl::CCombinConfig< + return new mode_impl::CCombineConfig< typename std::remove_pointer(t), decltype(config_type::___ConfigTypeTemplate)>::type>::type...>; diff --git a/src/bigbang/mode/mode_impl.h b/src/bigbang/mode/mode_impl.h index 7340cd12..c7c63357 100644 --- a/src/bigbang/mode/mode_impl.h +++ b/src/bigbang/mode/mode_impl.h @@ -20,11 +20,11 @@ namespace mode_impl * Combination of inheriting all need config class. */ template -class CCombinConfig : virtual public std::enable_if::value, U>::type... +class CCombineConfig : virtual public std::enable_if::value, U>::type... { public: - CCombinConfig() {} - virtual ~CCombinConfig() {} + CCombineConfig() {} + virtual ~CCombineConfig() {} virtual bool PostLoad() { @@ -51,7 +51,7 @@ class CCombinConfig : virtual public std::enable_if -class CCombinConfig<> : virtual public CCombinConfig +class CCombineConfig<> : virtual public CCombineConfig { }; From 19b2f8f21bf4ee9fc6a33b35059d08850d91cd04 Mon Sep 17 00:00:00 2001 From: OuYun Date: Fri, 6 Mar 2020 10:54:57 +0800 Subject: [PATCH 007/181] remove unused source code head file --- src/bigbang/CMakeLists.txt | 1 - src/bigbang/storage.h | 45 -------------------------------------- 2 files changed, 46 deletions(-) delete mode 100644 src/bigbang/storage.h diff --git a/src/bigbang/CMakeLists.txt b/src/bigbang/CMakeLists.txt index 8759627b..b58c16e6 100755 --- a/src/bigbang/CMakeLists.txt +++ b/src/bigbang/CMakeLists.txt @@ -54,7 +54,6 @@ set(src base.h struct.h param.h - storage.h version.h mqevent.h ) diff --git a/src/bigbang/storage.h b/src/bigbang/storage.h deleted file mode 100644 index d93446a7..00000000 --- a/src/bigbang/storage.h +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) 2019-2020 The Bigbang developers -// Distributed under the MIT/X11 software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BIGBANG_STORAGE_H -#define BIGBANG_STORAGE_H - -#include -#include - -#include "config.h" - -namespace bigbang -{ - -class CEntry : public xengine::CEntry -{ -public: - CEntry(); - ~CEntry(); - bool Initialize(int argc, char* argv[]); - bool Run(); - void Exit(); - -protected: - bool InitializeService(); - bool InitializeClient(); - // xengine::CHttpHostConfig GetRPCHostConfig(); - // xengine::CHttpHostConfig GetWebUIHostConfig(); - - boost::filesystem::path GetDefaultDataDir(); - - bool SetupEnvironment(); - bool RunInBackground(const boost::filesystem::path& pathData); - void ExitBackground(const boost::filesystem::path& pathData); - -protected: - CConfig config; - xengine::CLog log; - xengine::CDocker docker; -}; - -} // namespace bigbang - -#endif //BIGBANG_STORAGE_H From 8d970574ab34d6424037e24b2753c81c986ad654 Mon Sep 17 00:00:00 2001 From: OuYun Date: Fri, 6 Mar 2020 17:06:21 +0800 Subject: [PATCH 008/181] add identifying of node category to module --- src/bigbang/entry.cpp | 2 +- src/bigbang/mqcluster.cpp | 150 ++++++++++++++++++++++++++++++-------- src/bigbang/mqcluster.h | 11 ++- 3 files changed, 126 insertions(+), 37 deletions(-) diff --git a/src/bigbang/entry.cpp b/src/bigbang/entry.cpp index c711c834..d5a80c06 100644 --- a/src/bigbang/entry.cpp +++ b/src/bigbang/entry.cpp @@ -408,7 +408,7 @@ bool CBbEntry::InitializeModules(const EModeType& mode) } case EModuleType::MQCLUSTER: { - if (!AttachModule(new CMQCluster())) + if (!AttachModule(new CMQCluster(config.GetConfig()->nCatOfNode))) { return false; } diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 9e99e03a..d196b879 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -4,13 +4,14 @@ #include "mqcluster.h" -#include -#include -#include -#include // For sleep #include #include +#include #include +#include +#include +#include // For sleep + #include "async_client.h" using namespace std; @@ -18,11 +19,26 @@ using namespace std; namespace bigbang { -CMQCluster::CMQCluster() +CMQCluster::CMQCluster(int catNodeIn) : thrMqttClient("mqttcli", boost::bind(&CMQCluster::MqttThreadFunc, this)), - pCoreProtocol(nullptr), pBlockChain(nullptr), pService(nullptr), fAuth(false), fAbort(false) + pCoreProtocol(nullptr), + pBlockChain(nullptr), + pService(nullptr), + fAuth(false), + fAbort(false) { - + switch (catNodeIn) + { + case 0: + catNode = NODE_CATEGORY::BBCNODE; + break; + case 1: + catNode = NODE_CATEGORY::FORKNODE; + break; + case 2: + catNode = NODE_CATEGORY::DPOSNODE; + break; + } } bool CMQCluster::IsAuthenticated() @@ -32,6 +48,20 @@ bool CMQCluster::IsAuthenticated() bool CMQCluster::HandleInitialize() { + if (NODE_CATEGORY::FORKNODE == catNode) + { + clientID = "FORKNODE-01"; + } + else if (NODE_CATEGORY::DPOSNODE == catNode) + { + clientID = "DPOSNODE"; + } + else if (NODE_CATEGORY::BBCNODE == catNode) + { + Log("CMQCluster::HandleInitialize(): bbc node so go passby"); + return true; + } + if (!GetObject("coreprotocol", pCoreProtocol)) { Error("Failed to request coreprotocol"); @@ -63,6 +93,12 @@ void CMQCluster::HandleDeinitialize() bool CMQCluster::HandleInvoke() { + if (NODE_CATEGORY::BBCNODE == catNode) + { + Log("CMQCluster::HandleInvoke(): bbc node so go passby"); + return true; + } + if (!ThreadDelayStart(thrMqttClient)) { return false; @@ -101,13 +137,22 @@ bool CMQCluster::HandleEvent(CEventMQAgreement& eventMqAgreement) class callback : public virtual mqtt::callback { public: - void connection_lost(const string& cause) override { + void connected(const string& cause) override + { + cout << "\nConnection connected" << endl; + if (!cause.empty()) + cout << "\tcause: " << cause << endl; + } + + void connection_lost(const string& cause) override + { cout << "\nConnection lost" << endl; if (!cause.empty()) cout << "\tcause: " << cause << endl; } - void delivery_complete(mqtt::delivery_token_ptr tok) override { + void delivery_complete(mqtt::delivery_token_ptr tok) override + { cout << "\tDelivery complete for token: " << (tok ? tok->get_message_id() : -1) << endl; } @@ -116,12 +161,14 @@ class callback : public virtual mqtt::callback class action_listener : public virtual mqtt::iaction_listener { protected: - void on_failure(const mqtt::token& tok) override { + void on_failure(const mqtt::token& tok) override + { cout << "\tListener failure for token: " << tok.get_message_id() << endl; } - void on_success(const mqtt::token& tok) override { + void on_success(const mqtt::token& tok) override + { cout << "\tListener success for token: " << tok.get_message_id() << endl; } @@ -131,31 +178,40 @@ class delivery_action_listener : public action_listener { atomic done_; - void on_failure(const mqtt::token& tok) override { + void on_failure(const mqtt::token& tok) override + { action_listener::on_failure(tok); done_ = true; } - void on_success(const mqtt::token& tok) override { + void on_success(const mqtt::token& tok) override + { action_listener::on_success(tok); done_ = true; } public: - delivery_action_listener() : done_(false) {} - bool is_done() const { return done_; } + delivery_action_listener() + : done_(false) {} + bool is_done() const + { + return done_; + } }; -void CMQCluster::Publish() +bool CMQCluster::ConnectBroker() +{ + return true; +} + +bool CMQCluster::Publish() { - const std::string SERVER_ADDRESS { "tcp://localhost:1883" }; - const std::string CLIENT_ID { "DPOS-NODE1" }; - const string TOPIC { "Cluster01/dpos/SyncBlockReq" }; + const std::string SERVER_ADDRESS{ "tcp://localhost:1883" }; + const string TOPIC{ "Cluster01/dpos/SyncBlockReq" }; const char* PAYLOAD = "Request for secure main block chain for BBC."; const char* LWT_PAYLOAD = "Last will and testament."; - string address = SERVER_ADDRESS; - clientID = CLIENT_ID; + string address = SERVER_ADDRESS; cout << "Initializing for server '" << address << "'..." << endl; mqtt::async_client client(address, clientID); @@ -170,22 +226,27 @@ void CMQCluster::Publish() cout << " ...OK" << endl; - try { + try + { cout << "\nConnecting..." << endl; mqtt::token_ptr conntok = client.connect(conopts); cout << "Waiting for the connection..." << endl; conntok->wait(); cout << " ...OK" << endl; - cout << "\nSending message..." << endl; - delivery_action_listener deliveryListener; - mqtt::message_ptr pubmsg = mqtt::make_message(TOPIC, PAYLOAD); - client.publish(pubmsg, nullptr, deliveryListener); + for (int i = 0; i < 10; ++i) + { + cout << "\nSending message" << to_string(i) << "..." << endl; + delivery_action_listener deliveryListener; + mqtt::message_ptr pubmsg = mqtt::make_message(TOPIC, PAYLOAD); + client.publish(pubmsg, nullptr, deliveryListener); - while (!deliveryListener.is_done()) { - this_thread::sleep_for(std::chrono::milliseconds(100)); + while (!deliveryListener.is_done()) + { + this_thread::sleep_for(std::chrono::milliseconds(100)); + } + cout << "OK" << endl; } - cout << "OK" << endl; // Double check that there are no pending tokens @@ -199,17 +260,37 @@ void CMQCluster::Publish() conntok->wait(); cout << " ...OK" << endl; } - catch (const mqtt::exception& exc) { + catch (const mqtt::exception& exc) + { cerr << exc.what() << endl; + return false; } + return true; +} + +bool CMQCluster::Subscribe() +{ + return true; +} + +bool CMQCluster::DisconnectBroker() +{ + return true; } void CMQCluster::MqttThreadFunc() { Log("entering thread function of MQTT"); - while(!fAbort) + //establish connection + ; + + //subscribe topics + Subscribe(); + + //publish topics + while (!fAbort) { - boost::system_time const timeout = boost::get_system_time() + boost::posix_time::seconds(5); + boost::system_time const timeout = boost::get_system_time() + boost::posix_time::seconds(30); { boost::unique_lock lock(mutex); while (!fAbort) @@ -223,6 +304,11 @@ void CMQCluster::MqttThreadFunc() Publish(); Log("thread function of MQTT: go through an iteration"); } + + //disconnect to broker + ; + + Log("exiting thread function of MQTT"); } } // namespace bigbang diff --git a/src/bigbang/mqcluster.h b/src/bigbang/mqcluster.h index 5a5e7a17..9b7e25c6 100644 --- a/src/bigbang/mqcluster.h +++ b/src/bigbang/mqcluster.h @@ -22,9 +22,9 @@ class IMQCluster : public xengine::IIOModule, virtual public CMQEventListener class CMQCluster : public IMQCluster { - enum NODE_TYPE {BBCNODE, FORKNODE, DPOSNODE}; + enum class NODE_CATEGORY : char {BBCNODE = 0, FORKNODE, DPOSNODE}; public: - CMQCluster(); + CMQCluster(int catNodeIn); ~CMQCluster() = default; protected: @@ -50,12 +50,15 @@ class CMQCluster : public IMQCluster xengine::CThread thrMqttClient; private: - void Publish(); - //bool Publish(); + bool ConnectBroker(); + bool Publish(); + bool Subscribe(); + bool DisconnectBroker(); void MqttThreadFunc(); bool fAuth; bool fAbort; string clientID; + NODE_CATEGORY catNode; }; } // namespace bigbang From 27a993228b344e116c152f56f3669a6027b2a37f Mon Sep 17 00:00:00 2001 From: OuYun Date: Mon, 9 Mar 2020 11:50:44 +0800 Subject: [PATCH 009/181] add storage function for enrolling fork nodes --- src/bigbang/mqcluster.cpp | 7 ++- src/storage/CMakeLists.txt | 2 +- src/storage/blockbase.cpp | 16 +++++++ src/storage/blockbase.h | 1 + src/storage/blockdb.cpp | 12 +++++ src/storage/blockdb.h | 3 ++ src/storage/mqdb.cpp | 96 ++++++++++++++++++++++++++++++++++++++ src/storage/mqdb.h | 53 +++++++++++++++++++++ 8 files changed, 188 insertions(+), 2 deletions(-) create mode 100644 src/storage/mqdb.cpp create mode 100644 src/storage/mqdb.h diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index d196b879..d4e9fd6a 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -108,10 +108,15 @@ bool CMQCluster::HandleInvoke() void CMQCluster::HandleHalt() { + if (NODE_CATEGORY::BBCNODE == catNode) + { + Log("CMQCluster::HandleHalt(): bbc node so go passby"); + return; + } + IIOModule::HandleHalt(); condMQ.notify_all(); - if (thrMqttClient.IsRunning()) { thrMqttClient.Interrupt(); diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index dfe18a9c..e80ea7b7 100755 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -20,7 +20,7 @@ set(sources leveldbeng.cpp leveldbeng.h txindexdb.cpp txindexdb.h ctsdb.cpp ctsdb.h -) + mqdb.cpp mqdb.h) add_library(storage ${sources}) diff --git a/src/storage/blockbase.cpp b/src/storage/blockbase.cpp index e5933466..55c13325 100644 --- a/src/storage/blockbase.cpp +++ b/src/storage/blockbase.cpp @@ -493,6 +493,22 @@ bool CBlockBase::AddNewForkContext(const CForkContext& ctxt) return true; } +bool CBlockBase::AddNewForkNode(const CForkNode& forkNode) +{ + if (!dbBlock.AddNewForkNode(forkNode)) + { + Error("F", "Failed to addnew forknode in %s", forkNode.forkNodeID.c_str()); + return false; + } + Log("F", "AddNew forknode,cliID=%s with %n forks", forkNode.forkNodeID.c_str(), + forkNode.vecOwnedForks.size()); + for (const auto& i : forkNode.vecOwnedForks) + { + Log("F", "AddNew forknode with fork [%s]", i.ToString().c_str()); + } + return true; +} + bool CBlockBase::Retrieve(const uint256& hash, CBlock& block) { block.SetNull(); diff --git a/src/storage/blockbase.h b/src/storage/blockbase.h index 81ea5082..e1bf0df7 100644 --- a/src/storage/blockbase.h +++ b/src/storage/blockbase.h @@ -233,6 +233,7 @@ class CBlockBase bool Initiate(const uint256& hashGenesis, const CBlock& blockGenesis, const uint256& nChainTrust); bool AddNew(const uint256& hash, CBlockEx& block, CBlockIndex** ppIndexNew, const uint256& nChainTrust); bool AddNewForkContext(const CForkContext& ctxt); + bool AddNewForkNode(const CForkNode& forkNode); bool Retrieve(const uint256& hash, CBlock& block); bool Retrieve(const CBlockIndex* pIndex, CBlock& block); bool Retrieve(const uint256& hash, CBlockEx& block); diff --git a/src/storage/blockdb.cpp b/src/storage/blockdb.cpp index 4eb0a4f9..de164831 100644 --- a/src/storage/blockdb.cpp +++ b/src/storage/blockdb.cpp @@ -51,6 +51,11 @@ bool CBlockDB::Initialize(const boost::filesystem::path& pathData) return false; } + if (!dbForkNode.Initialize(pathData)) + { + return false; + } + return LoadFork(); } @@ -61,6 +66,7 @@ void CBlockDB::Deinitialize() dbTxIndex.Deinitialize(); dbBlockIndex.Deinitialize(); dbFork.Deinitialize(); + dbForkNode.Deinitialize(); } bool CBlockDB::RemoveAll() @@ -70,6 +76,7 @@ bool CBlockDB::RemoveAll() dbTxIndex.Clear(); dbBlockIndex.Clear(); dbFork.Clear(); + dbForkNode.Clear(); return true; } @@ -244,5 +251,10 @@ bool CBlockDB::LoadFork() return true; } +bool CBlockDB::AddNewForkNode(const CForkNode& forkNode) +{ + return dbForkNode.AddNewForkNode(forkNode); +} + } // namespace storage } // namespace bigbang diff --git a/src/storage/blockdb.h b/src/storage/blockdb.h index a49c9942..84a90f0c 100644 --- a/src/storage/blockdb.h +++ b/src/storage/blockdb.h @@ -13,6 +13,7 @@ #include "transaction.h" #include "txindexdb.h" #include "unspentdb.h" +#include "mqdb.h" namespace bigbang { @@ -47,6 +48,7 @@ class CBlockDB bool RetrieveDelegate(const uint256& hash, std::map& mapDelegate); bool RetrieveEnroll(int height, const std::vector& vBlockRange, std::map& mapEnrollTxPos); + bool AddNewForkNode(const CForkNode& forkNode); protected: bool LoadFork(); @@ -57,6 +59,7 @@ class CBlockDB CTxIndexDB dbTxIndex; CUnspentDB dbUnspent; CDelegateDB dbDelegate; + CForkNodeDB dbForkNode; }; } // namespace storage diff --git a/src/storage/mqdb.cpp b/src/storage/mqdb.cpp new file mode 100644 index 00000000..f44285e7 --- /dev/null +++ b/src/storage/mqdb.cpp @@ -0,0 +1,96 @@ +// Copyright (c) 2019-2020 The Bigbang developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "mqdb.h" + +#include "leveldbeng.h" + +using namespace std; +using namespace xengine; + +namespace bigbang +{ +namespace storage +{ + +////////////////////////////// +// CForkNodeDB + +bool CForkNodeDB::Initialize(const boost::filesystem::path& pathData) +{ + CLevelDBArguments args; + args.path = (pathData / "forknode").string(); + args.syncwrite = true; + args.files = 16; + args.cache = 2 << 20; + + CLevelDBEngine* engine = new CLevelDBEngine(args); + + if (!Open(engine)) + { + delete engine; + return false; + } + + return true; +} + +void CForkNodeDB::Deinitialize() +{ + Close(); +} + +bool CForkNodeDB::AddNewForkNode(const CForkNode& cli) +{ + return Write(cli.forkNodeID, cli.vecOwnedForks, true); //overwrite +} + +bool CForkNodeDB::RemoveForkNode(const string& cliID) +{ + return Erase(cliID); +} + +bool CForkNodeDB::RetrieveForkNode(const string& cliID, CForkNode& cli) +{ + return Read(cliID, cli); +} + +bool CForkNodeDB::ListForkNode(std::vector& vCli) +{ + map> mapCli; + + if (!WalkThrough(boost::bind(&CForkNodeDB::LoadForkNodeWalker, this, _1, _2, boost::ref(mapCli)))) + { + return false; + } + + vCli.reserve(mapCli.size()); + for (const auto& it : mapCli) + { + CForkNode node; + node.forkNodeID = it.first; + node.vecOwnedForks = it.second; + vCli.emplace_back(node); + } + return true; +} + +void CForkNodeDB::Clear() +{ + RemoveAll(); +} + +bool CForkNodeDB::LoadForkNodeWalker(xengine::CBufStream& ssKey, xengine::CBufStream& ssValue, + map>& mapCli) +{ + string strCliID; + ssKey >> strCliID; + std::vector forks; + ssValue >> forks; + mapCli.insert(make_pair(strCliID, forks)); + return true; +} + +} // namespace storage +} // namespace bigbang diff --git a/src/storage/mqdb.h b/src/storage/mqdb.h new file mode 100644 index 00000000..f5a2508e --- /dev/null +++ b/src/storage/mqdb.h @@ -0,0 +1,53 @@ +// Copyright (c) 2019-2020 The Bigbang developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BIGBANG_MQDB_H +#define BIGBANG_MQDB_H + +#include "uint256.h" +#include "xengine.h" + +namespace bigbang +{ +namespace storage +{ + +class CForkNode +{ + friend class xengine::CStream; + +public: + std::string forkNodeID; + std::vector vecOwnedForks; + +protected: + template + void Serialize(xengine::CStream& s, O& opt) + { + s.Serialize(forkNodeID, opt); + s.Serialize(vecOwnedForks, opt); + } +}; + +class CForkNodeDB : public xengine::CKVDB +{ +public: + CForkNodeDB() {} + bool Initialize(const boost::filesystem::path& pathData); + void Deinitialize(); + bool AddNewForkNode(const CForkNode& cli); + bool RemoveForkNode(const std::string& cliID); + bool RetrieveForkNode(const std::string& forkNodeID, CForkNode& cli); + bool ListForkNode(std::vector& vCli); + void Clear(); + +protected: + bool LoadForkNodeWalker(xengine::CBufStream& ssKey, xengine::CBufStream& ssValue, + std::map>& mapCli); +}; + +} // namespace storage +} // namespace bigbang + +#endif //BIGBANG_MQDB_H From fc14c5493773e36f981f76fccc338b6ea250d4b0 Mon Sep 17 00:00:00 2001 From: OuYun Date: Mon, 9 Mar 2020 18:52:35 +0800 Subject: [PATCH 010/181] add new RPC command: enrollforknode --- script/template/rpc.json | 41 ++++++++++++++++++++++++++++++++++++++ src/bigbang/base.h | 3 +++ src/bigbang/blockchain.cpp | 9 +++++++++ src/bigbang/blockchain.h | 1 + src/bigbang/mqcluster.cpp | 4 ++-- src/bigbang/rpcmod.cpp | 33 ++++++++++++++++++++++++++++++ src/bigbang/rpcmod.h | 1 + src/bigbang/service.cpp | 5 +++++ src/bigbang/service.h | 2 ++ src/storage/blockbase.cpp | 2 +- src/storage/mqdb.cpp | 11 +++++++++- 11 files changed, 108 insertions(+), 4 deletions(-) diff --git a/script/template/rpc.json b/script/template/rpc.json index 0fad784a..0ce18180 100755 --- a/script/template/rpc.json +++ b/script/template/rpc.json @@ -638,6 +638,47 @@ } ] }, + "enrollforknode": { + "type": "command", + "name": "EnrollForkNode", + "desc": "Enroll fork node info for MQ communication on both sides of dpos and fork nodes.", + "request": { + "type": "object", + "content": { + "clientid" : + { + "type": "string", + "desc": "client id" + }, + "forks" : + { + "type" : "array", + "content" : { + "fork" : + { + "type" : "string", + "dest" : "fork hash" + } + } + } + } + }, + "response": { + "type": "bool", + "name": "retflag", + "desc": "successful or not" + }, + "example": [ + { + "request": "bigbang-cli enrollforknode forknode01 '[\"575f2041770496489120bb102d9dd55f5e75b0c4aa528d5762b92b59acd6d939\",\"6236d780f9f743707d57b3feb19d21c8a867577f5e83c163774222bb7ef8d8cb\",\"7c6a6aba05cec77a998c19649ee1fa0e29c7b5246d0e3a6501ee1d4d81dd73ea\"]'", + "response": "true" + }, + { + "request": "curl -d '{\"id\":3,\"method\":\"enrollforknode\",\"jsonrpc\":\"2.0\",\"params\":{[\"575f2041770496489120bb102d9dd55f5e75b0c4aa528d5762b92b59acd6d939\",\"6236d780f9f743707d57b3feb19d21c8a867577f5e83c163774222bb7ef8d8cb\",\"7c6a6aba05cec77a998c19649ee1fa0e29c7b5246d0e3a6501ee1d4d81dd73ea\"]}}' http://127.0.0.1:9902", + "response": "{\"id\":3,\"jsonrpc\":\"2.0\",\"result\":true}" + } + ] + }, "listpeer": { "type": "command", "name": "ListPeer", diff --git a/src/bigbang/base.h b/src/bigbang/base.h index 9fce5a10..3f7348dc 100644 --- a/src/bigbang/base.h +++ b/src/bigbang/base.h @@ -16,6 +16,7 @@ #include "destination.h" #include "error.h" #include "key.h" +#include "mqdb.h" #include "param.h" #include "peer.h" #include "profile.h" @@ -89,6 +90,7 @@ class IBlockChain : public xengine::IBase virtual Errno AddNewForkContext(const CTransaction& txFork, CForkContext& ctxt) = 0; virtual Errno AddNewBlock(const CBlock& block, CBlockChainUpdate& update) = 0; virtual Errno AddNewOrigin(const CBlock& block, CBlockChainUpdate& update) = 0; + virtual Errno AddNewForkNode(const storage::CForkNode& block) = 0; virtual bool GetProofOfWorkTarget(const uint256& hashPrev, int nAlgo, int& nBits, int64& nReward) = 0; virtual bool GetBlockMintReward(const uint256& hashPrev, int64& nReward) = 0; virtual bool GetBlockLocator(const uint256& hashFork, CBlockLocator& locator, uint256& hashDepth, int nIncStep) = 0; @@ -318,6 +320,7 @@ class IService : public xengine::IBase = 0; /* Util */ virtual bool GetTxSender(const uint256& txid, CAddress& sender) = 0; + virtual bool AddForkNode(const storage::CForkNode& node) = 0; }; class IDataStat : public xengine::IIOModule diff --git a/src/bigbang/blockchain.cpp b/src/bigbang/blockchain.cpp index 4ac6812d..441552d9 100644 --- a/src/bigbang/blockchain.cpp +++ b/src/bigbang/blockchain.cpp @@ -641,6 +641,15 @@ Errno CBlockChain::AddNewOrigin(const CBlock& block, CBlockChainUpdate& update) return OK; } +Errno CBlockChain::AddNewForkNode(const storage::CForkNode& node) +{ + if (!cntrBlock.AddNewForkNode(node)) + { + return FAILED; + } + return OK; +} + // uint320 GetBlockTrust() const // { // if (IsVacant() && vchProof.empty()) diff --git a/src/bigbang/blockchain.h b/src/bigbang/blockchain.h index 7524be95..12fc1a23 100644 --- a/src/bigbang/blockchain.h +++ b/src/bigbang/blockchain.h @@ -44,6 +44,7 @@ class CBlockChain : public IBlockChain Errno AddNewForkContext(const CTransaction& txFork, CForkContext& ctxt) override; Errno AddNewBlock(const CBlock& block, CBlockChainUpdate& update) override; Errno AddNewOrigin(const CBlock& block, CBlockChainUpdate& update) override; + Errno AddNewForkNode(const storage::CForkNode& node) override; bool GetProofOfWorkTarget(const uint256& hashPrev, int nAlgo, int& nBits, int64& nReward) override; bool GetBlockMintReward(const uint256& hashPrev, int64& nReward) override; bool GetBlockLocator(const uint256& hashFork, CBlockLocator& locator, uint256& hashDepth, int nIncStep) override; diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index d4e9fd6a..aeb3136b 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -239,7 +239,7 @@ bool CMQCluster::Publish() conntok->wait(); cout << " ...OK" << endl; - for (int i = 0; i < 10; ++i) + for (int i = 0; i < 1; ++i) { cout << "\nSending message" << to_string(i) << "..." << endl; delivery_action_listener deliveryListener; @@ -295,7 +295,7 @@ void CMQCluster::MqttThreadFunc() //publish topics while (!fAbort) { - boost::system_time const timeout = boost::get_system_time() + boost::posix_time::seconds(30); + boost::system_time const timeout = boost::get_system_time() + boost::posix_time::seconds(300); { boost::unique_lock lock(mutex); while (!fAbort) diff --git a/src/bigbang/rpcmod.cpp b/src/bigbang/rpcmod.cpp index d26c2db5..17e155c2 100644 --- a/src/bigbang/rpcmod.cpp +++ b/src/bigbang/rpcmod.cpp @@ -18,6 +18,7 @@ #include "template/proof.h" #include "template/template.h" #include "version.h" +#include "mqdb.h" using namespace std; using namespace xengine; @@ -265,6 +266,8 @@ CRPCMod::CRPCMod() ("decodetransaction", &CRPCMod::RPCDecodeTransaction) // ("listunspent", &CRPCMod::RPCListUnspent) + // + ("enrollforknode", &CRPCMod::RPCEnrollForkNode) /* Mint */ ("getwork", &CRPCMod::RPCGetWork) // @@ -2395,6 +2398,36 @@ CRPCResultPtr CRPCMod::RPCListUnspent(CRPCParamPtr param) return spResult; } +CRPCResultPtr CRPCMod::RPCEnrollForkNode(rpc::CRPCParamPtr param) +{ + auto spParam = CastParamPtr(param); + std::vector vFork = spParam->vecForks; + std::string id = spParam->strClientid; + + std::vector forks; + for (const auto& i : vFork) + { + uint256 fork; + if (fork.SetHex(i) != i.size()) + { + throw CRPCException(RPC_INVALID_PARAMETER, "Invalid fork hash"); + } + forks.emplace_back(fork); + } + storage::CForkNode node; + node.forkNodeID = std::move(id); + node.vecOwnedForks = std::move(forks); + + if(OK != pService->AddForkNode(node)) + { + throw CRPCException(RPC_INTERNAL_ERROR, "Enroll fork node failed"); + } + + auto spResult = MakeCEnrollForkNodeResultPtr(); + spResult->fRetflag = true; + return spResult; +} + // /* Mint */ CRPCResultPtr CRPCMod::RPCGetWork(CRPCParamPtr param) { diff --git a/src/bigbang/rpcmod.h b/src/bigbang/rpcmod.h index 44a31629..4de85efd 100644 --- a/src/bigbang/rpcmod.h +++ b/src/bigbang/rpcmod.h @@ -125,6 +125,7 @@ class CRPCMod : public xengine::IIOModule, virtual public xengine::CHttpEventLis rpc::CRPCResultPtr RPCMakeTemplate(rpc::CRPCParamPtr param); rpc::CRPCResultPtr RPCDecodeTransaction(rpc::CRPCParamPtr param); rpc::CRPCResultPtr RPCListUnspent(rpc::CRPCParamPtr param); + rpc::CRPCResultPtr RPCEnrollForkNode(rpc::CRPCParamPtr param); /* Mint */ rpc::CRPCResultPtr RPCGetWork(rpc::CRPCParamPtr param); rpc::CRPCResultPtr RPCSubmitWork(rpc::CRPCParamPtr param); diff --git a/src/bigbang/service.cpp b/src/bigbang/service.cpp index 0e796251..aac6bae4 100644 --- a/src/bigbang/service.cpp +++ b/src/bigbang/service.cpp @@ -669,6 +669,11 @@ bool CService::GetTxSender(const uint256& txid, CAddress& sender) return true; } +bool CService::AddForkNode(const storage::CForkNode& node) +{ + return pBlockChain->AddNewForkNode(node); +} + CAddress CService::GetBackSender(const uint256& txid) { CTransaction tx; diff --git a/src/bigbang/service.h b/src/bigbang/service.h index 5925cb40..0cadc13f 100644 --- a/src/bigbang/service.h +++ b/src/bigbang/service.h @@ -6,6 +6,7 @@ #define BIGBANG_SERVICE_H #include "base.h" +#include "mqdb.h" #include "network.h" #include "xengine.h" @@ -80,6 +81,7 @@ class CService : public IService crypto::CKey& keyMint, uint256& hashBlock) override; /* Util */ bool GetTxSender(const uint256& txid, CAddress& sender) override; + bool AddForkNode(const storage::CForkNode& node) override; protected: bool HandleInitialize() override; diff --git a/src/storage/blockbase.cpp b/src/storage/blockbase.cpp index 55c13325..bef62fe9 100644 --- a/src/storage/blockbase.cpp +++ b/src/storage/blockbase.cpp @@ -500,7 +500,7 @@ bool CBlockBase::AddNewForkNode(const CForkNode& forkNode) Error("F", "Failed to addnew forknode in %s", forkNode.forkNodeID.c_str()); return false; } - Log("F", "AddNew forknode,cliID=%s with %n forks", forkNode.forkNodeID.c_str(), + Log("F", "AddNew forknode,cliID=%s with %d forks", forkNode.forkNodeID.c_str(), forkNode.vecOwnedForks.size()); for (const auto& i : forkNode.vecOwnedForks) { diff --git a/src/storage/mqdb.cpp b/src/storage/mqdb.cpp index f44285e7..71df6416 100644 --- a/src/storage/mqdb.cpp +++ b/src/storage/mqdb.cpp @@ -43,7 +43,11 @@ void CForkNodeDB::Deinitialize() bool CForkNodeDB::AddNewForkNode(const CForkNode& cli) { - return Write(cli.forkNodeID, cli.vecOwnedForks, true); //overwrite + vector nodes; + ListForkNode(nodes); + bool ret = Write(cli.forkNodeID, cli.vecOwnedForks, true); //overwrite + ListForkNode(nodes); + return ret; } bool CForkNodeDB::RemoveForkNode(const string& cliID) @@ -70,8 +74,13 @@ bool CForkNodeDB::ListForkNode(std::vector& vCli) { CForkNode node; node.forkNodeID = it.first; + cout << node.forkNodeID << endl; node.vecOwnedForks = it.second; vCli.emplace_back(node); + for (const auto& i : node.vecOwnedForks) + { + cout << "the fork is:" << i.ToString() << endl; + } } return true; } From 688cd6eb0fc0292f403aa763937837a27db660c5 Mon Sep 17 00:00:00 2001 From: OuYun Date: Tue, 10 Mar 2020 15:36:21 +0800 Subject: [PATCH 011/181] add interface for mq storage --- src/bigbang/base.h | 1 + src/bigbang/blockchain.cpp | 10 ++++++++++ src/bigbang/blockchain.h | 1 + src/storage/blockbase.cpp | 19 +++++++++++++++++++ src/storage/blockbase.h | 1 + src/storage/blockdb.cpp | 5 +++++ src/storage/blockdb.h | 1 + 7 files changed, 38 insertions(+) diff --git a/src/bigbang/base.h b/src/bigbang/base.h index 3f7348dc..c67fca6f 100644 --- a/src/bigbang/base.h +++ b/src/bigbang/base.h @@ -91,6 +91,7 @@ class IBlockChain : public xengine::IBase virtual Errno AddNewBlock(const CBlock& block, CBlockChainUpdate& update) = 0; virtual Errno AddNewOrigin(const CBlock& block, CBlockChainUpdate& update) = 0; virtual Errno AddNewForkNode(const storage::CForkNode& block) = 0; + virtual bool ListForkNode(std::vector& nodes) = 0; virtual bool GetProofOfWorkTarget(const uint256& hashPrev, int nAlgo, int& nBits, int64& nReward) = 0; virtual bool GetBlockMintReward(const uint256& hashPrev, int64& nReward) = 0; virtual bool GetBlockLocator(const uint256& hashFork, CBlockLocator& locator, uint256& hashDepth, int nIncStep) = 0; diff --git a/src/bigbang/blockchain.cpp b/src/bigbang/blockchain.cpp index 441552d9..47b9f4c1 100644 --- a/src/bigbang/blockchain.cpp +++ b/src/bigbang/blockchain.cpp @@ -650,6 +650,16 @@ Errno CBlockChain::AddNewForkNode(const storage::CForkNode& node) return OK; } +bool CBlockChain::ListForkNode(std::vector& nodes) +{ + if (!cntrBlock.ListForkNode(nodes)) + { + return false; + } + + return true; +} + // uint320 GetBlockTrust() const // { // if (IsVacant() && vchProof.empty()) diff --git a/src/bigbang/blockchain.h b/src/bigbang/blockchain.h index 12fc1a23..e675770f 100644 --- a/src/bigbang/blockchain.h +++ b/src/bigbang/blockchain.h @@ -45,6 +45,7 @@ class CBlockChain : public IBlockChain Errno AddNewBlock(const CBlock& block, CBlockChainUpdate& update) override; Errno AddNewOrigin(const CBlock& block, CBlockChainUpdate& update) override; Errno AddNewForkNode(const storage::CForkNode& node) override; + bool ListForkNode(std::vector& nodes) override; bool GetProofOfWorkTarget(const uint256& hashPrev, int nAlgo, int& nBits, int64& nReward) override; bool GetBlockMintReward(const uint256& hashPrev, int64& nReward) override; bool GetBlockLocator(const uint256& hashFork, CBlockLocator& locator, uint256& hashDepth, int nIncStep) override; diff --git a/src/storage/blockbase.cpp b/src/storage/blockbase.cpp index bef62fe9..b385e361 100644 --- a/src/storage/blockbase.cpp +++ b/src/storage/blockbase.cpp @@ -509,6 +509,25 @@ bool CBlockBase::AddNewForkNode(const CForkNode& forkNode) return true; } +bool CBlockBase::ListForkNode(std::vector& nodes) +{ + if (!dbBlock.ListForkNode(nodes)) + { + Error("F", "Failed to list forknode"); + return false; + } + Log("F", "List forknode successfully"); + for (const auto& node : nodes) + { + for (const auto& fork : node.vecOwnedForks) + { + Log("F", "forknode client ID [%s] : fork [%s]", + node.forkNodeID.c_str(), fork.ToString().c_str()); + } + } + return true; +} + bool CBlockBase::Retrieve(const uint256& hash, CBlock& block) { block.SetNull(); diff --git a/src/storage/blockbase.h b/src/storage/blockbase.h index e1bf0df7..cb7ad53b 100644 --- a/src/storage/blockbase.h +++ b/src/storage/blockbase.h @@ -234,6 +234,7 @@ class CBlockBase bool AddNew(const uint256& hash, CBlockEx& block, CBlockIndex** ppIndexNew, const uint256& nChainTrust); bool AddNewForkContext(const CForkContext& ctxt); bool AddNewForkNode(const CForkNode& forkNode); + bool ListForkNode(std::vector& nodes); bool Retrieve(const uint256& hash, CBlock& block); bool Retrieve(const CBlockIndex* pIndex, CBlock& block); bool Retrieve(const uint256& hash, CBlockEx& block); diff --git a/src/storage/blockdb.cpp b/src/storage/blockdb.cpp index de164831..5bb4ac6e 100644 --- a/src/storage/blockdb.cpp +++ b/src/storage/blockdb.cpp @@ -256,5 +256,10 @@ bool CBlockDB::AddNewForkNode(const CForkNode& forkNode) return dbForkNode.AddNewForkNode(forkNode); } +bool CBlockDB::ListForkNode(std::vector& nodes) +{ + return dbForkNode.ListForkNode(nodes); +} + } // namespace storage } // namespace bigbang diff --git a/src/storage/blockdb.h b/src/storage/blockdb.h index 84a90f0c..cacdeb99 100644 --- a/src/storage/blockdb.h +++ b/src/storage/blockdb.h @@ -49,6 +49,7 @@ class CBlockDB bool RetrieveEnroll(int height, const std::vector& vBlockRange, std::map& mapEnrollTxPos); bool AddNewForkNode(const CForkNode& forkNode); + bool ListForkNode(std::vector& nodes); protected: bool LoadFork(); From d4d99a3638aaec5b0cea4503f96d4cca4d4a03e4 Mon Sep 17 00:00:00 2001 From: OuYun Date: Tue, 10 Mar 2020 16:29:00 +0800 Subject: [PATCH 012/181] update mq component: - add structures for MQ protocol - refactor behaviour of mq actions according to library nature --- src/bigbang/mqcluster.cpp | 131 +++++++++++++++++++------------------- src/bigbang/mqcluster.h | 97 ++++++++++++++++++++++++++-- 2 files changed, 159 insertions(+), 69 deletions(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index aeb3136b..61ee41a3 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -25,7 +25,8 @@ CMQCluster::CMQCluster(int catNodeIn) pBlockChain(nullptr), pService(nullptr), fAuth(false), - fAbort(false) + fAbort(false), + srvAddr("tcp://localhost:1883") { switch (catNodeIn) { @@ -99,7 +100,13 @@ bool CMQCluster::HandleInvoke() return true; } - if (!ThreadDelayStart(thrMqttClient)) + if (!pBlockChain->ListForkNode(vForkNode)) + { + Log("CMQCluster::HandleInvoke(): list fork node failed"); + return false; + } + + if (!ThreadStart(thrMqttClient)) { return false; } @@ -204,82 +211,77 @@ class delivery_action_listener : public action_listener } }; -bool CMQCluster::ConnectBroker() +bool CMQCluster::ClientAgent(MQ_CLI_ACTION action) { - return true; -} - -bool CMQCluster::Publish() -{ - const std::string SERVER_ADDRESS{ "tcp://localhost:1883" }; const string TOPIC{ "Cluster01/dpos/SyncBlockReq" }; const char* PAYLOAD = "Request for secure main block chain for BBC."; - const char* LWT_PAYLOAD = "Last will and testament."; - - string address = SERVER_ADDRESS; - cout << "Initializing for server '" << address << "'..." << endl; - mqtt::async_client client(address, clientID); + static mqtt::async_client client(srvAddr, clientID); - callback cb; - client.set_callback(cb); - - mqtt::connect_options conopts; - mqtt::message willmsg(TOPIC, LWT_PAYLOAD, 1, true); - mqtt::will_options will(willmsg); - conopts.set_will(will); - - cout << " ...OK" << endl; + static callback cb; try { - cout << "\nConnecting..." << endl; - mqtt::token_ptr conntok = client.connect(conopts); - cout << "Waiting for the connection..." << endl; - conntok->wait(); - cout << " ...OK" << endl; - - for (int i = 0; i < 1; ++i) + static mqtt::token_ptr conntok; + switch (action) { - cout << "\nSending message" << to_string(i) << "..." << endl; - delivery_action_listener deliveryListener; - mqtt::message_ptr pubmsg = mqtt::make_message(TOPIC, PAYLOAD); - client.publish(pubmsg, nullptr, deliveryListener); - - while (!deliveryListener.is_done()) + case MQ_CLI_ACTION::CONN: + { + cout << "Initializing for server '" << srvAddr << "'..." << endl; + client.set_callback(cb); + cout << " ...OK" << endl; + + cout << "\nConnecting..." << endl; + conntok = client.connect(); + cout << "Waiting for the connection..." << endl; + conntok->wait(); + cout << " ...OK" << endl; + break; + } + case MQ_CLI_ACTION::SUB: + { + break; + } + case MQ_CLI_ACTION::PUB: + { + for (int i = 0; i < 3; ++i) { - this_thread::sleep_for(std::chrono::milliseconds(100)); + cout << "\nSending message" << to_string(i) << "..." << endl; + delivery_action_listener deliveryListener; + mqtt::message_ptr pubmsg = mqtt::make_message(TOPIC, PAYLOAD); + client.publish(pubmsg, nullptr, deliveryListener); + + while (!deliveryListener.is_done()) + { + this_thread::sleep_for(std::chrono::milliseconds(100)); + } + cout << "OK" << endl; } - cout << "OK" << endl; + break; + } + case MQ_CLI_ACTION::DISCONN: + { + // Double check that there are no pending tokens + + auto toks = client.get_pending_delivery_tokens(); + if (!toks.empty()) + cout << "Error: There are pending delivery tokens!" << endl; + + // Disconnect + cout << "\nDisconnecting..." << endl; + conntok = client.disconnect(); + conntok->wait(); + cout << " ...OK" << endl; + break; + } } - - // Double check that there are no pending tokens - - auto toks = client.get_pending_delivery_tokens(); - if (!toks.empty()) - cout << "Error: There are pending delivery tokens!" << endl; - - // Disconnect - cout << "\nDisconnecting..." << endl; - conntok = client.disconnect(); - conntok->wait(); - cout << " ...OK" << endl; } catch (const mqtt::exception& exc) { cerr << exc.what() << endl; return false; } - return true; -} - -bool CMQCluster::Subscribe() -{ - return true; -} -bool CMQCluster::DisconnectBroker() -{ return true; } @@ -287,15 +289,16 @@ void CMQCluster::MqttThreadFunc() { Log("entering thread function of MQTT"); //establish connection - ; + ClientAgent(MQ_CLI_ACTION::CONN); //subscribe topics - Subscribe(); + ClientAgent(MQ_CLI_ACTION::SUB); //publish topics while (!fAbort) { - boost::system_time const timeout = boost::get_system_time() + boost::posix_time::seconds(300); + boost::system_time const timeout = boost::get_system_time() + + boost::posix_time::seconds(30); { boost::unique_lock lock(mutex); while (!fAbort) @@ -306,12 +309,12 @@ void CMQCluster::MqttThreadFunc() } } } - Publish(); + ClientAgent(MQ_CLI_ACTION::PUB); Log("thread function of MQTT: go through an iteration"); } //disconnect to broker - ; + ClientAgent(MQ_CLI_ACTION::DISCONN); Log("exiting thread function of MQTT"); } diff --git a/src/bigbang/mqcluster.h b/src/bigbang/mqcluster.h index 9b7e25c6..13260653 100644 --- a/src/bigbang/mqcluster.h +++ b/src/bigbang/mqcluster.h @@ -6,12 +6,88 @@ #define BIGBANG_CMQCLUSTER_H #include "base.h" +#include "mqdb.h" #include "mqevent.h" #include "xengine.h" namespace bigbang { +class CSyncBlockRequest +{ + friend xengine::CStream; + +public: + uint32 ipAddr; + uint8 forkNodeIdLen; + string forkNodeId; + uint8 forkNum; + std::vector forkList; + int32 lastHeight; + uint256 lastHash; + uint32 tsRequest; + int16 nonce; + +protected: + template + void Serialize(xengine::CStream& s, O& opt) + { + s.Serialize(ipAddr, opt); + s.Serialize(forkNodeIdLen, opt); + s.Serialize(forkNodeId, opt); + s.Serialize(forkNum, opt); + s.Serialize(forkList, opt); + s.Serialize(lastHeight, opt); + s.Serialize(lastHash, opt); + s.Serialize(tsRequest, opt); + s.Serialize(nonce, opt); + } +}; + +class CSyncBlockResponse +{ + friend xengine::CStream; + +public: + int32 height; + uint256 hash; + uint8 isBest; + int32 blockSize; + CBlockEx block; + +protected: + template + void Serialize(xengine::CStream& s, O& opt) + { + s.Serialize(height, opt); + s.Serialize(hash, opt); + s.Serialize(isBest, opt); + s.Serialize(blockSize, opt); + s.Serialize(block, opt); + } +}; + +class CRollbackBlock +{ + friend xengine::CStream; + +public: + int32 rbHeight; + uint256 rbHash; + uint8 rbSize; + std::vector hashList; + +protected: + template + void Serialize(xengine::CStream& s, O& opt) + { + s.Serialize(rbHeight, opt); + s.Serialize(rbHash, opt); + s.Serialize(rbSize, opt); + s.Serialize(hashList, opt); + } +}; + class IMQCluster : public xengine::IIOModule, virtual public CMQEventListener { public: @@ -22,7 +98,18 @@ class IMQCluster : public xengine::IIOModule, virtual public CMQEventListener class CMQCluster : public IMQCluster { - enum class NODE_CATEGORY : char {BBCNODE = 0, FORKNODE, DPOSNODE}; + enum class NODE_CATEGORY : char { + BBCNODE = 0, + FORKNODE, + DPOSNODE + }; + enum class MQ_CLI_ACTION : char { + CONN = 0, + SUB, + PUB, + DISCONN + }; + public: CMQCluster(int catNodeIn); ~CMQCluster() = default; @@ -50,15 +137,15 @@ class CMQCluster : public IMQCluster xengine::CThread thrMqttClient; private: - bool ConnectBroker(); - bool Publish(); - bool Subscribe(); - bool DisconnectBroker(); + bool ClientAgent(MQ_CLI_ACTION action); void MqttThreadFunc(); bool fAuth; bool fAbort; + string srvAddr; string clientID; NODE_CATEGORY catNode; + std::vector vForkNode; + std::map mapActiveForkNode; }; } // namespace bigbang From d1d6ff8037959f76a03d571214a8e9e2df0b9d2e Mon Sep 17 00:00:00 2001 From: OuYun Date: Wed, 11 Mar 2020 14:14:49 +0800 Subject: [PATCH 013/181] update mq framework to enable pub/sub simultaniously and establish channels between mq and bbc --- src/bigbang/mqcluster.cpp | 181 ++++++++++++++++++++++++++++---------- src/bigbang/mqcluster.h | 9 ++ 2 files changed, 145 insertions(+), 45 deletions(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 61ee41a3..d8ccc225 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -123,6 +123,8 @@ void CMQCluster::HandleHalt() IIOModule::HandleHalt(); + fAbort = true; + condMQ.notify_all(); if (thrMqttClient.IsRunning()) { @@ -146,68 +148,153 @@ bool CMQCluster::HandleEvent(CEventMQAgreement& eventMqAgreement) return true; } -class callback : public virtual mqtt::callback +bool CMQCluster::LogEvent(const string& info) { -public: - void connected(const string& cause) override - { - cout << "\nConnection connected" << endl; - if (!cause.empty()) - cout << "\tcause: " << cause << endl; - } + cout << "callback to CMQCluster when MQ-EVENT" << info << endl; + Log("CMQCluster::LogMQEvent[%s]", info.c_str()); + return true; +} - void connection_lost(const string& cause) override +class action_listener : public virtual mqtt::iaction_listener +{ + std::string name_; + + void on_failure(const mqtt::token& tok) override { - cout << "\nConnection lost" << endl; - if (!cause.empty()) - cout << "\tcause: " << cause << endl; + std::cout << name_ << " failure"; + if (tok.get_message_id() != 0) + std::cout << " for token: [" << tok.get_message_id() << "]" << std::endl; + std::cout << std::endl; } - void delivery_complete(mqtt::delivery_token_ptr tok) override + void on_success(const mqtt::token& tok) override { - cout << "\tDelivery complete for token: " - << (tok ? tok->get_message_id() : -1) << endl; + std::cout << name_ << " success"; + if (tok.get_message_id() != 0) + std::cout << " for token: [" << tok.get_message_id() << "]" << std::endl; + auto top = tok.get_topics(); + if (top && !top->empty()) + std::cout << "\ttoken topic: '" << (*top)[0] << "', ..." << std::endl; + std::cout << std::endl; } + +public: + action_listener(const std::string& name) + : name_(name) {} }; -class action_listener : public virtual mqtt::iaction_listener +const int RETRY_ATTEMPTS = 3; +class callback : + public virtual mqtt::callback, + public virtual mqtt::iaction_listener { -protected: + mqtt::async_client& cli_; + mqtt::connect_options& connOpts_; + CMQCluster& cluster_; + uint8 retry_; + action_listener subListener_; + +public: + callback(mqtt::async_client& cli, mqtt::connect_options& connOpts, CMQCluster& clusterIn) + : cli_(cli), connOpts_(connOpts), cluster_(clusterIn), retry_(0), subListener_("sublistener") {} + + void reconnect() + { + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + try + { + cli_.connect(connOpts_, nullptr, *this); + cluster_.LogEvent("[reconnect...]"); + } + catch (const mqtt::exception& exc) + { + cerr << "Error: " << exc.what() << std::endl; + exit(1); + } + cluster_.LogEvent("[reconnected]"); + } + void on_failure(const mqtt::token& tok) override { - cout << "\tListener failure for token: " + cout << "\tListener failure for token: [multiple]" << tok.get_message_id() << endl; + cluster_.LogEvent("[on_failure]"); + if (++retry_ > RETRY_ATTEMPTS) + { + exit(1); + } + reconnect(); } void on_success(const mqtt::token& tok) override { - cout << "\tListener success for token: " + cout << "\tListener success for token: [multiple]" << tok.get_message_id() << endl; + cluster_.LogEvent("[on_success]"); } -}; -class delivery_action_listener : public action_listener -{ - atomic done_; + void connected(const string& cause) override + { + cout << "\nConnection success" << endl; + if (!cause.empty()) + { + cout << "\tcause: " << cause << endl; + } + cluster_.LogEvent("[connected]"); + if (CMQCluster::NODE_CATEGORY::FORKNODE == cluster_.catNode) + { + string TOPIC = "Cluster01/" + cluster_.clientID + "/SyncBlockResp"; + cli_.subscribe(TOPIC, CMQCluster::QOS1, nullptr, subListener_); + cout << "\nSubscribing to topic '" << TOPIC << "'\n" + << "\tfor client " << cluster_.clientID + << " using QoS" << CMQCluster::QOS1 << endl; + + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + TOPIC = "Cluster01/DPOSNODE/UpdateBlock"; + cli_.subscribe(TOPIC, CMQCluster::QOS1, nullptr, subListener_); + cout << "\nSubscribing to topic '" << TOPIC << "'\n" + << "\tfor client " << cluster_.clientID + << " using QoS" << CMQCluster::QOS1 << endl; + } + else if (CMQCluster::NODE_CATEGORY::DPOSNODE == cluster_.catNode) + { + const string TOPIC{ "Cluster01/+/SyncBlockReq" }; + cli_.subscribe(TOPIC, CMQCluster::QOS1, nullptr, subListener_); + cout << "\nSubscribing to topic '" << TOPIC << "'\n" + << "\tfor client " << cluster_.clientID + << " using QoS" << CMQCluster::QOS1 << endl; + } + cout << endl; + cluster_.LogEvent("[subscribed]"); + cout << endl; + } - void on_failure(const mqtt::token& tok) override + void connection_lost(const string& cause) override { - action_listener::on_failure(tok); - done_ = true; + cout << "\nConnection lost" << endl; + if (!cause.empty()) + { + cout << "\tcause: " << cause << endl; + } + cluster_.LogEvent("[connection_lost]"); + retry_ = 0; + reconnect(); } - void on_success(const mqtt::token& tok) override + void message_arrived(mqtt::const_message_ptr msg) override { - action_listener::on_success(tok); - done_ = true; + cout << "Message arrived" << endl; + cout << "\ttopic: '" << msg->get_topic() << "'" << endl; + cout << "\tpayload: '" << msg->to_string() << "'\n" + << endl; + cluster_.LogEvent("[message_arrived]"); } -public: - delivery_action_listener() - : done_(false) {} - bool is_done() const + void delivery_complete(mqtt::delivery_token_ptr tok) override { - return done_; + cout << "\tDelivery complete for token: " + << (tok ? tok->get_message_id() : -1) << endl; + cluster_.LogEvent("[delivery_complete]"); } }; @@ -216,13 +303,19 @@ bool CMQCluster::ClientAgent(MQ_CLI_ACTION action) const string TOPIC{ "Cluster01/dpos/SyncBlockReq" }; const char* PAYLOAD = "Request for secure main block chain for BBC."; - static mqtt::async_client client(srvAddr, clientID); - - static callback cb; - try { + static mqtt::async_client client(srvAddr, clientID); + + static mqtt::connect_options connOpts; + connOpts.set_keep_alive_interval(20); + connOpts.set_clean_session(true); + static mqtt::token_ptr conntok; + static mqtt::delivery_token_ptr delitok; + + static callback cb(client, connOpts, *this); + switch (action) { case MQ_CLI_ACTION::CONN: @@ -247,14 +340,12 @@ bool CMQCluster::ClientAgent(MQ_CLI_ACTION action) for (int i = 0; i < 3; ++i) { cout << "\nSending message" << to_string(i) << "..." << endl; - delivery_action_listener deliveryListener; + mqtt::message_ptr pubmsg = mqtt::make_message(TOPIC, PAYLOAD); - client.publish(pubmsg, nullptr, deliveryListener); + pubmsg->set_qos(QOS1); + delitok = client.publish(pubmsg, nullptr, cb); + delitok->wait_for(100); - while (!deliveryListener.is_done()) - { - this_thread::sleep_for(std::chrono::milliseconds(100)); - } cout << "OK" << endl; } break; @@ -298,7 +389,7 @@ void CMQCluster::MqttThreadFunc() while (!fAbort) { boost::system_time const timeout = boost::get_system_time() - + boost::posix_time::seconds(30); + + boost::posix_time::seconds(10); { boost::unique_lock lock(mutex); while (!fAbort) diff --git a/src/bigbang/mqcluster.h b/src/bigbang/mqcluster.h index 13260653..1182a648 100644 --- a/src/bigbang/mqcluster.h +++ b/src/bigbang/mqcluster.h @@ -98,6 +98,8 @@ class IMQCluster : public xengine::IIOModule, virtual public CMQEventListener class CMQCluster : public IMQCluster { + friend class callback; + enum class NODE_CATEGORY : char { BBCNODE = 0, FORKNODE, @@ -109,11 +111,18 @@ class CMQCluster : public IMQCluster PUB, DISCONN }; + enum { + QOS0 = 0, + QOS1, + QOS2 + }; public: CMQCluster(int catNodeIn); ~CMQCluster() = default; + bool LogEvent(const std::string& info); + protected: bool HandleInitialize() override; void HandleDeinitialize() override; From 3bd9965c1d4e5fd36c304057a7b3efd743775a02 Mon Sep 17 00:00:00 2001 From: OuYun Date: Wed, 11 Mar 2020 20:55:39 +0800 Subject: [PATCH 014/181] add message loop procedure for fork node sending main chain block request --- src/bigbang/mqcluster.cpp | 93 ++++++++++++++++++++++++++++++++------- src/bigbang/mqcluster.h | 15 ++++++- src/storage/mqdb.cpp | 5 --- 3 files changed, 91 insertions(+), 22 deletions(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index d8ccc225..0a01abdb 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -100,11 +100,32 @@ bool CMQCluster::HandleInvoke() return true; } - if (!pBlockChain->ListForkNode(vForkNode)) + std::vector nodes; + if (!pBlockChain->ListForkNode(nodes)) { Log("CMQCluster::HandleInvoke(): list fork node failed"); return false; } + for (const auto& node : nodes) + { + mapForkNode.insert(make_pair(node.forkNodeID, node.vecOwnedForks)); + cout << "fork node of MQ: " << node.forkNodeID << endl; + for (const auto& fork : node.vecOwnedForks) + { + cout << "the fork producing subsidiary: " << fork.ToString() << endl; + Log("CMQCluster::HandleInvoke(): list fork node [%s] fork [%s]", + node.forkNodeID.c_str(), fork.ToString().c_str()); + } + } + + if (NODE_CATEGORY::FORKNODE == catNode) + { + topicReqBlk = "Cluster01/" + clientID + "/SyncBlockReq"; + if (mapForkNode.size() != 1 || !PostBlockRequest()) + { + return false; + } + } if (!ThreadStart(thrMqttClient)) { @@ -155,6 +176,48 @@ bool CMQCluster::LogEvent(const string& info) return true; } +bool CMQCluster::PostBlockRequest() +{ + uint256 hash; + int height; + int64 ts; + if (!pBlockChain->GetLastBlock(pCoreProtocol->GetGenesisBlockHash(), hash, height, ts)) + { + return false; + } + + CSyncBlockRequest req; + req.ipAddr = 16777343; //127.0.0.1 + req.ipAddr = 1111638320; + req.forkNodeIdLen = clientID.size(); + req.forkNodeId = clientID; + auto enroll = mapForkNode.begin(); + req.forkNum = (*enroll).second.size(); + req.forkList = (*enroll).second; + req.lastHeight = height; + req.lastHash = hash; + req.tsRequest = ts; + req.nonce = 1; + + CBufferPtr spSS(new CBufStream); + *spSS.get() << req; + + AppendSendQueue(topicReqBlk, spSS); + return true; +} + +bool CMQCluster::AppendSendQueue(const std::string& topic, + CBufferPtr payload) +{ + { + boost::unique_lock lock(mtxSend); + deqSendBuff.emplace_back(make_pair(topic, payload)); + } + condSend.notify_all(); + + return true; +} + class action_listener : public virtual mqtt::iaction_listener { std::string name_; @@ -286,7 +349,7 @@ class callback : cout << "Message arrived" << endl; cout << "\ttopic: '" << msg->get_topic() << "'" << endl; cout << "\tpayload: '" << msg->to_string() << "'\n" - << endl; + << endl; cluster_.LogEvent("[message_arrived]"); } @@ -301,7 +364,7 @@ class callback : bool CMQCluster::ClientAgent(MQ_CLI_ACTION action) { const string TOPIC{ "Cluster01/dpos/SyncBlockReq" }; - const char* PAYLOAD = "Request for secure main block chain for BBC."; + // const char* PAYLOAD = "Request for secure main block chain for BBC."; try { @@ -337,16 +400,19 @@ bool CMQCluster::ClientAgent(MQ_CLI_ACTION action) } case MQ_CLI_ACTION::PUB: { - for (int i = 0; i < 3; ++i) + while (!deqSendBuff.empty()) { - cout << "\nSending message" << to_string(i) << "..." << endl; + pair buf = deqSendBuff.front(); + cout << "\nSending message to [" << buf.first << "]..." << endl; + buf.second->Dump(); - mqtt::message_ptr pubmsg = mqtt::make_message(TOPIC, PAYLOAD); + mqtt::message_ptr pubmsg = mqtt::make_message(buf.first, buf.second->GetData()); pubmsg->set_qos(QOS1); delitok = client.publish(pubmsg, nullptr, cb); delitok->wait_for(100); - cout << "OK" << endl; + + deqSendBuff.pop_front(); } break; } @@ -388,20 +454,15 @@ void CMQCluster::MqttThreadFunc() //publish topics while (!fAbort) { - boost::system_time const timeout = boost::get_system_time() - + boost::posix_time::seconds(10); { - boost::unique_lock lock(mutex); + boost::unique_lock lock(mtxSend); while (!fAbort) { - if (!condMQ.timed_wait(lock, timeout)) - { - break; - } + ClientAgent(MQ_CLI_ACTION::PUB); + Log("thread function of MQTT: go through an iteration"); + condSend.wait(lock); } } - ClientAgent(MQ_CLI_ACTION::PUB); - Log("thread function of MQTT: go through an iteration"); } //disconnect to broker diff --git a/src/bigbang/mqcluster.h b/src/bigbang/mqcluster.h index 1182a648..c0c50d03 100644 --- a/src/bigbang/mqcluster.h +++ b/src/bigbang/mqcluster.h @@ -100,6 +100,8 @@ class CMQCluster : public IMQCluster { friend class callback; + typedef std::shared_ptr CBufferPtr; + enum class NODE_CATEGORY : char { BBCNODE = 0, FORKNODE, @@ -146,15 +148,26 @@ class CMQCluster : public IMQCluster xengine::CThread thrMqttClient; private: + bool PostBlockRequest(); + bool AppendSendQueue(const std::string& topic, CBufferPtr payload); bool ClientAgent(MQ_CLI_ACTION action); void MqttThreadFunc(); bool fAuth; bool fAbort; string srvAddr; string clientID; + string topicReqBlk; NODE_CATEGORY catNode; - std::vector vForkNode; +// std::vector vForkNode; + std::map> mapForkNode; std::map mapActiveForkNode; + + boost::mutex mtxSend; + boost::condition_variable condSend; + std::deque> deqSendBuff; //topic vs. payload + boost::shared_mutex rwRecv; + boost::condition_variable condRecv; + std::deque> deqRecvBuff; //topic vs. payload }; } // namespace bigbang diff --git a/src/storage/mqdb.cpp b/src/storage/mqdb.cpp index 71df6416..4dcd1d78 100644 --- a/src/storage/mqdb.cpp +++ b/src/storage/mqdb.cpp @@ -74,13 +74,8 @@ bool CForkNodeDB::ListForkNode(std::vector& vCli) { CForkNode node; node.forkNodeID = it.first; - cout << node.forkNodeID << endl; node.vecOwnedForks = it.second; vCli.emplace_back(node); - for (const auto& i : node.vecOwnedForks) - { - cout << "the fork is:" << i.ToString() << endl; - } } return true; } From 4dfeef94ac708dcab86b3de88bd917d60e5868aa Mon Sep 17 00:00:00 2001 From: OuYun Date: Thu, 12 Mar 2020 10:51:23 +0800 Subject: [PATCH 015/181] resolve a tricky issue of publishing message pointed to smart pointer without any errors or exception but actually nothing happens caused by paho library using walk around method --- src/bigbang/mqcluster.cpp | 63 +++++++++++++++++++++++++++++++++++---- 1 file changed, 57 insertions(+), 6 deletions(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 0a01abdb..fc414b3e 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -187,7 +187,7 @@ bool CMQCluster::PostBlockRequest() } CSyncBlockRequest req; - req.ipAddr = 16777343; //127.0.0.1 +// req.ipAddr = 16777343; //127.0.0.1 req.ipAddr = 1111638320; req.forkNodeIdLen = clientID.size(); req.forkNodeId = clientID; @@ -400,20 +400,71 @@ bool CMQCluster::ClientAgent(MQ_CLI_ACTION action) } case MQ_CLI_ACTION::PUB: { +/* { + CSyncBlockRequest req; + req.ipAddr = 1111638320; + CBufStream ss; + ss << req; + ss.Dump(); + string topic = "Cluster01/dpos/SyncBlockReq"; + delitok = client.publish(topic, ss.GetData(), ss.GetSize(), QOS1, false, nullptr, cb); + delitok->wait_for(1000); + cout << "_._._OK" << endl; + }*/ while (!deqSendBuff.empty()) { pair buf = deqSendBuff.front(); cout << "\nSending message to [" << buf.first << "]..." << endl; - buf.second->Dump(); +// buf.second->Dump(); - mqtt::message_ptr pubmsg = mqtt::make_message(buf.first, buf.second->GetData()); + mqtt::message_ptr pubmsg = mqtt::make_message(buf.first, buf.second->GetData(), buf.second->GetSize()); pubmsg->set_qos(QOS1); - delitok = client.publish(pubmsg, nullptr, cb); - delitok->wait_for(100); - cout << "OK" << endl; + string s = pubmsg->to_string(); + + if (buf.first == topicReqBlk) + { + CSyncBlockRequest req; + *(buf.second).get() >> req; + req.ipAddr = 1111638320; + CBufStream ss; + ss << req; + ss.Dump(); + string topic = "Cluster01/dpos/SyncBlockReq";//buf.first; + delitok = client.publish(topic, ss.GetData(), ss.GetSize(), QOS1, false, nullptr, cb); + delitok->wait_for(1000); + cout << "_._._OK" << endl; + } + +// mqtt::message msg(buf.first, buf.second->GetData(), buf.second->GetSize(), QOS1, false); +// delitok = client.publish(buf.first, buf.second->GetData(), buf.second->GetSize(), QOS1, false, nullptr, cb); +// delitok = client.publish(pubmsg, nullptr, cb); +// delitok->wait_for(100); +// cout << "___OK" << endl; deqSendBuff.pop_front(); } +/* for (int i = 0; i < 1; ++i) + { + cout << "\nSending message" << to_string(i) << "..." << endl; + + CSyncBlockRequest blkreq; + blkreq.ipAddr = 2037340495; + CBufStream ss; + ss << blkreq; + + ss.Dump(); + + mqtt::message_ptr pubmsg = mqtt::make_message(TOPIC, ss.GetData(), ss.GetSize()); + //mqtt::message_ptr pubmsg = mqtt::make_message(TOPIC, PAYLOAD); + + pubmsg->set_qos(QOS1); + string s = pubmsg->to_string(); + delitok = client.publish(pubmsg, nullptr, cb); + delitok->wait_for(1000); + + cout << "___OK" << endl; + } +*/ break; } case MQ_CLI_ACTION::DISCONN: From fb7734b121cfe49601b120cf672f38d05f5ce6b3 Mon Sep 17 00:00:00 2001 From: OuYun Date: Thu, 12 Mar 2020 11:23:42 +0800 Subject: [PATCH 016/181] clean up code --- src/bigbang/mqcluster.cpp | 67 ++++----------------------------------- 1 file changed, 7 insertions(+), 60 deletions(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index fc414b3e..35d5d52b 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -187,7 +187,7 @@ bool CMQCluster::PostBlockRequest() } CSyncBlockRequest req; -// req.ipAddr = 16777343; //127.0.0.1 + // req.ipAddr = 16777343; //127.0.0.1 req.ipAddr = 1111638320; req.forkNodeIdLen = clientID.size(); req.forkNodeId = clientID; @@ -363,9 +363,6 @@ class callback : bool CMQCluster::ClientAgent(MQ_CLI_ACTION action) { - const string TOPIC{ "Cluster01/dpos/SyncBlockReq" }; - // const char* PAYLOAD = "Request for secure main block chain for BBC."; - try { static mqtt::async_client client(srvAddr, clientID); @@ -400,71 +397,21 @@ bool CMQCluster::ClientAgent(MQ_CLI_ACTION action) } case MQ_CLI_ACTION::PUB: { -/* { - CSyncBlockRequest req; - req.ipAddr = 1111638320; - CBufStream ss; - ss << req; - ss.Dump(); - string topic = "Cluster01/dpos/SyncBlockReq"; - delitok = client.publish(topic, ss.GetData(), ss.GetSize(), QOS1, false, nullptr, cb); - delitok->wait_for(1000); - cout << "_._._OK" << endl; - }*/ while (!deqSendBuff.empty()) { pair buf = deqSendBuff.front(); cout << "\nSending message to [" << buf.first << "]..." << endl; -// buf.second->Dump(); + buf.second->Dump(); - mqtt::message_ptr pubmsg = mqtt::make_message(buf.first, buf.second->GetData(), buf.second->GetSize()); + mqtt::message_ptr pubmsg = mqtt::make_message( + buf.first, buf.second->GetData(), buf.second->GetSize()); pubmsg->set_qos(QOS1); - string s = pubmsg->to_string(); - - if (buf.first == topicReqBlk) - { - CSyncBlockRequest req; - *(buf.second).get() >> req; - req.ipAddr = 1111638320; - CBufStream ss; - ss << req; - ss.Dump(); - string topic = "Cluster01/dpos/SyncBlockReq";//buf.first; - delitok = client.publish(topic, ss.GetData(), ss.GetSize(), QOS1, false, nullptr, cb); - delitok->wait_for(1000); - cout << "_._._OK" << endl; - } - -// mqtt::message msg(buf.first, buf.second->GetData(), buf.second->GetSize(), QOS1, false); -// delitok = client.publish(buf.first, buf.second->GetData(), buf.second->GetSize(), QOS1, false, nullptr, cb); -// delitok = client.publish(pubmsg, nullptr, cb); -// delitok->wait_for(100); -// cout << "___OK" << endl; - - deqSendBuff.pop_front(); - } -/* for (int i = 0; i < 1; ++i) - { - cout << "\nSending message" << to_string(i) << "..." << endl; - - CSyncBlockRequest blkreq; - blkreq.ipAddr = 2037340495; - CBufStream ss; - ss << blkreq; - - ss.Dump(); - - mqtt::message_ptr pubmsg = mqtt::make_message(TOPIC, ss.GetData(), ss.GetSize()); - //mqtt::message_ptr pubmsg = mqtt::make_message(TOPIC, PAYLOAD); - - pubmsg->set_qos(QOS1); - string s = pubmsg->to_string(); delitok = client.publish(pubmsg, nullptr, cb); - delitok->wait_for(1000); + delitok->wait_for(100); + cout << "_._._OK" << endl; - cout << "___OK" << endl; + deqSendBuff.pop_front(); } -*/ break; } case MQ_CLI_ACTION::DISCONN: From 83051b0194b82254e60485b2dcd39547598cc02a Mon Sep 17 00:00:00 2001 From: OuYun Date: Thu, 12 Mar 2020 18:14:32 +0800 Subject: [PATCH 017/181] add procedures of receiving message from broker for both dpos node and fork nodes --- src/bigbang/mqcluster.cpp | 191 ++++++++++++++++++++++++++++++++++++-- src/bigbang/mqcluster.h | 6 ++ src/storage/mqdb.h | 4 + 3 files changed, 192 insertions(+), 9 deletions(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 35d5d52b..d416f4bd 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -75,6 +75,12 @@ bool CMQCluster::HandleInitialize() return false; } + if (!GetObject("dispatcher", pDispatcher)) + { + Error("Failed to request dispatcher\n"); + return false; + } + if (!GetObject("service", pService)) { Error("Failed to request service"); @@ -89,6 +95,7 @@ void CMQCluster::HandleDeinitialize() { pCoreProtocol = nullptr; pBlockChain = nullptr; + pDispatcher = nullptr; pService = nullptr; } @@ -218,6 +225,166 @@ bool CMQCluster::AppendSendQueue(const std::string& topic, return true; } +void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) +{ + payload.Dump(); + + switch (catNode) + { + case NODE_CATEGORY::BBCNODE: + Error("CMQCluster::OnReceiveMessage(): bbc node should not come here!"); + return; + case NODE_CATEGORY::FORKNODE: + { + if (topicRbBlk != topic) + {//respond to request block of main chain + //unpack payload + CSyncBlockResponse resp; + try + { + payload >> resp; + } + catch (exception& e) + { + StdError(__PRETTY_FUNCTION__, e.what()); + Error("CMQCluster::OnReceiveMessage(): failed to unpack respond msg"); + return; + } + + //check if this msg is just for me + //if (topicReqBlk != clientID) + + //validate this coming block + if (!pCoreProtocol->ValidateBlock(resp.block)) + { + Error("CMQCluster::OnReceiveMessage(): failed to validate block"); + return; + } + + //notify to add new block + Errno err = pDispatcher->AddNewBlock(resp.block); + if (err != OK) + { + Error("CMQCluster::OnReceiveMessage(): failed to validate block (%d) : %s\n", err, ErrorString(err)); + return; + } + lastHeightResp = resp.height; + + //iterate to retrieve next one + if (!PostBlockRequest()) + { + Error("CMQCluster::OnReceiveMessage(): failed to post request"); + return; + } + } + else + {//roll back blocks on main chain + //unpack payload + CRollbackBlock rb; + try + { + payload >> rb; + } + catch (exception& e) + { + StdError(__PRETTY_FUNCTION__, e.what()); + Error("CMQCluster::OnReceiveMessage(): failed to unpack rollback msg"); + return; + } + + } + break; + } + case NODE_CATEGORY::DPOSNODE: + { + //unpack payload + CSyncBlockRequest req; + try + { + payload >> req; + } + catch (exception& e) + { + StdError(__PRETTY_FUNCTION__, e.what()); + Error("CMQCluster::OnReceiveMessage(): failed to unpack request msg"); + return; + } + + //check if requesting fork node has been enrolled + auto node = mapForkNode.find(req.forkNodeId); + if (node == mapForkNode.end()) + { + Error("CMQCluster::OnReceiveMessage(): requesting fork node has not enrolled yet"); + return; + } + //check if requesting fork node matches the corresponding one enrolled + if (node->second.size() != req.forkNum) + { + Error("CMQCluster::OnReceiveMessage(): requesting fork node number does not match"); + return; + } + for (const auto& fork : req.forkList) + { + auto pos = find(node->second.begin(), node->second.end(), fork); + if (pos == node->second.end()) + { + Error("CMQCluster::OnReceiveMessage(): requesting fork node detailed forks does not match"); + return; + } + } + + //add this requesting fork node to active list + mapActiveForkNode[req.ipAddr] = storage::CForkNode(req.forkNodeId, req.forkList); + + //processing request from fork node + //check height and hash are matched + uint256 hash; + if (!pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), req.lastHeight, hash)) + { + Error("CMQCluster::OnReceiveMessage(): failed to get checking height and hash match"); + return; + } + if (hash != req.lastHash) + { + Error("CMQCluster::OnReceiveMessage(): height and hash do not match"); + return; + } + if (!pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), req.lastHeight + 1, hash)) + { + Error("CMQCluster::OnReceiveMessage(): failed to get next block hash"); + return; + } + CBlockEx block; + if (!pBlockChain->GetBlockEx(hash, block)) + { + Error("CMQCluster::OnReceiveMessage(): failed to get next block"); + return; + } + + //reply block requested + CSyncBlockResponse resp; + resp.height = req.lastHeight + 1; + resp.hash = hash; + resp.isBest = resp.height < pBlockChain->GetBlockCount(pCoreProtocol->GetGenesisBlockHash()) + ? 0 + : 1; + resp.blockSize = xengine::GetSerializeSize(block); + resp.block = move(block); + + CBufferPtr spSS(new CBufStream); + *spSS.get() << resp; + string topicRsp = "Cluster01/" + req.forkNodeId + "/SyncBlockResp"; + { + boost::unique_lock lock(mtxSend); + deqSendBuff.emplace_back(make_pair(topicRsp, spSS)); + } + condSend.notify_all(); + + break; + } + } +} + class action_listener : public virtual mqtt::iaction_listener { std::string name_; @@ -272,6 +439,7 @@ class callback : catch (const mqtt::exception& exc) { cerr << "Error: " << exc.what() << std::endl; + cluster_.LogEvent("[on_reconnect_ERROR!]"); exit(1); } cluster_.LogEvent("[reconnected]"); @@ -284,6 +452,7 @@ class callback : cluster_.LogEvent("[on_failure]"); if (++retry_ > RETRY_ATTEMPTS) { + cluster_.LogEvent("[on_retry_FAILURE!]"); exit(1); } reconnect(); @@ -306,24 +475,24 @@ class callback : cluster_.LogEvent("[connected]"); if (CMQCluster::NODE_CATEGORY::FORKNODE == cluster_.catNode) { - string TOPIC = "Cluster01/" + cluster_.clientID + "/SyncBlockResp"; - cli_.subscribe(TOPIC, CMQCluster::QOS1, nullptr, subListener_); - cout << "\nSubscribing to topic '" << TOPIC << "'\n" + cluster_.topicRespBlk = "Cluster01/" + cluster_.clientID + "/SyncBlockResp"; + cli_.subscribe(cluster_.topicRespBlk, CMQCluster::QOS1, nullptr, subListener_); + cout << "\nSubscribing to topic '" << cluster_.topicRespBlk << "'\n" << "\tfor client " << cluster_.clientID << " using QoS" << CMQCluster::QOS1 << endl; std::this_thread::sleep_for(std::chrono::milliseconds(100)); - TOPIC = "Cluster01/DPOSNODE/UpdateBlock"; - cli_.subscribe(TOPIC, CMQCluster::QOS1, nullptr, subListener_); - cout << "\nSubscribing to topic '" << TOPIC << "'\n" + cluster_.topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; + cli_.subscribe(cluster_.topicRbBlk, CMQCluster::QOS1, nullptr, subListener_); + cout << "\nSubscribing to topic '" << cluster_.topicRbBlk << "'\n" << "\tfor client " << cluster_.clientID << " using QoS" << CMQCluster::QOS1 << endl; } else if (CMQCluster::NODE_CATEGORY::DPOSNODE == cluster_.catNode) { - const string TOPIC{ "Cluster01/+/SyncBlockReq" }; - cli_.subscribe(TOPIC, CMQCluster::QOS1, nullptr, subListener_); - cout << "\nSubscribing to topic '" << TOPIC << "'\n" + cluster_.topicReqBlk = "Cluster01/+/SyncBlockReq"; + cli_.subscribe(cluster_.topicReqBlk, CMQCluster::QOS1, nullptr, subListener_); + cout << "\nSubscribing to topic '" << cluster_.topicReqBlk << "'\n" << "\tfor client " << cluster_.clientID << " using QoS" << CMQCluster::QOS1 << endl; } @@ -351,6 +520,9 @@ class callback : cout << "\tpayload: '" << msg->to_string() << "'\n" << endl; cluster_.LogEvent("[message_arrived]"); + xengine::CBufStream ss; + ss.Write((const char*)&msg->get_payload()[0], msg->get_payload().size()); + cluster_.OnReceiveMessage(msg->get_topic(), ss); } void delivery_complete(mqtt::delivery_token_ptr tok) override @@ -406,6 +578,7 @@ bool CMQCluster::ClientAgent(MQ_CLI_ACTION action) mqtt::message_ptr pubmsg = mqtt::make_message( buf.first, buf.second->GetData(), buf.second->GetSize()); pubmsg->set_qos(QOS1); + pubmsg->set_retained(mqtt::message::DFLT_RETAINED); delitok = client.publish(pubmsg, nullptr, cb); delitok->wait_for(100); cout << "_._._OK" << endl; diff --git a/src/bigbang/mqcluster.h b/src/bigbang/mqcluster.h index c0c50d03..f8903851 100644 --- a/src/bigbang/mqcluster.h +++ b/src/bigbang/mqcluster.h @@ -141,6 +141,7 @@ class CMQCluster : public IMQCluster mutable boost::shared_mutex rwAccess; ICoreProtocol* pCoreProtocol; IBlockChain* pBlockChain; + IDispatcher* pDispatcher; IService* pService; boost::mutex mutex; @@ -150,6 +151,7 @@ class CMQCluster : public IMQCluster private: bool PostBlockRequest(); bool AppendSendQueue(const std::string& topic, CBufferPtr payload); + void OnReceiveMessage(const std::string& topic, CBufStream& payload); bool ClientAgent(MQ_CLI_ACTION action); void MqttThreadFunc(); bool fAuth; @@ -157,6 +159,8 @@ class CMQCluster : public IMQCluster string srvAddr; string clientID; string topicReqBlk; + string topicRespBlk; + string topicRbBlk; NODE_CATEGORY catNode; // std::vector vForkNode; std::map> mapForkNode; @@ -168,6 +172,8 @@ class CMQCluster : public IMQCluster boost::shared_mutex rwRecv; boost::condition_variable condRecv; std::deque> deqRecvBuff; //topic vs. payload + + std::atomic lastHeightResp; }; } // namespace bigbang diff --git a/src/storage/mqdb.h b/src/storage/mqdb.h index f5a2508e..f3f5c9d4 100644 --- a/src/storage/mqdb.h +++ b/src/storage/mqdb.h @@ -21,6 +21,10 @@ class CForkNode std::string forkNodeID; std::vector vecOwnedForks; +public: + CForkNode(std::string id = std::string(), std::vector forks = std::vector()) + : forkNodeID(id), vecOwnedForks(forks) {} + protected: template void Serialize(xengine::CStream& s, O& opt) From 8d4be7da3621ca848401d507d6cf98fe94ad3fbe Mon Sep 17 00:00:00 2001 From: OuYun Date: Thu, 12 Mar 2020 23:58:00 +0800 Subject: [PATCH 018/181] correct local variable shadow --- src/storage/blockbase.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/storage/blockbase.cpp b/src/storage/blockbase.cpp index b385e361..7fde756d 100644 --- a/src/storage/blockbase.cpp +++ b/src/storage/blockbase.cpp @@ -2026,9 +2026,9 @@ bool CBlockBase::GetTxNewIndex(CBlockView& view, CBlockIndex* pIndexNew, vector< CVarInt var(block.vtx.size()); nOffset += ss.GetSerializeSize(var); - for (int i = 0; i < block.vtx.size(); i++) + for (int j = 0; j < block.vtx.size(); j++) { - CTransaction& tx = block.vtx[i]; + CTransaction& tx = block.vtx[j]; uint256 txid = tx.GetHash(); CTxIndex txIndex(nHeight, pIndex->nFile, nOffset); vTxNew.push_back(make_pair(txid, txIndex)); From 2f914420b29beac28b4d4ee478a64f05a66da372 Mon Sep 17 00:00:00 2001 From: OuYun Date: Fri, 13 Mar 2020 13:39:06 +0800 Subject: [PATCH 019/181] reduce payload size by removing block context --- src/bigbang/mqcluster.cpp | 4 ++-- src/bigbang/mqcluster.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index d416f4bd..3de2e8a9 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -354,8 +354,8 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) Error("CMQCluster::OnReceiveMessage(): failed to get next block hash"); return; } - CBlockEx block; - if (!pBlockChain->GetBlockEx(hash, block)) + CBlock block; + if (!pBlockChain->GetBlock(hash, block)) { Error("CMQCluster::OnReceiveMessage(): failed to get next block"); return; diff --git a/src/bigbang/mqcluster.h b/src/bigbang/mqcluster.h index f8903851..0d805055 100644 --- a/src/bigbang/mqcluster.h +++ b/src/bigbang/mqcluster.h @@ -53,7 +53,7 @@ class CSyncBlockResponse uint256 hash; uint8 isBest; int32 blockSize; - CBlockEx block; + CBlock block; protected: template From 07f85bb2bad22a4cb97c7204fba75571287561e4 Mon Sep 17 00:00:00 2001 From: OuYun Date: Fri, 13 Mar 2020 22:01:14 +0800 Subject: [PATCH 020/181] update rollback function for mq component --- script/template/options.json | 16 ++-- src/bigbang/dispatcher.cpp | 38 +++++++++ src/bigbang/dispatcher.h | 3 + src/bigbang/mqcluster.cpp | 157 +++++++++++++++++++++++++++++++---- src/bigbang/mqcluster.h | 6 +- src/bigbang/mqevent.h | 17 ++-- src/bigbang/struct.h | 9 ++ 7 files changed, 209 insertions(+), 37 deletions(-) diff --git a/script/template/options.json b/script/template/options.json index f41f5472..6d82ca57 100755 --- a/script/template/options.json +++ b/script/template/options.json @@ -8,6 +8,14 @@ "format": "-testnet", "desc": "Use the test network" }, + { + "name": "nCatOfNode", + "type": "int", + "opt": "catofnode", + "default": 0, + "format": "-catofnode=", + "desc": "Identify node category: 0 - BBCNODE; 1 - FORKNODE; 2 - DPOSNODE, 0 by default" + }, { "name": "fWallet", "type": "bool", @@ -56,14 +64,6 @@ "format": "-blocknotify", "desc": "Execute command when the best block changes (%s in cmd is replaced by block hash)" }, - { - "name": "nCatOfNode", - "type": "int", - "opt": "catofnode", - "default": 0, - "format": "-catofnode=", - "desc": "Identify node category: 0 - BBCNODE; 1 - FORKNODE; 2 - DPOSNODE, 0 by default" - }, { "name": "nLogFileSize", "type": "int", diff --git a/src/bigbang/dispatcher.cpp b/src/bigbang/dispatcher.cpp index cfe88c73..1d4782fc 100644 --- a/src/bigbang/dispatcher.cpp +++ b/src/bigbang/dispatcher.cpp @@ -32,6 +32,7 @@ CDispatcher::CDispatcher() pNetChannel = nullptr; // pDelegatedChannel = nullptr; pDataStat = nullptr; + pMqcluster = nullptr; } CDispatcher::~CDispatcher() @@ -106,6 +107,17 @@ bool CDispatcher::HandleInitialize() return false; } strCmd = dynamic_cast(Config())->strBlocknotify; + + nNodeCat = dynamic_cast(Config())->nCatOfNode; + if (2 == nNodeCat) + { + if (!GetObject("mqcluster", pMqcluster)) + { + Error("Failed to request mqcluster"); + return false; + } + } + return true; } @@ -122,6 +134,7 @@ void CDispatcher::HandleDeinitialize() pNetChannel = nullptr; // pDelegatedChannel = nullptr; pDataStat = nullptr; + pMqcluster = nullptr; } bool CDispatcher::HandleInvoke() @@ -348,6 +361,31 @@ void CDispatcher::UpdatePrimaryBlock(const CBlock& block, const CBlockChainUpdat } // SyncForkHeight(updateBlockChain.nLastBlockHeight); + + if (2 == nNodeCat && !updateBlockChain.vBlockRemove.empty()) + { + CEventMQChainUpdate* pMqChainUpdate = new CEventMQChainUpdate(0); + if (pMqChainUpdate != nullptr) + { + pMqChainUpdate->data.shortLen = updateBlockChain.vBlockRemove.size(); + pMqChainUpdate->data.vShort.reserve(updateBlockChain.vBlockRemove.size()); + bool fFirst = true; + for (const auto& rb : updateBlockChain.vBlockRemove) + { + if (fFirst) + { + pMqChainUpdate->data.triHeight = rb.GetBlockHeight() - 1; + pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), + pMqChainUpdate->data.triHeight, + pMqChainUpdate->data.triHash); + fFirst = false; + } + pMqChainUpdate->data.vShort.push_back(rb.GetHash()); + } + + pMqcluster->PostEvent(pMqChainUpdate); + } + } } void CDispatcher::ActivateFork(const uint256& hashFork, const uint64& nNonce) diff --git a/src/bigbang/dispatcher.h b/src/bigbang/dispatcher.h index 58888df2..41af0fd2 100644 --- a/src/bigbang/dispatcher.h +++ b/src/bigbang/dispatcher.h @@ -7,6 +7,7 @@ #include "base.h" #include "peernet.h" +#include "mqcluster.h" namespace bigbang { @@ -46,6 +47,8 @@ class CDispatcher : public IDispatcher // network::IDelegatedChannel* pDelegatedChannel; IDataStat* pDataStat; std::string strCmd; + int nNodeCat; + IMQCluster* pMqcluster; }; } // namespace bigbang diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 3de2e8a9..87acd2f1 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -26,7 +26,8 @@ CMQCluster::CMQCluster(int catNodeIn) pService(nullptr), fAuth(false), fAbort(false), - srvAddr("tcp://localhost:1883") + srvAddr("tcp://localhost:1883"), + nReqBlkTimerID(0) { switch (catNodeIn) { @@ -128,8 +129,14 @@ bool CMQCluster::HandleInvoke() if (NODE_CATEGORY::FORKNODE == catNode) { topicReqBlk = "Cluster01/" + clientID + "/SyncBlockReq"; - if (mapForkNode.size() != 1 || !PostBlockRequest()) + if (mapForkNode.size() != 1) { + Error("CMQCluster::HandleHalt(): fork node should have one single enrollment"); + return false; + } + if (!PostBlockRequest(-1)) + { + Error("CMQCluster::HandleHalt(): failed to post requesting block"); return false; } } @@ -166,8 +173,31 @@ bool CMQCluster::HandleEvent(CEventMQSyncBlock& eventMqSyncBlock) return true; } -bool CMQCluster::HandleEvent(CEventMQUpdateBlock& eventMqUpdateBlock) +bool CMQCluster::HandleEvent(CEventMQChainUpdate& eventMqUpdateChain) { + CMqRollbackUpdate& update = eventMqUpdateChain.data; + + if (catNode != NODE_CATEGORY::DPOSNODE) + { + Error("CMQCluster::HandleEvent(): only dpos fork should receive this kind of event"); + return false; + } + + CRollbackBlock rbc; + rbc.rbHeight = update.triHeight; + rbc.rbHash = update.triHash; + rbc.rbSize = update.shortLen; + rbc.hashList = update.vShort; + + CBufferPtr spRBC(new CBufStream); + *spRBC.get() << rbc; + + { + boost::unique_lock lock(mtxSend); + deqSendBuff.emplace_back(make_pair(topicRbBlk, spRBC)); + } + condSend.notify_all(); + return true; } @@ -183,14 +213,33 @@ bool CMQCluster::LogEvent(const string& info) return true; } -bool CMQCluster::PostBlockRequest() +bool CMQCluster::PostBlockRequest(int syncHeight) { + if (mapForkNode.empty() || mapForkNode.size() > 1) + { + Error("CMQCluster::PostBlockRequest(): enrollment is incorrect for fork node"); + return false; + } + uint256 hash; int height; - int64 ts; - if (!pBlockChain->GetLastBlock(pCoreProtocol->GetGenesisBlockHash(), hash, height, ts)) + if (-1 == syncHeight) { - return false; + int64 ts; + if (!pBlockChain->GetLastBlock(pCoreProtocol->GetGenesisBlockHash(), hash, height, ts)) + { + Error("CMQCluster::PostBlockRequest(): failed to get last block"); + return false; + } + } + else + { + if (!pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), syncHeight, hash)) + { + Error("CMQCluster::PostBlockRequest(): failed to get specific block"); + return false; + } + height = syncHeight; } CSyncBlockRequest req; @@ -203,7 +252,7 @@ bool CMQCluster::PostBlockRequest() req.forkList = (*enroll).second; req.lastHeight = height; req.lastHash = hash; - req.tsRequest = ts; + req.tsRequest = GetTime(); req.nonce = 1; CBufferPtr spSS(new CBufStream); @@ -225,6 +274,19 @@ bool CMQCluster::AppendSendQueue(const std::string& topic, return true; } +void CMQCluster::RequestBlockTimerFunc(uint32 nTimer) +{ + if (nReqBlkTimerID == nTimer) + { + if (!PostBlockRequest(-1)) + { + Error("CMQCluster::RequestBlockTimerFunc(): failed to post request"); + } + nReqBlkTimerID = SetTimer(1000 * 60, + boost::bind(&CMQCluster::RequestBlockTimerFunc, this, _1)); + } +} + void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) { payload.Dump(); @@ -237,7 +299,7 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) case NODE_CATEGORY::FORKNODE: { if (topicRbBlk != topic) - {//respond to request block of main chain + { //respond to request block of main chain //unpack payload CSyncBlockResponse resp; try @@ -271,14 +333,28 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) lastHeightResp = resp.height; //iterate to retrieve next one - if (!PostBlockRequest()) + + if (resp.isBest) + {//when reaching best height, send request by timer + nReqBlkTimerID = SetTimer(1000 * 60, + boost::bind(&CMQCluster::RequestBlockTimerFunc, this, _1)); + } + else { - Error("CMQCluster::OnReceiveMessage(): failed to post request"); - return; + if (0 != nReqBlkTimerID) + { + CancelTimer(nReqBlkTimerID); + nReqBlkTimerID = 0; + } + if (!PostBlockRequest(lastHeightResp)) + { + Error("CMQCluster::OnReceiveMessage(): failed to post request"); + return; + } } } else - {//roll back blocks on main chain + { //roll back blocks on main chain //unpack payload CRollbackBlock rb; try @@ -292,7 +368,56 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) return; } + if (rb.rbHeight < lastHeightResp) + { + //check hard fork point + uint256 hash; + if (!pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), rb.rbHeight, hash)) + { + Error("CMQCluster::OnReceiveMessage(): failed to get hard fork block hash"); + return; + } + bool fMatch = true; + int nSync = rb.rbHeight - 1; + if (hash != rb.rbHash) + { + Log("CMQCluster::OnReceiveMessage(): hard fork block hash does not match"); + fMatch = false; + } + + //check blocks in rollback + if (fMatch) + { + for (int i = 0; i < rb.rbSize; ++i) + { + if (!pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), rb.rbHeight + i + 1, hash)) + { + Error("CMQCluster::OnReceiveMessage(): failed to get rollback block hash"); + return; + } + if (hash == rb.hashList[i]) + { + Log("CMQCluster::OnReceiveMessage(): fork node has not been rolled back yet"); + fMatch = false; + break; + } + ++nSync; + } + } + + //re-request from hard fork point to sync the best chain + if (!fMatch) + { + lastHeightResp = nSync; + if (!PostBlockRequest(lastHeightResp)) + { + Error("CMQCluster::OnReceiveMessage(): failed to post request"); + return; + } + } + } } + break; } case NODE_CATEGORY::DPOSNODE: @@ -439,8 +564,7 @@ class callback : catch (const mqtt::exception& exc) { cerr << "Error: " << exc.what() << std::endl; - cluster_.LogEvent("[on_reconnect_ERROR!]"); - exit(1); + cluster_.Error("[MQTT_reconnect_ERROR!]"); } cluster_.LogEvent("[reconnected]"); } @@ -452,8 +576,7 @@ class callback : cluster_.LogEvent("[on_failure]"); if (++retry_ > RETRY_ATTEMPTS) { - cluster_.LogEvent("[on_retry_FAILURE!]"); - exit(1); + cluster_.Error("[MQTT_retry_to_reconnect_FAILURE!]"); } reconnect(); } diff --git a/src/bigbang/mqcluster.h b/src/bigbang/mqcluster.h index 0d805055..4c729a78 100644 --- a/src/bigbang/mqcluster.h +++ b/src/bigbang/mqcluster.h @@ -132,7 +132,7 @@ class CMQCluster : public IMQCluster void HandleHalt() override; bool HandleEvent(CEventMQSyncBlock& eventMqSyncBlock) override; - bool HandleEvent(CEventMQUpdateBlock& eventMqUpdateBlock) override; + bool HandleEvent(CEventMQChainUpdate& eventMqUpdateChain) override; bool HandleEvent(CEventMQAgreement& eventMqAgreement) override; bool IsAuthenticated() override; @@ -149,8 +149,9 @@ class CMQCluster : public IMQCluster xengine::CThread thrMqttClient; private: - bool PostBlockRequest(); + bool PostBlockRequest(int syncHeight = -1); bool AppendSendQueue(const std::string& topic, CBufferPtr payload); + void RequestBlockTimerFunc(uint32 nTimer); void OnReceiveMessage(const std::string& topic, CBufStream& payload); bool ClientAgent(MQ_CLI_ACTION action); void MqttThreadFunc(); @@ -174,6 +175,7 @@ class CMQCluster : public IMQCluster std::deque> deqRecvBuff; //topic vs. payload std::atomic lastHeightResp; + uint32 nReqBlkTimerID; }; } // namespace bigbang diff --git a/src/bigbang/mqevent.h b/src/bigbang/mqevent.h index 6feaf914..c1895fe8 100644 --- a/src/bigbang/mqevent.h +++ b/src/bigbang/mqevent.h @@ -17,27 +17,24 @@ enum { EVENT_MQ_BASE = EVENT_USER_BASE, EVENT_MQ_SYNCBLOCK, - EVENT_MQ_UPDATEBLOCK, + EVENT_MQ_UPDATEBLOCK, //dpos node deliver rollback block EVENT_MQ_AGREEMENT }; class CMQEventListener; +#define TYPE_MQEVENT(type, body) \ + CEventCategory -#define TYPE_MQEVENT(type, body, res) \ - CEventCategory - - - -typedef TYPE_MQEVENT(EVENT_MQ_SYNCBLOCK, int, std::string) CEventMQSyncBlock; -typedef TYPE_MQEVENT(EVENT_MQ_UPDATEBLOCK, CDelegateAgreement, int) CEventMQUpdateBlock; -typedef TYPE_MQEVENT(EVENT_MQ_AGREEMENT, CDelegateAgreement, int) CEventMQAgreement; +typedef TYPE_MQEVENT(EVENT_MQ_SYNCBLOCK, std::string) CEventMQSyncBlock; +typedef TYPE_MQEVENT(EVENT_MQ_UPDATEBLOCK, CMqRollbackUpdate) CEventMQChainUpdate; +typedef TYPE_MQEVENT(EVENT_MQ_AGREEMENT, CDelegateAgreement) CEventMQAgreement; class CMQEventListener : virtual public CEventListener { public: virtual ~CMQEventListener() {} DECLARE_EVENTHANDLER(CEventMQSyncBlock); - DECLARE_EVENTHANDLER(CEventMQUpdateBlock); + DECLARE_EVENTHANDLER(CEventMQChainUpdate); DECLARE_EVENTHANDLER(CEventMQAgreement); }; diff --git a/src/bigbang/struct.h b/src/bigbang/struct.h index 0cc22f3d..62258de3 100644 --- a/src/bigbang/struct.h +++ b/src/bigbang/struct.h @@ -256,6 +256,15 @@ class CBlockMakerUpdate uint16 nMintType; }; +class CMqRollbackUpdate +{ +public: + int32 triHeight; + uint256 triHash; + uint8 shortLen; + std::vector vShort; +}; + /* Net Channel */ class CPeerKnownTx { From 3a1f4ce173d51aaa176abcf38178d3040a81d836 Mon Sep 17 00:00:00 2001 From: OuYun Date: Mon, 16 Mar 2020 17:42:31 +0800 Subject: [PATCH 021/181] refactor code --- src/bigbang/mqcluster.cpp | 88 +++++++++++++++++++-------------------- src/bigbang/mqcluster.h | 2 +- 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 87acd2f1..14b09e3d 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -510,13 +510,13 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) } } -class action_listener : public virtual mqtt::iaction_listener +class CActionListener : public virtual mqtt::iaction_listener { - std::string name_; + std::string strName; void on_failure(const mqtt::token& tok) override { - std::cout << name_ << " failure"; + std::cout << strName << " failure"; if (tok.get_message_id() != 0) std::cout << " for token: [" << tok.get_message_id() << "]" << std::endl; std::cout << std::endl; @@ -524,7 +524,7 @@ class action_listener : public virtual mqtt::iaction_listener void on_success(const mqtt::token& tok) override { - std::cout << name_ << " success"; + std::cout << strName << " success"; if (tok.get_message_id() != 0) std::cout << " for token: [" << tok.get_message_id() << "]" << std::endl; auto top = tok.get_topics(); @@ -534,49 +534,49 @@ class action_listener : public virtual mqtt::iaction_listener } public: - action_listener(const std::string& name) - : name_(name) {} + CActionListener(const std::string& name) + : strName(name) {} }; const int RETRY_ATTEMPTS = 3; -class callback : +class CMQCallback : public virtual mqtt::callback, public virtual mqtt::iaction_listener { - mqtt::async_client& cli_; - mqtt::connect_options& connOpts_; - CMQCluster& cluster_; - uint8 retry_; - action_listener subListener_; + mqtt::async_client& asynCli; + mqtt::connect_options& connOpts; + CMQCluster& mqCluster; + uint8 nRetry; + CActionListener subListener; public: - callback(mqtt::async_client& cli, mqtt::connect_options& connOpts, CMQCluster& clusterIn) - : cli_(cli), connOpts_(connOpts), cluster_(clusterIn), retry_(0), subListener_("sublistener") {} + CMQCallback(mqtt::async_client& cli, mqtt::connect_options& connOpts, CMQCluster& clusterIn) + : asynCli(cli), connOpts(connOpts), mqCluster(clusterIn), nRetry(0), subListener("sublistener") {} void reconnect() { std::this_thread::sleep_for(std::chrono::milliseconds(300)); try { - cli_.connect(connOpts_, nullptr, *this); - cluster_.LogEvent("[reconnect...]"); + asynCli.connect(connOpts, nullptr, *this); + mqCluster.LogEvent("[reconnect...]"); } catch (const mqtt::exception& exc) { cerr << "Error: " << exc.what() << std::endl; - cluster_.Error("[MQTT_reconnect_ERROR!]"); + mqCluster.Error("[MQTT_reconnect_ERROR!]"); } - cluster_.LogEvent("[reconnected]"); + mqCluster.LogEvent("[reconnected]"); } void on_failure(const mqtt::token& tok) override { cout << "\tListener failure for token: [multiple]" << tok.get_message_id() << endl; - cluster_.LogEvent("[on_failure]"); - if (++retry_ > RETRY_ATTEMPTS) + mqCluster.LogEvent("[on_failure]"); + if (++nRetry > RETRY_ATTEMPTS) { - cluster_.Error("[MQTT_retry_to_reconnect_FAILURE!]"); + mqCluster.Error("[MQTT_retry_to_reconnect_FAILURE!]"); } reconnect(); } @@ -585,7 +585,7 @@ class callback : { cout << "\tListener success for token: [multiple]" << tok.get_message_id() << endl; - cluster_.LogEvent("[on_success]"); + mqCluster.LogEvent("[on_success]"); } void connected(const string& cause) override @@ -595,32 +595,32 @@ class callback : { cout << "\tcause: " << cause << endl; } - cluster_.LogEvent("[connected]"); - if (CMQCluster::NODE_CATEGORY::FORKNODE == cluster_.catNode) + mqCluster.LogEvent("[connected]"); + if (CMQCluster::NODE_CATEGORY::FORKNODE == mqCluster.catNode) { - cluster_.topicRespBlk = "Cluster01/" + cluster_.clientID + "/SyncBlockResp"; - cli_.subscribe(cluster_.topicRespBlk, CMQCluster::QOS1, nullptr, subListener_); - cout << "\nSubscribing to topic '" << cluster_.topicRespBlk << "'\n" - << "\tfor client " << cluster_.clientID + mqCluster.topicRespBlk = "Cluster01/" + mqCluster.clientID + "/SyncBlockResp"; + asynCli.subscribe(mqCluster.topicRespBlk, CMQCluster::QOS1, nullptr, subListener); + cout << "\nSubscribing to topic '" << mqCluster.topicRespBlk << "'\n" + << "\tfor client " << mqCluster.clientID << " using QoS" << CMQCluster::QOS1 << endl; std::this_thread::sleep_for(std::chrono::milliseconds(100)); - cluster_.topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; - cli_.subscribe(cluster_.topicRbBlk, CMQCluster::QOS1, nullptr, subListener_); - cout << "\nSubscribing to topic '" << cluster_.topicRbBlk << "'\n" - << "\tfor client " << cluster_.clientID + mqCluster.topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; + asynCli.subscribe(mqCluster.topicRbBlk, CMQCluster::QOS1, nullptr, subListener); + cout << "\nSubscribing to topic '" << mqCluster.topicRbBlk << "'\n" + << "\tfor client " << mqCluster.clientID << " using QoS" << CMQCluster::QOS1 << endl; } - else if (CMQCluster::NODE_CATEGORY::DPOSNODE == cluster_.catNode) + else if (CMQCluster::NODE_CATEGORY::DPOSNODE == mqCluster.catNode) { - cluster_.topicReqBlk = "Cluster01/+/SyncBlockReq"; - cli_.subscribe(cluster_.topicReqBlk, CMQCluster::QOS1, nullptr, subListener_); - cout << "\nSubscribing to topic '" << cluster_.topicReqBlk << "'\n" - << "\tfor client " << cluster_.clientID + mqCluster.topicReqBlk = "Cluster01/+/SyncBlockReq"; + asynCli.subscribe(mqCluster.topicReqBlk, CMQCluster::QOS1, nullptr, subListener); + cout << "\nSubscribing to topic '" << mqCluster.topicReqBlk << "'\n" + << "\tfor client " << mqCluster.clientID << " using QoS" << CMQCluster::QOS1 << endl; } cout << endl; - cluster_.LogEvent("[subscribed]"); + mqCluster.LogEvent("[subscribed]"); cout << endl; } @@ -631,8 +631,8 @@ class callback : { cout << "\tcause: " << cause << endl; } - cluster_.LogEvent("[connection_lost]"); - retry_ = 0; + mqCluster.LogEvent("[connection_lost]"); + nRetry = 0; reconnect(); } @@ -642,17 +642,17 @@ class callback : cout << "\ttopic: '" << msg->get_topic() << "'" << endl; cout << "\tpayload: '" << msg->to_string() << "'\n" << endl; - cluster_.LogEvent("[message_arrived]"); + mqCluster.LogEvent("[message_arrived]"); xengine::CBufStream ss; ss.Write((const char*)&msg->get_payload()[0], msg->get_payload().size()); - cluster_.OnReceiveMessage(msg->get_topic(), ss); + mqCluster.OnReceiveMessage(msg->get_topic(), ss); } void delivery_complete(mqtt::delivery_token_ptr tok) override { cout << "\tDelivery complete for token: " << (tok ? tok->get_message_id() : -1) << endl; - cluster_.LogEvent("[delivery_complete]"); + mqCluster.LogEvent("[delivery_complete]"); } }; @@ -669,7 +669,7 @@ bool CMQCluster::ClientAgent(MQ_CLI_ACTION action) static mqtt::token_ptr conntok; static mqtt::delivery_token_ptr delitok; - static callback cb(client, connOpts, *this); + static CMQCallback cb(client, connOpts, *this); switch (action) { diff --git a/src/bigbang/mqcluster.h b/src/bigbang/mqcluster.h index 4c729a78..7705d445 100644 --- a/src/bigbang/mqcluster.h +++ b/src/bigbang/mqcluster.h @@ -98,7 +98,7 @@ class IMQCluster : public xengine::IIOModule, virtual public CMQEventListener class CMQCluster : public IMQCluster { - friend class callback; + friend class CMQCallback; typedef std::shared_ptr CBufferPtr; From c5c9442fc685141dd7f52d1e6532e72bfd17a82d Mon Sep 17 00:00:00 2001 From: OuYun Date: Tue, 17 Mar 2020 11:44:05 +0800 Subject: [PATCH 022/181] resolve build issue for install script --- INSTALL.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/INSTALL.sh b/INSTALL.sh index 46eefd34..ae0343f1 100755 --- a/INSTALL.sh +++ b/INSTALL.sh @@ -19,6 +19,8 @@ else flag="-DCMAKE_BUILD_TYPE=Release" fi +flag+=" -DPAHO_BUILD_STATIC=TRUE -DPAHO_BUILD_SHARED=FALSE" + cmake .. $flag if [ $? -ne 0 ]; then cd $origin_path From 67fc8a932cb21e40e66f5a2f7fbd96cbf41729fc Mon Sep 17 00:00:00 2001 From: OuYun Date: Tue, 17 Mar 2020 17:30:09 +0800 Subject: [PATCH 023/181] optimise startup process of mq component --- src/bigbang/mqcluster.cpp | 91 ++++++++++++++++++++++++++++++++------- src/bigbang/mqcluster.h | 2 + src/bigbang/netchn.h | 9 ++-- 3 files changed, 83 insertions(+), 19 deletions(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 14b09e3d..964860b6 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -60,7 +60,7 @@ bool CMQCluster::HandleInitialize() } else if (NODE_CATEGORY::BBCNODE == catNode) { - Log("CMQCluster::HandleInitialize(): bbc node so go passby"); + Log("CMQCluster::HandleInitialize(): bbc node so bypass"); return true; } @@ -104,7 +104,7 @@ bool CMQCluster::HandleInvoke() { if (NODE_CATEGORY::BBCNODE == catNode) { - Log("CMQCluster::HandleInvoke(): bbc node so go passby"); + Log("CMQCluster::HandleInvoke(): bbc node so bypass"); return true; } @@ -117,11 +117,18 @@ bool CMQCluster::HandleInvoke() for (const auto& node : nodes) { mapForkNode.insert(make_pair(node.forkNodeID, node.vecOwnedForks)); - cout << "fork node of MQ: " << node.forkNodeID << endl; + if (1 == node.vecOwnedForks.size() + && node.vecOwnedForks[0] == pCoreProtocol->GetGenesisBlockHash()) + { + Log("dpos node of MQ: [%s]", node.forkNodeID.c_str()); + } + else + { + Log("fork node of MQ: [%s]", node.forkNodeID.c_str()); + } for (const auto& fork : node.vecOwnedForks) { - cout << "the fork producing subsidiary: " << fork.ToString() << endl; - Log("CMQCluster::HandleInvoke(): list fork node [%s] fork [%s]", + Log("CMQCluster::HandleInvoke(): list fork/dpos node [%s] with fork [%s]", node.forkNodeID.c_str(), fork.ToString().c_str()); } } @@ -129,16 +136,53 @@ bool CMQCluster::HandleInvoke() if (NODE_CATEGORY::FORKNODE == catNode) { topicReqBlk = "Cluster01/" + clientID + "/SyncBlockReq"; - if (mapForkNode.size() != 1) + if (mapForkNode.size() == 0) { - Error("CMQCluster::HandleHalt(): fork node should have one single enrollment"); - return false; + Log("CMQCluster::HandleInvoke(): this fork node has not enrolled " + "itself to dpos node yet[%d]", + mapForkNode.size()); } - if (!PostBlockRequest(-1)) + if (mapForkNode.size() > 1) { - Error("CMQCluster::HandleHalt(): failed to post requesting block"); + Error("CMQCluster::HandleInvoke(): fork node should have one " + "single enrollment[%d]", + mapForkNode.size()); return false; } + if (1 == mapForkNode.size()) + { + clientID = mapForkNode.begin()->first; + topicRespBlk = "Cluster01/" + clientID + "/SyncBlockResp"; + topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; + Log("CMQCluster::HandleInvoke(): fork node clientid [%s] with topics " + "[%s] \n [%s]:", clientID.c_str(), + topicRespBlk.c_str(), topicRbBlk.c_str()); + for (const auto& fork : mapForkNode.begin()->second) + { + Log("CMQCluster::HandleInvoke(): fork [%s] intended to be produced " + "by this node [%s]:", fork.ToString().c_str(), clientID.c_str()); + } + + if (!PostBlockRequest(-1)) + { + Error("CMQCluster::HandleInvoke(): failed to post requesting block"); + return false; + } + } + } + else if (NODE_CATEGORY::DPOSNODE == catNode) + { + for (const auto& node : mapForkNode) + { + if (1 == node.second.size() + && node.second[0] == pCoreProtocol->GetGenesisBlockHash()) + { //dpos node + clientID = node.first; + topicReqBlk = "Cluster01/+/SyncBlockReq"; + Log("CMQCluster::HandleInvoke(): dpos node clientid [%s] with topic [%s]", + clientID.c_str(), topicReqBlk.c_str()); + } + } } if (!ThreadStart(thrMqttClient)) @@ -215,9 +259,15 @@ bool CMQCluster::LogEvent(const string& info) bool CMQCluster::PostBlockRequest(int syncHeight) { - if (mapForkNode.empty() || mapForkNode.size() > 1) + if (mapForkNode.empty()) { - Error("CMQCluster::PostBlockRequest(): enrollment is incorrect for fork node"); + Log("CMQCluster::PostBlockRequest(): enrollment is empty for this fork node"); + return true; + } + + if (mapForkNode.size() > 1) + { + Error("CMQCluster::PostBlockRequest(): enrollment is incorrect for this fork node"); return false; } @@ -335,7 +385,7 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) //iterate to retrieve next one if (resp.isBest) - {//when reaching best height, send request by timer + { //when reaching best height, send request by timer nReqBlkTimerID = SetTimer(1000 * 60, boost::bind(&CMQCluster::RequestBlockTimerFunc, this, _1)); } @@ -598,14 +648,12 @@ class CMQCallback : mqCluster.LogEvent("[connected]"); if (CMQCluster::NODE_CATEGORY::FORKNODE == mqCluster.catNode) { - mqCluster.topicRespBlk = "Cluster01/" + mqCluster.clientID + "/SyncBlockResp"; asynCli.subscribe(mqCluster.topicRespBlk, CMQCluster::QOS1, nullptr, subListener); cout << "\nSubscribing to topic '" << mqCluster.topicRespBlk << "'\n" << "\tfor client " << mqCluster.clientID << " using QoS" << CMQCluster::QOS1 << endl; std::this_thread::sleep_for(std::chrono::milliseconds(100)); - mqCluster.topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; asynCli.subscribe(mqCluster.topicRbBlk, CMQCluster::QOS1, nullptr, subListener); cout << "\nSubscribing to topic '" << mqCluster.topicRbBlk << "'\n" << "\tfor client " << mqCluster.clientID @@ -613,7 +661,6 @@ class CMQCallback : } else if (CMQCluster::NODE_CATEGORY::DPOSNODE == mqCluster.catNode) { - mqCluster.topicReqBlk = "Cluster01/+/SyncBlockReq"; asynCli.subscribe(mqCluster.topicReqBlk, CMQCluster::QOS1, nullptr, subListener); cout << "\nSubscribing to topic '" << mqCluster.topicReqBlk << "'\n" << "\tfor client " << mqCluster.clientID @@ -739,6 +786,18 @@ bool CMQCluster::ClientAgent(MQ_CLI_ACTION action) void CMQCluster::MqttThreadFunc() { Log("entering thread function of MQTT"); + + //wait for supernode itself status available + if (!fAbort) + { + boost::unique_lock lock(mtxStatus); + while (mapForkNode.empty()) + { + Log("there is no enrollment info, waiting for it coming..."); + condStatus.wait(lock); + } + } + //establish connection ClientAgent(MQ_CLI_ACTION::CONN); diff --git a/src/bigbang/mqcluster.h b/src/bigbang/mqcluster.h index 7705d445..bf0afc4b 100644 --- a/src/bigbang/mqcluster.h +++ b/src/bigbang/mqcluster.h @@ -164,6 +164,8 @@ class CMQCluster : public IMQCluster string topicRbBlk; NODE_CATEGORY catNode; // std::vector vForkNode; + boost::mutex mtxStatus; + boost::condition_variable condStatus; std::map> mapForkNode; std::map mapActiveForkNode; diff --git a/src/bigbang/netchn.h b/src/bigbang/netchn.h index a057520a..40f40b3f 100644 --- a/src/bigbang/netchn.h +++ b/src/bigbang/netchn.h @@ -18,8 +18,10 @@ class CNetChannelPeer { public: CNetChannelPeerFork() - : fSynchronized(false), nSynTxInvStatus(SYNTXINV_STATUS_INIT), nSynTxInvSendTime(0), nSynTxInvRecvTime(0), nPrevGetDataTime(0), - nSingleSynTxInvCount(network::CInv::MAX_INV_COUNT / 2), fWaitGetTxComplete(false), nCacheSynTxCount(NETCHANNEL_KNOWNINV_MAXCOUNT) + : fSynchronized(false), nSynTxInvStatus(SYNTXINV_STATUS_INIT), + nSynTxInvSendTime(0), nSynTxInvRecvTime(0), nPrevGetDataTime(0), + nSingleSynTxInvCount(network::CInv::MAX_INV_COUNT / 2), + fWaitGetTxComplete(false), nCacheSynTxCount(NETCHANNEL_KNOWNINV_MAXCOUNT) { } enum @@ -94,7 +96,8 @@ class CNetChannelPeer } break; case SYNTXINV_STATUS_WAIT_PEER_COMPLETE: - if (nCurTime - nPrevGetDataTime >= SYNTXINV_GETDATA_TIMEOUT || nCurTime - nSynTxInvRecvTime >= SYNTXINV_COMPLETE_TIMEOUT) + if (nCurTime - nPrevGetDataTime >= SYNTXINV_GETDATA_TIMEOUT + || nCurTime - nSynTxInvRecvTime >= SYNTXINV_COMPLETE_TIMEOUT) { InitTxInvSynData(); nSingleSynTxInvCount = network::CInv::MIN_INV_COUNT; From 7a25592ae94e0fdc6c9bb5cd5896796f31c4fc8b Mon Sep 17 00:00:00 2001 From: OuYun Date: Tue, 17 Mar 2020 18:03:31 +0800 Subject: [PATCH 024/181] rename enrolling rpc command and update its descriptions --- script/template/rpc.json | 14 +++++++------- src/bigbang/rpcmod.cpp | 8 ++++---- src/bigbang/rpcmod.h | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/script/template/rpc.json b/script/template/rpc.json index 0ce18180..f3ce439d 100755 --- a/script/template/rpc.json +++ b/script/template/rpc.json @@ -638,17 +638,17 @@ } ] }, - "enrollforknode": { + "enrollsupernode": { "type": "command", - "name": "EnrollForkNode", - "desc": "Enroll fork node info for MQ communication on both sides of dpos and fork nodes.", + "name": "EnrollSuperNode", + "desc": "Enroll supernode info for MQ communication on both sides of dpos and fork nodes.", "request": { "type": "object", "content": { "clientid" : { "type": "string", - "desc": "client id" + "desc": "client id for dpos node or fork nodes" }, "forks" : { @@ -657,7 +657,7 @@ "fork" : { "type" : "string", - "dest" : "fork hash" + "dest" : "hash of fork produced by super nodes, dpos node gets main chain" } } } @@ -670,11 +670,11 @@ }, "example": [ { - "request": "bigbang-cli enrollforknode forknode01 '[\"575f2041770496489120bb102d9dd55f5e75b0c4aa528d5762b92b59acd6d939\",\"6236d780f9f743707d57b3feb19d21c8a867577f5e83c163774222bb7ef8d8cb\",\"7c6a6aba05cec77a998c19649ee1fa0e29c7b5246d0e3a6501ee1d4d81dd73ea\"]'", + "request": "bigbang-cli enrollsupernode forknode01 '[\"575f2041770496489120bb102d9dd55f5e75b0c4aa528d5762b92b59acd6d939\",\"6236d780f9f743707d57b3feb19d21c8a867577f5e83c163774222bb7ef8d8cb\",\"7c6a6aba05cec77a998c19649ee1fa0e29c7b5246d0e3a6501ee1d4d81dd73ea\"]'", "response": "true" }, { - "request": "curl -d '{\"id\":3,\"method\":\"enrollforknode\",\"jsonrpc\":\"2.0\",\"params\":{[\"575f2041770496489120bb102d9dd55f5e75b0c4aa528d5762b92b59acd6d939\",\"6236d780f9f743707d57b3feb19d21c8a867577f5e83c163774222bb7ef8d8cb\",\"7c6a6aba05cec77a998c19649ee1fa0e29c7b5246d0e3a6501ee1d4d81dd73ea\"]}}' http://127.0.0.1:9902", + "request": "curl -d '{\"id\":3,\"method\":\"enrollsupernode\",\"jsonrpc\":\"2.0\",\"params\":{[\"575f2041770496489120bb102d9dd55f5e75b0c4aa528d5762b92b59acd6d939\",\"6236d780f9f743707d57b3feb19d21c8a867577f5e83c163774222bb7ef8d8cb\",\"7c6a6aba05cec77a998c19649ee1fa0e29c7b5246d0e3a6501ee1d4d81dd73ea\"]}}' http://127.0.0.1:9902", "response": "{\"id\":3,\"jsonrpc\":\"2.0\",\"result\":true}" } ] diff --git a/src/bigbang/rpcmod.cpp b/src/bigbang/rpcmod.cpp index 17e155c2..8986b642 100644 --- a/src/bigbang/rpcmod.cpp +++ b/src/bigbang/rpcmod.cpp @@ -267,7 +267,7 @@ CRPCMod::CRPCMod() // ("listunspent", &CRPCMod::RPCListUnspent) // - ("enrollforknode", &CRPCMod::RPCEnrollForkNode) + ("enrollsupernode", &CRPCMod::RPCEnrollSuperNode) /* Mint */ ("getwork", &CRPCMod::RPCGetWork) // @@ -2398,9 +2398,9 @@ CRPCResultPtr CRPCMod::RPCListUnspent(CRPCParamPtr param) return spResult; } -CRPCResultPtr CRPCMod::RPCEnrollForkNode(rpc::CRPCParamPtr param) +CRPCResultPtr CRPCMod::RPCEnrollSuperNode(rpc::CRPCParamPtr param) { - auto spParam = CastParamPtr(param); + auto spParam = CastParamPtr(param); std::vector vFork = spParam->vecForks; std::string id = spParam->strClientid; @@ -2423,7 +2423,7 @@ CRPCResultPtr CRPCMod::RPCEnrollForkNode(rpc::CRPCParamPtr param) throw CRPCException(RPC_INTERNAL_ERROR, "Enroll fork node failed"); } - auto spResult = MakeCEnrollForkNodeResultPtr(); + auto spResult = MakeCEnrollSuperNodeResultPtr(); spResult->fRetflag = true; return spResult; } diff --git a/src/bigbang/rpcmod.h b/src/bigbang/rpcmod.h index 4de85efd..de024417 100644 --- a/src/bigbang/rpcmod.h +++ b/src/bigbang/rpcmod.h @@ -125,7 +125,7 @@ class CRPCMod : public xengine::IIOModule, virtual public xengine::CHttpEventLis rpc::CRPCResultPtr RPCMakeTemplate(rpc::CRPCParamPtr param); rpc::CRPCResultPtr RPCDecodeTransaction(rpc::CRPCParamPtr param); rpc::CRPCResultPtr RPCListUnspent(rpc::CRPCParamPtr param); - rpc::CRPCResultPtr RPCEnrollForkNode(rpc::CRPCParamPtr param); + rpc::CRPCResultPtr RPCEnrollSuperNode(rpc::CRPCParamPtr param); /* Mint */ rpc::CRPCResultPtr RPCGetWork(rpc::CRPCParamPtr param); rpc::CRPCResultPtr RPCSubmitWork(rpc::CRPCParamPtr param); From bff2a41344f75887b22abc6e0dc48a31dfba2b66 Mon Sep 17 00:00:00 2001 From: OuYun Date: Tue, 17 Mar 2020 18:53:36 +0800 Subject: [PATCH 025/181] refactor name as supernode --- src/bigbang/base.h | 6 ++--- src/bigbang/blockchain.cpp | 8 +++---- src/bigbang/blockchain.h | 4 ++-- src/bigbang/mqcluster.cpp | 46 +++++++++++++++++++------------------- src/bigbang/mqcluster.h | 5 ++--- src/bigbang/rpcmod.cpp | 6 ++--- src/bigbang/service.cpp | 4 ++-- src/bigbang/service.h | 2 +- src/bigbang/struct.h | 7 ++++++ src/storage/blockbase.cpp | 26 ++++++++++----------- src/storage/blockbase.h | 4 ++-- src/storage/blockdb.cpp | 14 ++++++------ src/storage/blockdb.h | 6 ++--- src/storage/mqdb.cpp | 34 ++++++++++++++-------------- src/storage/mqdb.h | 24 ++++++++++---------- 15 files changed, 101 insertions(+), 95 deletions(-) diff --git a/src/bigbang/base.h b/src/bigbang/base.h index c67fca6f..8eae2782 100644 --- a/src/bigbang/base.h +++ b/src/bigbang/base.h @@ -90,8 +90,8 @@ class IBlockChain : public xengine::IBase virtual Errno AddNewForkContext(const CTransaction& txFork, CForkContext& ctxt) = 0; virtual Errno AddNewBlock(const CBlock& block, CBlockChainUpdate& update) = 0; virtual Errno AddNewOrigin(const CBlock& block, CBlockChainUpdate& update) = 0; - virtual Errno AddNewForkNode(const storage::CForkNode& block) = 0; - virtual bool ListForkNode(std::vector& nodes) = 0; + virtual Errno AddNewSuperNode(const storage::CSuperNode& node) = 0; + virtual bool ListSuperNode(std::vector& nodes) = 0; virtual bool GetProofOfWorkTarget(const uint256& hashPrev, int nAlgo, int& nBits, int64& nReward) = 0; virtual bool GetBlockMintReward(const uint256& hashPrev, int64& nReward) = 0; virtual bool GetBlockLocator(const uint256& hashFork, CBlockLocator& locator, uint256& hashDepth, int nIncStep) = 0; @@ -321,7 +321,7 @@ class IService : public xengine::IBase = 0; /* Util */ virtual bool GetTxSender(const uint256& txid, CAddress& sender) = 0; - virtual bool AddForkNode(const storage::CForkNode& node) = 0; + virtual bool AddSuperNode(const storage::CSuperNode& node) = 0; }; class IDataStat : public xengine::IIOModule diff --git a/src/bigbang/blockchain.cpp b/src/bigbang/blockchain.cpp index 47b9f4c1..bae121ce 100644 --- a/src/bigbang/blockchain.cpp +++ b/src/bigbang/blockchain.cpp @@ -641,18 +641,18 @@ Errno CBlockChain::AddNewOrigin(const CBlock& block, CBlockChainUpdate& update) return OK; } -Errno CBlockChain::AddNewForkNode(const storage::CForkNode& node) +Errno CBlockChain::AddNewSuperNode(const storage::CSuperNode& node) { - if (!cntrBlock.AddNewForkNode(node)) + if (!cntrBlock.AddNewSuperNode(node)) { return FAILED; } return OK; } -bool CBlockChain::ListForkNode(std::vector& nodes) +bool CBlockChain::ListSuperNode(std::vector& nodes) { - if (!cntrBlock.ListForkNode(nodes)) + if (!cntrBlock.ListSuperNode(nodes)) { return false; } diff --git a/src/bigbang/blockchain.h b/src/bigbang/blockchain.h index e675770f..d3b4ff37 100644 --- a/src/bigbang/blockchain.h +++ b/src/bigbang/blockchain.h @@ -44,8 +44,8 @@ class CBlockChain : public IBlockChain Errno AddNewForkContext(const CTransaction& txFork, CForkContext& ctxt) override; Errno AddNewBlock(const CBlock& block, CBlockChainUpdate& update) override; Errno AddNewOrigin(const CBlock& block, CBlockChainUpdate& update) override; - Errno AddNewForkNode(const storage::CForkNode& node) override; - bool ListForkNode(std::vector& nodes) override; + Errno AddNewSuperNode(const storage::CSuperNode& node) override; + bool ListSuperNode(std::vector& nodes) override; bool GetProofOfWorkTarget(const uint256& hashPrev, int nAlgo, int& nBits, int64& nReward) override; bool GetBlockMintReward(const uint256& hashPrev, int64& nReward) override; bool GetBlockLocator(const uint256& hashFork, CBlockLocator& locator, uint256& hashDepth, int nIncStep) override; diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 964860b6..b90d263b 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -108,56 +108,56 @@ bool CMQCluster::HandleInvoke() return true; } - std::vector nodes; - if (!pBlockChain->ListForkNode(nodes)) + std::vector nodes; + if (!pBlockChain->ListSuperNode(nodes)) { - Log("CMQCluster::HandleInvoke(): list fork node failed"); + Log("CMQCluster::HandleInvoke(): list super node failed"); return false; } for (const auto& node : nodes) { - mapForkNode.insert(make_pair(node.forkNodeID, node.vecOwnedForks)); + mapSuperNode.insert(make_pair(node.superNodeID, node.vecOwnedForks)); if (1 == node.vecOwnedForks.size() && node.vecOwnedForks[0] == pCoreProtocol->GetGenesisBlockHash()) { - Log("dpos node of MQ: [%s]", node.forkNodeID.c_str()); + Log("dpos node of MQ: [%s]", node.superNodeID.c_str()); } else { - Log("fork node of MQ: [%s]", node.forkNodeID.c_str()); + Log("fork node of MQ: [%s]", node.superNodeID.c_str()); } for (const auto& fork : node.vecOwnedForks) { Log("CMQCluster::HandleInvoke(): list fork/dpos node [%s] with fork [%s]", - node.forkNodeID.c_str(), fork.ToString().c_str()); + node.superNodeID.c_str(), fork.ToString().c_str()); } } if (NODE_CATEGORY::FORKNODE == catNode) { topicReqBlk = "Cluster01/" + clientID + "/SyncBlockReq"; - if (mapForkNode.size() == 0) + if (mapSuperNode.size() == 0) { Log("CMQCluster::HandleInvoke(): this fork node has not enrolled " "itself to dpos node yet[%d]", - mapForkNode.size()); + mapSuperNode.size()); } - if (mapForkNode.size() > 1) + if (mapSuperNode.size() > 1) { Error("CMQCluster::HandleInvoke(): fork node should have one " "single enrollment[%d]", - mapForkNode.size()); + mapSuperNode.size()); return false; } - if (1 == mapForkNode.size()) + if (1 == mapSuperNode.size()) { - clientID = mapForkNode.begin()->first; + clientID = mapSuperNode.begin()->first; topicRespBlk = "Cluster01/" + clientID + "/SyncBlockResp"; topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; Log("CMQCluster::HandleInvoke(): fork node clientid [%s] with topics " "[%s] \n [%s]:", clientID.c_str(), topicRespBlk.c_str(), topicRbBlk.c_str()); - for (const auto& fork : mapForkNode.begin()->second) + for (const auto& fork : mapSuperNode.begin()->second) { Log("CMQCluster::HandleInvoke(): fork [%s] intended to be produced " "by this node [%s]:", fork.ToString().c_str(), clientID.c_str()); @@ -172,7 +172,7 @@ bool CMQCluster::HandleInvoke() } else if (NODE_CATEGORY::DPOSNODE == catNode) { - for (const auto& node : mapForkNode) + for (const auto& node : mapSuperNode) { if (1 == node.second.size() && node.second[0] == pCoreProtocol->GetGenesisBlockHash()) @@ -223,7 +223,7 @@ bool CMQCluster::HandleEvent(CEventMQChainUpdate& eventMqUpdateChain) if (catNode != NODE_CATEGORY::DPOSNODE) { - Error("CMQCluster::HandleEvent(): only dpos fork should receive this kind of event"); + Error("CMQCluster::HandleEvent(): only dpos node should receive this kind of event"); return false; } @@ -259,13 +259,13 @@ bool CMQCluster::LogEvent(const string& info) bool CMQCluster::PostBlockRequest(int syncHeight) { - if (mapForkNode.empty()) + if (mapSuperNode.empty()) { Log("CMQCluster::PostBlockRequest(): enrollment is empty for this fork node"); return true; } - if (mapForkNode.size() > 1) + if (mapSuperNode.size() > 1) { Error("CMQCluster::PostBlockRequest(): enrollment is incorrect for this fork node"); return false; @@ -297,7 +297,7 @@ bool CMQCluster::PostBlockRequest(int syncHeight) req.ipAddr = 1111638320; req.forkNodeIdLen = clientID.size(); req.forkNodeId = clientID; - auto enroll = mapForkNode.begin(); + auto enroll = mapSuperNode.begin(); req.forkNum = (*enroll).second.size(); req.forkList = (*enroll).second; req.lastHeight = height; @@ -486,8 +486,8 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) } //check if requesting fork node has been enrolled - auto node = mapForkNode.find(req.forkNodeId); - if (node == mapForkNode.end()) + auto node = mapSuperNode.find(req.forkNodeId); + if (node == mapSuperNode.end()) { Error("CMQCluster::OnReceiveMessage(): requesting fork node has not enrolled yet"); return; @@ -509,7 +509,7 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) } //add this requesting fork node to active list - mapActiveForkNode[req.ipAddr] = storage::CForkNode(req.forkNodeId, req.forkList); + mapActiveSuperNode[req.ipAddr] = storage::CSuperNode(req.forkNodeId, req.forkList); //processing request from fork node //check height and hash are matched @@ -791,7 +791,7 @@ void CMQCluster::MqttThreadFunc() if (!fAbort) { boost::unique_lock lock(mtxStatus); - while (mapForkNode.empty()) + while (mapSuperNode.empty()) { Log("there is no enrollment info, waiting for it coming..."); condStatus.wait(lock); diff --git a/src/bigbang/mqcluster.h b/src/bigbang/mqcluster.h index bf0afc4b..086d70fb 100644 --- a/src/bigbang/mqcluster.h +++ b/src/bigbang/mqcluster.h @@ -163,11 +163,10 @@ class CMQCluster : public IMQCluster string topicRespBlk; string topicRbBlk; NODE_CATEGORY catNode; -// std::vector vForkNode; boost::mutex mtxStatus; boost::condition_variable condStatus; - std::map> mapForkNode; - std::map mapActiveForkNode; + std::map> mapSuperNode; + std::map mapActiveSuperNode; boost::mutex mtxSend; boost::condition_variable condSend; diff --git a/src/bigbang/rpcmod.cpp b/src/bigbang/rpcmod.cpp index 8986b642..494b672b 100644 --- a/src/bigbang/rpcmod.cpp +++ b/src/bigbang/rpcmod.cpp @@ -2414,11 +2414,11 @@ CRPCResultPtr CRPCMod::RPCEnrollSuperNode(rpc::CRPCParamPtr param) } forks.emplace_back(fork); } - storage::CForkNode node; - node.forkNodeID = std::move(id); + storage::CSuperNode node; + node.superNodeID = std::move(id); node.vecOwnedForks = std::move(forks); - if(OK != pService->AddForkNode(node)) + if(OK != pService->AddSuperNode(node)) { throw CRPCException(RPC_INTERNAL_ERROR, "Enroll fork node failed"); } diff --git a/src/bigbang/service.cpp b/src/bigbang/service.cpp index aac6bae4..cf05e775 100644 --- a/src/bigbang/service.cpp +++ b/src/bigbang/service.cpp @@ -669,9 +669,9 @@ bool CService::GetTxSender(const uint256& txid, CAddress& sender) return true; } -bool CService::AddForkNode(const storage::CForkNode& node) +bool CService::AddSuperNode(const storage::CSuperNode& node) { - return pBlockChain->AddNewForkNode(node); + return pBlockChain->AddNewSuperNode(node); } CAddress CService::GetBackSender(const uint256& txid) diff --git a/src/bigbang/service.h b/src/bigbang/service.h index 0cadc13f..64dc968e 100644 --- a/src/bigbang/service.h +++ b/src/bigbang/service.h @@ -81,7 +81,7 @@ class CService : public IService crypto::CKey& keyMint, uint256& hashBlock) override; /* Util */ bool GetTxSender(const uint256& txid, CAddress& sender) override; - bool AddForkNode(const storage::CForkNode& node) override; + bool AddSuperNode(const storage::CSuperNode& node) override; protected: bool HandleInitialize() override; diff --git a/src/bigbang/struct.h b/src/bigbang/struct.h index 62258de3..9d904160 100644 --- a/src/bigbang/struct.h +++ b/src/bigbang/struct.h @@ -265,6 +265,13 @@ class CMqRollbackUpdate std::vector vShort; }; +class CMqSuperNodeUpdate +{ +public: + std::string superNodeID; + std::vector vecOwnedForks; +}; + /* Net Channel */ class CPeerKnownTx { diff --git a/src/storage/blockbase.cpp b/src/storage/blockbase.cpp index 7fde756d..f905ea4e 100644 --- a/src/storage/blockbase.cpp +++ b/src/storage/blockbase.cpp @@ -493,36 +493,36 @@ bool CBlockBase::AddNewForkContext(const CForkContext& ctxt) return true; } -bool CBlockBase::AddNewForkNode(const CForkNode& forkNode) +bool CBlockBase::AddNewSuperNode(const CSuperNode& superNode) { - if (!dbBlock.AddNewForkNode(forkNode)) + if (!dbBlock.AddNewSuperNode(superNode)) { - Error("F", "Failed to addnew forknode in %s", forkNode.forkNodeID.c_str()); + Error("F", "Failed to addnew supernode in %s", superNode.superNodeID.c_str()); return false; } - Log("F", "AddNew forknode,cliID=%s with %d forks", forkNode.forkNodeID.c_str(), - forkNode.vecOwnedForks.size()); - for (const auto& i : forkNode.vecOwnedForks) + Log("F", "AddNew supernode,cliID=%s with %d forks", superNode.superNodeID.c_str(), + superNode.vecOwnedForks.size()); + for (const auto& i : superNode.vecOwnedForks) { - Log("F", "AddNew forknode with fork [%s]", i.ToString().c_str()); + Log("F", "AddNew supernode with fork [%s]", i.ToString().c_str()); } return true; } -bool CBlockBase::ListForkNode(std::vector& nodes) +bool CBlockBase::ListSuperNode(vector& nodes) { - if (!dbBlock.ListForkNode(nodes)) + if (!dbBlock.ListSuperNode(nodes)) { - Error("F", "Failed to list forknode"); + Error("F", "Failed to list supernode"); return false; } - Log("F", "List forknode successfully"); + Log("F", "List supernode successfully"); for (const auto& node : nodes) { for (const auto& fork : node.vecOwnedForks) { - Log("F", "forknode client ID [%s] : fork [%s]", - node.forkNodeID.c_str(), fork.ToString().c_str()); + Log("F", "supernode client ID [%s] : fork [%s]", + node.superNodeID.c_str(), fork.ToString().c_str()); } } return true; diff --git a/src/storage/blockbase.h b/src/storage/blockbase.h index cb7ad53b..184185f3 100644 --- a/src/storage/blockbase.h +++ b/src/storage/blockbase.h @@ -233,8 +233,8 @@ class CBlockBase bool Initiate(const uint256& hashGenesis, const CBlock& blockGenesis, const uint256& nChainTrust); bool AddNew(const uint256& hash, CBlockEx& block, CBlockIndex** ppIndexNew, const uint256& nChainTrust); bool AddNewForkContext(const CForkContext& ctxt); - bool AddNewForkNode(const CForkNode& forkNode); - bool ListForkNode(std::vector& nodes); + bool AddNewSuperNode(const CSuperNode& superNode); + bool ListSuperNode(std::vector& nodes); bool Retrieve(const uint256& hash, CBlock& block); bool Retrieve(const CBlockIndex* pIndex, CBlock& block); bool Retrieve(const uint256& hash, CBlockEx& block); diff --git a/src/storage/blockdb.cpp b/src/storage/blockdb.cpp index 5bb4ac6e..54c21262 100644 --- a/src/storage/blockdb.cpp +++ b/src/storage/blockdb.cpp @@ -51,7 +51,7 @@ bool CBlockDB::Initialize(const boost::filesystem::path& pathData) return false; } - if (!dbForkNode.Initialize(pathData)) + if (!dbSuperNode.Initialize(pathData)) { return false; } @@ -66,7 +66,7 @@ void CBlockDB::Deinitialize() dbTxIndex.Deinitialize(); dbBlockIndex.Deinitialize(); dbFork.Deinitialize(); - dbForkNode.Deinitialize(); + dbSuperNode.Deinitialize(); } bool CBlockDB::RemoveAll() @@ -76,7 +76,7 @@ bool CBlockDB::RemoveAll() dbTxIndex.Clear(); dbBlockIndex.Clear(); dbFork.Clear(); - dbForkNode.Clear(); + dbSuperNode.Clear(); return true; } @@ -251,14 +251,14 @@ bool CBlockDB::LoadFork() return true; } -bool CBlockDB::AddNewForkNode(const CForkNode& forkNode) +bool CBlockDB::AddNewSuperNode(const CSuperNode& superNode) { - return dbForkNode.AddNewForkNode(forkNode); + return dbSuperNode.AddNewSuperNode(superNode); } -bool CBlockDB::ListForkNode(std::vector& nodes) +bool CBlockDB::ListSuperNode(std::vector& nodes) { - return dbForkNode.ListForkNode(nodes); + return dbSuperNode.ListSuperNode(nodes); } } // namespace storage diff --git a/src/storage/blockdb.h b/src/storage/blockdb.h index cacdeb99..17780caa 100644 --- a/src/storage/blockdb.h +++ b/src/storage/blockdb.h @@ -48,8 +48,8 @@ class CBlockDB bool RetrieveDelegate(const uint256& hash, std::map& mapDelegate); bool RetrieveEnroll(int height, const std::vector& vBlockRange, std::map& mapEnrollTxPos); - bool AddNewForkNode(const CForkNode& forkNode); - bool ListForkNode(std::vector& nodes); + bool AddNewSuperNode(const CSuperNode& superNode); + bool ListSuperNode(std::vector& nodes); protected: bool LoadFork(); @@ -60,7 +60,7 @@ class CBlockDB CTxIndexDB dbTxIndex; CUnspentDB dbUnspent; CDelegateDB dbDelegate; - CForkNodeDB dbForkNode; + CSuperNodeDB dbSuperNode; }; } // namespace storage diff --git a/src/storage/mqdb.cpp b/src/storage/mqdb.cpp index 4dcd1d78..8ba5482c 100644 --- a/src/storage/mqdb.cpp +++ b/src/storage/mqdb.cpp @@ -15,12 +15,12 @@ namespace storage { ////////////////////////////// -// CForkNodeDB +// CSuperNodeDB -bool CForkNodeDB::Initialize(const boost::filesystem::path& pathData) +bool CSuperNodeDB::Initialize(const boost::filesystem::path& pathData) { CLevelDBArguments args; - args.path = (pathData / "forknode").string(); + args.path = (pathData / "supernode").string(); args.syncwrite = true; args.files = 16; args.cache = 2 << 20; @@ -36,35 +36,35 @@ bool CForkNodeDB::Initialize(const boost::filesystem::path& pathData) return true; } -void CForkNodeDB::Deinitialize() +void CSuperNodeDB::Deinitialize() { Close(); } -bool CForkNodeDB::AddNewForkNode(const CForkNode& cli) +bool CSuperNodeDB::AddNewSuperNode(const CSuperNode& cli) { - vector nodes; - ListForkNode(nodes); - bool ret = Write(cli.forkNodeID, cli.vecOwnedForks, true); //overwrite - ListForkNode(nodes); + vector nodes; + ListSuperNode(nodes); + bool ret = Write(cli.superNodeID, cli.vecOwnedForks, true); //overwrite + ListSuperNode(nodes); return ret; } -bool CForkNodeDB::RemoveForkNode(const string& cliID) +bool CSuperNodeDB::RemoveSuperNode(const string& cliID) { return Erase(cliID); } -bool CForkNodeDB::RetrieveForkNode(const string& cliID, CForkNode& cli) +bool CSuperNodeDB::RetrieveSuperNode(const string& cliID, CSuperNode& cli) { return Read(cliID, cli); } -bool CForkNodeDB::ListForkNode(std::vector& vCli) +bool CSuperNodeDB::ListSuperNode(std::vector& vCli) { map> mapCli; - if (!WalkThrough(boost::bind(&CForkNodeDB::LoadForkNodeWalker, this, _1, _2, boost::ref(mapCli)))) + if (!WalkThrough(boost::bind(&CSuperNodeDB::LoadSuperNodeWalker, this, _1, _2, boost::ref(mapCli)))) { return false; } @@ -72,20 +72,20 @@ bool CForkNodeDB::ListForkNode(std::vector& vCli) vCli.reserve(mapCli.size()); for (const auto& it : mapCli) { - CForkNode node; - node.forkNodeID = it.first; + CSuperNode node; + node.superNodeID = it.first; node.vecOwnedForks = it.second; vCli.emplace_back(node); } return true; } -void CForkNodeDB::Clear() +void CSuperNodeDB::Clear() { RemoveAll(); } -bool CForkNodeDB::LoadForkNodeWalker(xengine::CBufStream& ssKey, xengine::CBufStream& ssValue, +bool CSuperNodeDB::LoadSuperNodeWalker(xengine::CBufStream& ssKey, xengine::CBufStream& ssValue, map>& mapCli) { string strCliID; diff --git a/src/storage/mqdb.h b/src/storage/mqdb.h index f3f5c9d4..7d0b9cb4 100644 --- a/src/storage/mqdb.h +++ b/src/storage/mqdb.h @@ -13,41 +13,41 @@ namespace bigbang namespace storage { -class CForkNode +class CSuperNode { friend class xengine::CStream; public: - std::string forkNodeID; + std::string superNodeID; std::vector vecOwnedForks; public: - CForkNode(std::string id = std::string(), std::vector forks = std::vector()) - : forkNodeID(id), vecOwnedForks(forks) {} + CSuperNode(std::string id = std::string(), std::vector forks = std::vector()) + : superNodeID(id), vecOwnedForks(forks) {} protected: template void Serialize(xengine::CStream& s, O& opt) { - s.Serialize(forkNodeID, opt); + s.Serialize(superNodeID, opt); s.Serialize(vecOwnedForks, opt); } }; -class CForkNodeDB : public xengine::CKVDB +class CSuperNodeDB : public xengine::CKVDB { public: - CForkNodeDB() {} + CSuperNodeDB() {} bool Initialize(const boost::filesystem::path& pathData); void Deinitialize(); - bool AddNewForkNode(const CForkNode& cli); - bool RemoveForkNode(const std::string& cliID); - bool RetrieveForkNode(const std::string& forkNodeID, CForkNode& cli); - bool ListForkNode(std::vector& vCli); + bool AddNewSuperNode(const CSuperNode& cli); + bool RemoveSuperNode(const std::string& cliID); + bool RetrieveSuperNode(const std::string& superNodeID, CSuperNode& cli); + bool ListSuperNode(std::vector& vCli); void Clear(); protected: - bool LoadForkNodeWalker(xengine::CBufStream& ssKey, xengine::CBufStream& ssValue, + bool LoadSuperNodeWalker(xengine::CBufStream& ssKey, xengine::CBufStream& ssValue, std::map>& mapCli); }; From 92f6d47d60199146e09dab7e38f561836b5c44cb Mon Sep 17 00:00:00 2001 From: OuYun Date: Wed, 18 Mar 2020 14:26:26 +0800 Subject: [PATCH 026/181] restrict non-super nodes to enroll fork info for mq communication --- script/template/rpc.json | 5 +++++ src/bigbang/rpcmod.cpp | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/script/template/rpc.json b/script/template/rpc.json index f3ce439d..91402f59 100755 --- a/script/template/rpc.json +++ b/script/template/rpc.json @@ -677,6 +677,11 @@ "request": "curl -d '{\"id\":3,\"method\":\"enrollsupernode\",\"jsonrpc\":\"2.0\",\"params\":{[\"575f2041770496489120bb102d9dd55f5e75b0c4aa528d5762b92b59acd6d939\",\"6236d780f9f743707d57b3feb19d21c8a867577f5e83c163774222bb7ef8d8cb\",\"7c6a6aba05cec77a998c19649ee1fa0e29c7b5246d0e3a6501ee1d4d81dd73ea\"]}}' http://127.0.0.1:9902", "response": "{\"id\":3,\"jsonrpc\":\"2.0\",\"result\":true}" } + ], + "error": [ + "{\"code\":-32600,\"message\":\"Invalid super node\"}", + "{\"code\":-6,\"message\":\"Unknown fork\"}", + "{\"code\":-32603,\"message\":\"Add super node enrollment failed\"}" ] }, "listpeer": { diff --git a/src/bigbang/rpcmod.cpp b/src/bigbang/rpcmod.cpp index 494b672b..dbb7f88e 100644 --- a/src/bigbang/rpcmod.cpp +++ b/src/bigbang/rpcmod.cpp @@ -2400,6 +2400,12 @@ CRPCResultPtr CRPCMod::RPCListUnspent(CRPCParamPtr param) CRPCResultPtr CRPCMod::RPCEnrollSuperNode(rpc::CRPCParamPtr param) { + int nNodeCat = dynamic_cast(Config())->nCatOfNode; + if (1 != nNodeCat && 2 != nNodeCat) + { + throw CRPCException(RPC_INVALID_REQUEST, "Only super node has the feature"); + } + auto spParam = CastParamPtr(param); std::vector vFork = spParam->vecForks; std::string id = spParam->strClientid; @@ -2420,7 +2426,7 @@ CRPCResultPtr CRPCMod::RPCEnrollSuperNode(rpc::CRPCParamPtr param) if(OK != pService->AddSuperNode(node)) { - throw CRPCException(RPC_INTERNAL_ERROR, "Enroll fork node failed"); + throw CRPCException(RPC_INTERNAL_ERROR, "Enroll super node failed"); } auto spResult = MakeCEnrollSuperNodeResultPtr(); From 5f74966c9fe5163bcf659b2f38bbb0e6d48395e2 Mon Sep 17 00:00:00 2001 From: OuYun Date: Wed, 18 Mar 2020 15:43:54 +0800 Subject: [PATCH 027/181] add trigger to active mq communication when RPC enrollment is called --- src/bigbang/dispatcher.cpp | 8 +++--- src/bigbang/dispatcher.h | 2 +- src/bigbang/mqcluster.cpp | 54 ++++++++++++++++++++++++++++++++++++-- src/bigbang/mqcluster.h | 1 + src/bigbang/mqevent.h | 3 +++ src/bigbang/service.cpp | 36 ++++++++++++++++++++++--- src/bigbang/service.h | 2 ++ src/bigbang/struct.h | 4 +-- 8 files changed, 97 insertions(+), 13 deletions(-) diff --git a/src/bigbang/dispatcher.cpp b/src/bigbang/dispatcher.cpp index 1d4782fc..6dbdd60e 100644 --- a/src/bigbang/dispatcher.cpp +++ b/src/bigbang/dispatcher.cpp @@ -32,7 +32,7 @@ CDispatcher::CDispatcher() pNetChannel = nullptr; // pDelegatedChannel = nullptr; pDataStat = nullptr; - pMqcluster = nullptr; + pMQCluster = nullptr; } CDispatcher::~CDispatcher() @@ -111,7 +111,7 @@ bool CDispatcher::HandleInitialize() nNodeCat = dynamic_cast(Config())->nCatOfNode; if (2 == nNodeCat) { - if (!GetObject("mqcluster", pMqcluster)) + if (!GetObject("mqcluster", pMQCluster)) { Error("Failed to request mqcluster"); return false; @@ -134,7 +134,7 @@ void CDispatcher::HandleDeinitialize() pNetChannel = nullptr; // pDelegatedChannel = nullptr; pDataStat = nullptr; - pMqcluster = nullptr; + pMQCluster = nullptr; } bool CDispatcher::HandleInvoke() @@ -383,7 +383,7 @@ void CDispatcher::UpdatePrimaryBlock(const CBlock& block, const CBlockChainUpdat pMqChainUpdate->data.vShort.push_back(rb.GetHash()); } - pMqcluster->PostEvent(pMqChainUpdate); + pMQCluster->PostEvent(pMqChainUpdate); } } } diff --git a/src/bigbang/dispatcher.h b/src/bigbang/dispatcher.h index 41af0fd2..6f2bb2a4 100644 --- a/src/bigbang/dispatcher.h +++ b/src/bigbang/dispatcher.h @@ -48,7 +48,7 @@ class CDispatcher : public IDispatcher IDataStat* pDataStat; std::string strCmd; int nNodeCat; - IMQCluster* pMqcluster; + IMQCluster* pMQCluster; }; } // namespace bigbang diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index b90d263b..4af9d5d5 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -52,11 +52,11 @@ bool CMQCluster::HandleInitialize() { if (NODE_CATEGORY::FORKNODE == catNode) { - clientID = "FORKNODE-01"; +// clientID = "FORKNODE-01"; } else if (NODE_CATEGORY::DPOSNODE == catNode) { - clientID = "DPOSNODE"; +// clientID = "DPOSNODE"; } else if (NODE_CATEGORY::BBCNODE == catNode) { @@ -245,6 +245,56 @@ bool CMQCluster::HandleEvent(CEventMQChainUpdate& eventMqUpdateChain) return true; } +bool CMQCluster::HandleEvent(CEventMQEnrollUpdate& eventMqUpdateEnroll) +{ + string id = eventMqUpdateEnroll.data.superNodeClientID; + vector forks = eventMqUpdateEnroll.data.vecForksOwned; + if (NODE_CATEGORY::FORKNODE == catNode) + { + clientID = id; + topicRespBlk = "Cluster01/" + clientID + "/SyncBlockResp"; + topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; + Log("CMQCluster::HandleEvent(): fork node clientid [%s] with topics " + "[%s] \n [%s]:", clientID.c_str(), + topicRespBlk.c_str(), topicRbBlk.c_str()); + for (const auto& fork : forks) + { + Log("CMQCluster::HandleEvent(): fork [%s] intended to be produced " + "by this node [%s]:", fork.ToString().c_str(), clientID.c_str()); + } + + if (!PostBlockRequest(-1)) + { + Error("CMQCluster::HandleEvent(): failed to post requesting block"); + return false; + } + } + else if (NODE_CATEGORY::DPOSNODE == catNode) + { + if (1 == forks.size() + && forks[0] == pCoreProtocol->GetGenesisBlockHash()) + { //dpos node + clientID = id; + topicReqBlk = "Cluster01/+/SyncBlockReq"; + Log("CMQCluster::HandleEvent(): dpos node clientid [%s] with topic [%s]", + clientID.c_str(), topicReqBlk.c_str()); + } + else + { //fork nodes either enrolled or p2p + //mapActiveSuperNode[ip] = storage::CSuperNode(); + Log("CMQCluster::HandleEvent(): dpos node register clientid [%s] with topic [%s]", + clientID.c_str(), topicReqBlk.c_str()); + } + } + { + boost::unique_lock lock(mtxStatus); + mapSuperNode.insert(make_pair(id, forks)); + } + condStatus.notify_all(); + + return true; +} + bool CMQCluster::HandleEvent(CEventMQAgreement& eventMqAgreement) { return true; diff --git a/src/bigbang/mqcluster.h b/src/bigbang/mqcluster.h index 086d70fb..2a95eed0 100644 --- a/src/bigbang/mqcluster.h +++ b/src/bigbang/mqcluster.h @@ -133,6 +133,7 @@ class CMQCluster : public IMQCluster bool HandleEvent(CEventMQSyncBlock& eventMqSyncBlock) override; bool HandleEvent(CEventMQChainUpdate& eventMqUpdateChain) override; + bool HandleEvent(CEventMQEnrollUpdate& eventMqUpdateEnroll) override; bool HandleEvent(CEventMQAgreement& eventMqAgreement) override; bool IsAuthenticated() override; diff --git a/src/bigbang/mqevent.h b/src/bigbang/mqevent.h index c1895fe8..d3b80ce6 100644 --- a/src/bigbang/mqevent.h +++ b/src/bigbang/mqevent.h @@ -18,6 +18,7 @@ enum EVENT_MQ_BASE = EVENT_USER_BASE, EVENT_MQ_SYNCBLOCK, EVENT_MQ_UPDATEBLOCK, //dpos node deliver rollback block + EVENT_MQ_ENROLLUPDATE, EVENT_MQ_AGREEMENT }; @@ -27,6 +28,7 @@ class CMQEventListener; typedef TYPE_MQEVENT(EVENT_MQ_SYNCBLOCK, std::string) CEventMQSyncBlock; typedef TYPE_MQEVENT(EVENT_MQ_UPDATEBLOCK, CMqRollbackUpdate) CEventMQChainUpdate; +typedef TYPE_MQEVENT(EVENT_MQ_ENROLLUPDATE, CMqSuperNodeUpdate) CEventMQEnrollUpdate; typedef TYPE_MQEVENT(EVENT_MQ_AGREEMENT, CDelegateAgreement) CEventMQAgreement; class CMQEventListener : virtual public CEventListener @@ -35,6 +37,7 @@ class CMQEventListener : virtual public CEventListener virtual ~CMQEventListener() {} DECLARE_EVENTHANDLER(CEventMQSyncBlock); DECLARE_EVENTHANDLER(CEventMQChainUpdate); + DECLARE_EVENTHANDLER(CEventMQEnrollUpdate); DECLARE_EVENTHANDLER(CEventMQAgreement); }; diff --git a/src/bigbang/service.cpp b/src/bigbang/service.cpp index cf05e775..697600b4 100644 --- a/src/bigbang/service.cpp +++ b/src/bigbang/service.cpp @@ -18,7 +18,14 @@ namespace bigbang // CService CService::CService() - : pCoreProtocol(nullptr), pBlockChain(nullptr), pTxPool(nullptr), pDispatcher(nullptr), pWallet(nullptr), pNetwork(nullptr), pForkManager(nullptr) + : pCoreProtocol(nullptr), + pBlockChain(nullptr), + pTxPool(nullptr), + pDispatcher(nullptr), + pWallet(nullptr), + pNetwork(nullptr), + pForkManager(nullptr), + pMQCluster(nullptr) { } @@ -70,6 +77,12 @@ bool CService::HandleInitialize() return false; } + if (!GetObject("mqcluster", pMQCluster)) + { + Error("Failed to request mqcluster"); + return false; + } + return true; } @@ -82,6 +95,7 @@ void CService::HandleDeinitialize() pWallet = nullptr; pNetwork = nullptr; pForkManager = nullptr; + pMQCluster = nullptr; } bool CService::HandleInvoke() @@ -461,8 +475,8 @@ bool CService::SignTransaction(CTransaction& tx, bool& fCompleted) } if (!fCompleted - || (pCoreProtocol->ValidateTransaction(tx) == OK - && pCoreProtocol->VerifyTransaction(tx, vUnspent, nForkHeight, hashFork) == OK)) + || (pCoreProtocol->ValidateTransaction(tx) == OK + && pCoreProtocol->VerifyTransaction(tx, vUnspent, nForkHeight, hashFork) == OK)) { return true; } @@ -671,7 +685,21 @@ bool CService::GetTxSender(const uint256& txid, CAddress& sender) bool CService::AddSuperNode(const storage::CSuperNode& node) { - return pBlockChain->AddNewSuperNode(node); + if (!pBlockChain->AddNewSuperNode(node)) + { + StdError("CService::AddSuperNode", "Add new superNode failed."); + return false; + } + + CEventMQEnrollUpdate *pEvent = new CEventMQEnrollUpdate(0); + if (nullptr != pEvent) + { + pEvent->data.superNodeClientID = node.superNodeID; + pEvent->data.vecForksOwned = node.vecOwnedForks; + pMQCluster->PostEvent(pEvent); + } + + return true; } CAddress CService::GetBackSender(const uint256& txid) diff --git a/src/bigbang/service.h b/src/bigbang/service.h index 64dc968e..d7ec96ba 100644 --- a/src/bigbang/service.h +++ b/src/bigbang/service.h @@ -7,6 +7,7 @@ #include "base.h" #include "mqdb.h" +#include "mqcluster.h" #include "network.h" #include "xengine.h" @@ -100,6 +101,7 @@ class CService : public IService IWallet* pWallet; CNetwork* pNetwork; IForkManager* pForkManager; + IMQCluster* pMQCluster; mutable boost::shared_mutex rwForkStatus; std::map mapForkStatus; }; diff --git a/src/bigbang/struct.h b/src/bigbang/struct.h index 9d904160..5e4c0542 100644 --- a/src/bigbang/struct.h +++ b/src/bigbang/struct.h @@ -268,8 +268,8 @@ class CMqRollbackUpdate class CMqSuperNodeUpdate { public: - std::string superNodeID; - std::vector vecOwnedForks; + std::string superNodeClientID; + std::vector vecForksOwned; }; /* Net Channel */ From 0c34d7e7cddad25770210ea0d80975fd8cc9e733 Mon Sep 17 00:00:00 2001 From: OuYun Date: Wed, 18 Mar 2020 16:41:51 +0800 Subject: [PATCH 028/181] fix issue of being not triggered when new enrollment entered --- src/bigbang/mqcluster.cpp | 10 +++++----- src/bigbang/service.cpp | 4 ++-- src/storage/blockbase.cpp | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 4af9d5d5..b09acfe4 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -135,7 +135,6 @@ bool CMQCluster::HandleInvoke() if (NODE_CATEGORY::FORKNODE == catNode) { - topicReqBlk = "Cluster01/" + clientID + "/SyncBlockReq"; if (mapSuperNode.size() == 0) { Log("CMQCluster::HandleInvoke(): this fork node has not enrolled " @@ -152,10 +151,11 @@ bool CMQCluster::HandleInvoke() if (1 == mapSuperNode.size()) { clientID = mapSuperNode.begin()->first; + topicReqBlk = "Cluster01/" + clientID + "/SyncBlockReq"; topicRespBlk = "Cluster01/" + clientID + "/SyncBlockResp"; topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; - Log("CMQCluster::HandleInvoke(): fork node clientid [%s] with topics " - "[%s] \n [%s]:", clientID.c_str(), + Log("CMQCluster::HandleInvoke(): fork node clientid [%s] with topics:" + "\t[%s]\n\t[%s]", clientID.c_str(), topicRespBlk.c_str(), topicRbBlk.c_str()); for (const auto& fork : mapSuperNode.begin()->second) { @@ -343,8 +343,8 @@ bool CMQCluster::PostBlockRequest(int syncHeight) } CSyncBlockRequest req; - // req.ipAddr = 16777343; //127.0.0.1 - req.ipAddr = 1111638320; + req.ipAddr = 16777343; //127.0.0.1 + //req.ipAddr = 1111638320; //"0ABB" req.forkNodeIdLen = clientID.size(); req.forkNodeId = clientID; auto enroll = mapSuperNode.begin(); diff --git a/src/bigbang/service.cpp b/src/bigbang/service.cpp index 697600b4..77b6f164 100644 --- a/src/bigbang/service.cpp +++ b/src/bigbang/service.cpp @@ -685,9 +685,9 @@ bool CService::GetTxSender(const uint256& txid, CAddress& sender) bool CService::AddSuperNode(const storage::CSuperNode& node) { - if (!pBlockChain->AddNewSuperNode(node)) + if (OK != pBlockChain->AddNewSuperNode(node)) { - StdError("CService::AddSuperNode", "Add new superNode failed."); + StdError("CService::AddSuperNode", "Add new super node failed."); return false; } diff --git a/src/storage/blockbase.cpp b/src/storage/blockbase.cpp index f905ea4e..6683c8de 100644 --- a/src/storage/blockbase.cpp +++ b/src/storage/blockbase.cpp @@ -497,14 +497,14 @@ bool CBlockBase::AddNewSuperNode(const CSuperNode& superNode) { if (!dbBlock.AddNewSuperNode(superNode)) { - Error("F", "Failed to addnew supernode in %s", superNode.superNodeID.c_str()); + Error("CBlockBase::AddNewSuperNode", "Failed to addnew supernode in %s", superNode.superNodeID.c_str()); return false; } - Log("F", "AddNew supernode,cliID=%s with %d forks", superNode.superNodeID.c_str(), + Log("CBlockBase::AddNewSuperNode", "AddNew supernode,cliID=%s with %d forks", superNode.superNodeID.c_str(), superNode.vecOwnedForks.size()); for (const auto& i : superNode.vecOwnedForks) { - Log("F", "AddNew supernode with fork [%s]", i.ToString().c_str()); + Log("CBlockBase::AddNewSuperNode", "AddNew supernode with fork [%s]", i.ToString().c_str()); } return true; } @@ -513,15 +513,15 @@ bool CBlockBase::ListSuperNode(vector& nodes) { if (!dbBlock.ListSuperNode(nodes)) { - Error("F", "Failed to list supernode"); + Error("CBlockBase::ListSuperNode", "Failed to list supernode"); return false; } - Log("F", "List supernode successfully"); + Log("CBlockBase::ListSuperNode", "List supernode successfully"); for (const auto& node : nodes) { for (const auto& fork : node.vecOwnedForks) { - Log("F", "supernode client ID [%s] : fork [%s]", + Log("CBlockBase::ListSuperNode", "supernode client ID [%s] : fork [%s]", node.superNodeID.c_str(), fork.ToString().c_str()); } } From 1931de28e06dac7c6eec0fed80c19bfda57186a5 Mon Sep 17 00:00:00 2001 From: OuYun Date: Wed, 18 Mar 2020 18:25:22 +0800 Subject: [PATCH 029/181] bugfix of sending request error on enrollment super node --- src/bigbang/mqcluster.cpp | 56 ++++++++++++++++++++++++--------------- src/bigbang/rpcmod.cpp | 2 +- 2 files changed, 35 insertions(+), 23 deletions(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index b09acfe4..bcb69bb1 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -252,6 +252,7 @@ bool CMQCluster::HandleEvent(CEventMQEnrollUpdate& eventMqUpdateEnroll) if (NODE_CATEGORY::FORKNODE == catNode) { clientID = id; + topicReqBlk = "Cluster01/" + clientID + "/SyncBlockReq"; topicRespBlk = "Cluster01/" + clientID + "/SyncBlockResp"; topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; Log("CMQCluster::HandleEvent(): fork node clientid [%s] with topics " @@ -263,6 +264,12 @@ bool CMQCluster::HandleEvent(CEventMQEnrollUpdate& eventMqUpdateEnroll) "by this node [%s]:", fork.ToString().c_str(), clientID.c_str()); } + { + boost::unique_lock lock(mtxStatus); + mapSuperNode.insert(make_pair(id, forks)); + } + condStatus.notify_all(); + if (!PostBlockRequest(-1)) { Error("CMQCluster::HandleEvent(): failed to post requesting block"); @@ -285,12 +292,13 @@ bool CMQCluster::HandleEvent(CEventMQEnrollUpdate& eventMqUpdateEnroll) Log("CMQCluster::HandleEvent(): dpos node register clientid [%s] with topic [%s]", clientID.c_str(), topicReqBlk.c_str()); } + + { + boost::unique_lock lock(mtxStatus); + mapSuperNode.insert(make_pair(id, forks)); + } + condStatus.notify_all(); } - { - boost::unique_lock lock(mtxStatus); - mapSuperNode.insert(make_pair(id, forks)); - } - condStatus.notify_all(); return true; } @@ -789,22 +797,26 @@ bool CMQCluster::ClientAgent(MQ_CLI_ACTION action) } case MQ_CLI_ACTION::PUB: { - while (!deqSendBuff.empty()) { - pair buf = deqSendBuff.front(); - cout << "\nSending message to [" << buf.first << "]..." << endl; - buf.second->Dump(); - - mqtt::message_ptr pubmsg = mqtt::make_message( - buf.first, buf.second->GetData(), buf.second->GetSize()); - pubmsg->set_qos(QOS1); - pubmsg->set_retained(mqtt::message::DFLT_RETAINED); - delitok = client.publish(pubmsg, nullptr, cb); - delitok->wait_for(100); - cout << "_._._OK" << endl; - - deqSendBuff.pop_front(); + boost::unique_lock lock(mtxSend); + while (!deqSendBuff.empty()) + { + pair buf = deqSendBuff.front(); + cout << "\nSending message to [" << buf.first << "]..." << endl; + buf.second->Dump(); + + mqtt::message_ptr pubmsg = mqtt::make_message( + buf.first, buf.second->GetData(), buf.second->GetSize()); + pubmsg->set_qos(QOS1); + pubmsg->set_retained(mqtt::message::DFLT_RETAINED); + delitok = client.publish(pubmsg, nullptr, cb); + delitok->wait_for(100); + cout << "_._._OK" << endl; + + deqSendBuff.pop_front(); + } } + condSend.notify_all(); break; } case MQ_CLI_ACTION::DISCONN: @@ -859,13 +871,13 @@ void CMQCluster::MqttThreadFunc() { { boost::unique_lock lock(mtxSend); - while (!fAbort) + while (deqSendBuff.empty()) { - ClientAgent(MQ_CLI_ACTION::PUB); - Log("thread function of MQTT: go through an iteration"); condSend.wait(lock); } } + ClientAgent(MQ_CLI_ACTION::PUB); + Log("thread function of MQTT: go through an iteration"); } //disconnect to broker diff --git a/src/bigbang/rpcmod.cpp b/src/bigbang/rpcmod.cpp index dbb7f88e..c98df374 100644 --- a/src/bigbang/rpcmod.cpp +++ b/src/bigbang/rpcmod.cpp @@ -2424,7 +2424,7 @@ CRPCResultPtr CRPCMod::RPCEnrollSuperNode(rpc::CRPCParamPtr param) node.superNodeID = std::move(id); node.vecOwnedForks = std::move(forks); - if(OK != pService->AddSuperNode(node)) + if(!pService->AddSuperNode(node)) { throw CRPCException(RPC_INTERNAL_ERROR, "Enroll super node failed"); } From 73b8dc1778704133c94fe1e595d6f186526f32eb Mon Sep 17 00:00:00 2001 From: OuYun Date: Wed, 18 Mar 2020 18:50:35 +0800 Subject: [PATCH 030/181] adjust log output --- src/bigbang/mqcluster.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index bcb69bb1..678260a8 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -255,8 +255,8 @@ bool CMQCluster::HandleEvent(CEventMQEnrollUpdate& eventMqUpdateEnroll) topicReqBlk = "Cluster01/" + clientID + "/SyncBlockReq"; topicRespBlk = "Cluster01/" + clientID + "/SyncBlockResp"; topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; - Log("CMQCluster::HandleEvent(): fork node clientid [%s] with topics " - "[%s] \n [%s]:", clientID.c_str(), + Log("CMQCluster::HandleEvent(): fork node clientid [%s] with topics:" + "\n[%s]\n[%s]", clientID.c_str(), topicRespBlk.c_str(), topicRbBlk.c_str()); for (const auto& fork : forks) { From decce2293d61687b9d85a323b449d58d5e5d41f9 Mon Sep 17 00:00:00 2001 From: OuYun Date: Thu, 19 Mar 2020 14:23:59 +0800 Subject: [PATCH 031/181] bugfix of receiving block validate failed --- src/bigbang/mqcluster.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 678260a8..27ea2f93 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -425,14 +425,15 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) //if (topicReqBlk != clientID) //validate this coming block - if (!pCoreProtocol->ValidateBlock(resp.block)) + Errno err = pCoreProtocol->ValidateBlock(resp.block); + if (OK != err) { Error("CMQCluster::OnReceiveMessage(): failed to validate block"); return; } //notify to add new block - Errno err = pDispatcher->AddNewBlock(resp.block); + err = pDispatcher->AddNewBlock(resp.block); if (err != OK) { Error("CMQCluster::OnReceiveMessage(): failed to validate block (%d) : %s\n", err, ErrorString(err)); From 39b44695f37e74e94606540667ecd1b9b0ecc4fa Mon Sep 17 00:00:00 2001 From: OuYun Date: Thu, 19 Mar 2020 16:32:34 +0800 Subject: [PATCH 032/181] resolve sync issue when generating main chain block on dpos node --- src/bigbang/mqcluster.cpp | 105 ++++++++++++++++++++++++++------------ 1 file changed, 73 insertions(+), 32 deletions(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 27ea2f93..4d73eadb 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -52,11 +52,11 @@ bool CMQCluster::HandleInitialize() { if (NODE_CATEGORY::FORKNODE == catNode) { -// clientID = "FORKNODE-01"; + // clientID = "FORKNODE-01"; } else if (NODE_CATEGORY::DPOSNODE == catNode) { -// clientID = "DPOSNODE"; + // clientID = "DPOSNODE"; } else if (NODE_CATEGORY::BBCNODE == catNode) { @@ -155,12 +155,14 @@ bool CMQCluster::HandleInvoke() topicRespBlk = "Cluster01/" + clientID + "/SyncBlockResp"; topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; Log("CMQCluster::HandleInvoke(): fork node clientid [%s] with topics:" - "\t[%s]\n\t[%s]", clientID.c_str(), + "\t[%s]\n\t[%s]", + clientID.c_str(), topicRespBlk.c_str(), topicRbBlk.c_str()); for (const auto& fork : mapSuperNode.begin()->second) { Log("CMQCluster::HandleInvoke(): fork [%s] intended to be produced " - "by this node [%s]:", fork.ToString().c_str(), clientID.c_str()); + "by this node [%s]:", + fork.ToString().c_str(), clientID.c_str()); } if (!PostBlockRequest(-1)) @@ -256,12 +258,14 @@ bool CMQCluster::HandleEvent(CEventMQEnrollUpdate& eventMqUpdateEnroll) topicRespBlk = "Cluster01/" + clientID + "/SyncBlockResp"; topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; Log("CMQCluster::HandleEvent(): fork node clientid [%s] with topics:" - "\n[%s]\n[%s]", clientID.c_str(), + "\n[%s]\n[%s]", + clientID.c_str(), topicRespBlk.c_str(), topicRbBlk.c_str()); for (const auto& fork : forks) { Log("CMQCluster::HandleEvent(): fork [%s] intended to be produced " - "by this node [%s]:", fork.ToString().c_str(), clientID.c_str()); + "by this node [%s]:", + fork.ToString().c_str(), clientID.c_str()); } { @@ -406,6 +410,7 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) return; case NODE_CATEGORY::FORKNODE: { + Log("CMQCluster::OnReceiveMessage(): current height is [%d]", int(lastHeightResp)); if (topicRbBlk != topic) { //respond to request block of main chain //unpack payload @@ -421,6 +426,14 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) return; } + if (-1 == resp.height) + { //has reached the best height for the first time communication, + // then set timer to process the following business rather than req/resp model + nReqBlkTimerID = SetTimer(1000 * 30, + boost::bind(&CMQCluster::RequestBlockTimerFunc, this, _1)); + return; + } + //check if this msg is just for me //if (topicReqBlk != clientID) @@ -444,7 +457,7 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) //iterate to retrieve next one if (resp.isBest) - { //when reaching best height, send request by timer + { //when reach best height, send request by timer nReqBlkTimerID = SetTimer(1000 * 60, boost::bind(&CMQCluster::RequestBlockTimerFunc, this, _1)); } @@ -568,42 +581,70 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) } //add this requesting fork node to active list - mapActiveSuperNode[req.ipAddr] = storage::CSuperNode(req.forkNodeId, req.forkList); - - //processing request from fork node - //check height and hash are matched - uint256 hash; - if (!pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), req.lastHeight, hash)) + if (!mapActiveSuperNode.count(req.ipAddr)) { - Error("CMQCluster::OnReceiveMessage(): failed to get checking height and hash match"); - return; + mapActiveSuperNode[req.ipAddr] = storage::CSuperNode(req.forkNodeId, req.forkList); } - if (hash != req.lastHash) + + //processing request from fork node + + //check height requested + int best = pBlockChain->GetBlockCount(pCoreProtocol->GetGenesisBlockHash()) - 1; + CSyncBlockResponse resp; + if (req.lastHeight > best) { - Error("CMQCluster::OnReceiveMessage(): height and hash do not match"); + Error("CMQCluster::OnReceiveMessage(): block height owned by fork node " + "should not be greater than the best one on dpos node"); return; } - if (!pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), req.lastHeight + 1, hash)) + else if (req.lastHeight == best) { - Error("CMQCluster::OnReceiveMessage(): failed to get next block hash"); - return; + Log("CMQCluster::OnReceiveMessage(): block height owned by fork node " + "has reached the best one on dpos node, please wait..."); + resp.height = -1; + resp.hash = uint256(); + resp.isBest = 1; + resp.block = CBlock(); + resp.blockSize = xengine::GetSerializeSize(resp.block); } - CBlock block; - if (!pBlockChain->GetBlock(hash, block)) + else { - Error("CMQCluster::OnReceiveMessage(): failed to get next block"); - return; - } + //check height and hash are matched + uint256 hash; + if (!pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), req.lastHeight, hash)) + { + Error("CMQCluster::OnReceiveMessage(): failed to get checking height and hash match"); + return; + } + if (hash != req.lastHash) + { + Error("CMQCluster::OnReceiveMessage(): height and hash do not match"); + return; + } + if (!pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), req.lastHeight + 1, hash)) + { + Error("CMQCluster::OnReceiveMessage(): failed to get next block hash"); + return; + } + CBlock block; + if (!pBlockChain->GetBlock(hash, block)) + { + Error("CMQCluster::OnReceiveMessage(): failed to get next block"); + return; + } - //reply block requested - CSyncBlockResponse resp; - resp.height = req.lastHeight + 1; - resp.hash = hash; - resp.isBest = resp.height < pBlockChain->GetBlockCount(pCoreProtocol->GetGenesisBlockHash()) + //reply block requested + resp.height = req.lastHeight + 1; + resp.hash = hash; + resp.isBest = req.lastHeight + 1 < best ? 0 : 1; - resp.blockSize = xengine::GetSerializeSize(block); - resp.block = move(block); + Log("CMQCluster::OnReceiveMessage(): request[%d] best[%d] isBest[%d]", + req.lastHeight + 1, best, resp.isBest); + resp.blockSize = xengine::GetSerializeSize(block); + resp.block = move(block); + } + CBufferPtr spSS(new CBufStream); *spSS.get() << resp; From 849d7bad6f330959b0b1f5203d946b792d1b4fb6 Mon Sep 17 00:00:00 2001 From: OuYun Date: Thu, 19 Mar 2020 16:32:34 +0800 Subject: [PATCH 033/181] resolve sync issue when occurs jitter of generating main chain block on dpos node --- src/bigbang/mqcluster.cpp | 105 ++++++++++++++++++++++++++------------ 1 file changed, 73 insertions(+), 32 deletions(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 27ea2f93..4d73eadb 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -52,11 +52,11 @@ bool CMQCluster::HandleInitialize() { if (NODE_CATEGORY::FORKNODE == catNode) { -// clientID = "FORKNODE-01"; + // clientID = "FORKNODE-01"; } else if (NODE_CATEGORY::DPOSNODE == catNode) { -// clientID = "DPOSNODE"; + // clientID = "DPOSNODE"; } else if (NODE_CATEGORY::BBCNODE == catNode) { @@ -155,12 +155,14 @@ bool CMQCluster::HandleInvoke() topicRespBlk = "Cluster01/" + clientID + "/SyncBlockResp"; topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; Log("CMQCluster::HandleInvoke(): fork node clientid [%s] with topics:" - "\t[%s]\n\t[%s]", clientID.c_str(), + "\t[%s]\n\t[%s]", + clientID.c_str(), topicRespBlk.c_str(), topicRbBlk.c_str()); for (const auto& fork : mapSuperNode.begin()->second) { Log("CMQCluster::HandleInvoke(): fork [%s] intended to be produced " - "by this node [%s]:", fork.ToString().c_str(), clientID.c_str()); + "by this node [%s]:", + fork.ToString().c_str(), clientID.c_str()); } if (!PostBlockRequest(-1)) @@ -256,12 +258,14 @@ bool CMQCluster::HandleEvent(CEventMQEnrollUpdate& eventMqUpdateEnroll) topicRespBlk = "Cluster01/" + clientID + "/SyncBlockResp"; topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; Log("CMQCluster::HandleEvent(): fork node clientid [%s] with topics:" - "\n[%s]\n[%s]", clientID.c_str(), + "\n[%s]\n[%s]", + clientID.c_str(), topicRespBlk.c_str(), topicRbBlk.c_str()); for (const auto& fork : forks) { Log("CMQCluster::HandleEvent(): fork [%s] intended to be produced " - "by this node [%s]:", fork.ToString().c_str(), clientID.c_str()); + "by this node [%s]:", + fork.ToString().c_str(), clientID.c_str()); } { @@ -406,6 +410,7 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) return; case NODE_CATEGORY::FORKNODE: { + Log("CMQCluster::OnReceiveMessage(): current height is [%d]", int(lastHeightResp)); if (topicRbBlk != topic) { //respond to request block of main chain //unpack payload @@ -421,6 +426,14 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) return; } + if (-1 == resp.height) + { //has reached the best height for the first time communication, + // then set timer to process the following business rather than req/resp model + nReqBlkTimerID = SetTimer(1000 * 30, + boost::bind(&CMQCluster::RequestBlockTimerFunc, this, _1)); + return; + } + //check if this msg is just for me //if (topicReqBlk != clientID) @@ -444,7 +457,7 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) //iterate to retrieve next one if (resp.isBest) - { //when reaching best height, send request by timer + { //when reach best height, send request by timer nReqBlkTimerID = SetTimer(1000 * 60, boost::bind(&CMQCluster::RequestBlockTimerFunc, this, _1)); } @@ -568,42 +581,70 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) } //add this requesting fork node to active list - mapActiveSuperNode[req.ipAddr] = storage::CSuperNode(req.forkNodeId, req.forkList); - - //processing request from fork node - //check height and hash are matched - uint256 hash; - if (!pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), req.lastHeight, hash)) + if (!mapActiveSuperNode.count(req.ipAddr)) { - Error("CMQCluster::OnReceiveMessage(): failed to get checking height and hash match"); - return; + mapActiveSuperNode[req.ipAddr] = storage::CSuperNode(req.forkNodeId, req.forkList); } - if (hash != req.lastHash) + + //processing request from fork node + + //check height requested + int best = pBlockChain->GetBlockCount(pCoreProtocol->GetGenesisBlockHash()) - 1; + CSyncBlockResponse resp; + if (req.lastHeight > best) { - Error("CMQCluster::OnReceiveMessage(): height and hash do not match"); + Error("CMQCluster::OnReceiveMessage(): block height owned by fork node " + "should not be greater than the best one on dpos node"); return; } - if (!pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), req.lastHeight + 1, hash)) + else if (req.lastHeight == best) { - Error("CMQCluster::OnReceiveMessage(): failed to get next block hash"); - return; + Log("CMQCluster::OnReceiveMessage(): block height owned by fork node " + "has reached the best one on dpos node, please wait..."); + resp.height = -1; + resp.hash = uint256(); + resp.isBest = 1; + resp.block = CBlock(); + resp.blockSize = xengine::GetSerializeSize(resp.block); } - CBlock block; - if (!pBlockChain->GetBlock(hash, block)) + else { - Error("CMQCluster::OnReceiveMessage(): failed to get next block"); - return; - } + //check height and hash are matched + uint256 hash; + if (!pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), req.lastHeight, hash)) + { + Error("CMQCluster::OnReceiveMessage(): failed to get checking height and hash match"); + return; + } + if (hash != req.lastHash) + { + Error("CMQCluster::OnReceiveMessage(): height and hash do not match"); + return; + } + if (!pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), req.lastHeight + 1, hash)) + { + Error("CMQCluster::OnReceiveMessage(): failed to get next block hash"); + return; + } + CBlock block; + if (!pBlockChain->GetBlock(hash, block)) + { + Error("CMQCluster::OnReceiveMessage(): failed to get next block"); + return; + } - //reply block requested - CSyncBlockResponse resp; - resp.height = req.lastHeight + 1; - resp.hash = hash; - resp.isBest = resp.height < pBlockChain->GetBlockCount(pCoreProtocol->GetGenesisBlockHash()) + //reply block requested + resp.height = req.lastHeight + 1; + resp.hash = hash; + resp.isBest = req.lastHeight + 1 < best ? 0 : 1; - resp.blockSize = xengine::GetSerializeSize(block); - resp.block = move(block); + Log("CMQCluster::OnReceiveMessage(): request[%d] best[%d] isBest[%d]", + req.lastHeight + 1, best, resp.isBest); + resp.blockSize = xengine::GetSerializeSize(block); + resp.block = move(block); + } + CBufferPtr spSS(new CBufStream); *spSS.get() << resp; From d4c00a64a8a8234e7c7eee2512dcc5e8b42f36c8 Mon Sep 17 00:00:00 2001 From: OuYun Date: Fri, 20 Mar 2020 14:34:04 +0800 Subject: [PATCH 034/181] modify strategy of add fork node type enrollment --- src/bigbang/mqcluster.cpp | 5 +++-- src/bigbang/rpcmod.cpp | 1 + src/storage/mqdb.cpp | 14 ++++++++++---- src/storage/mqdb.h | 2 ++ 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 4d73eadb..c11e89c6 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -118,11 +118,12 @@ bool CMQCluster::HandleInvoke() { mapSuperNode.insert(make_pair(node.superNodeID, node.vecOwnedForks)); if (1 == node.vecOwnedForks.size() - && node.vecOwnedForks[0] == pCoreProtocol->GetGenesisBlockHash()) + && node.vecOwnedForks[0] == pCoreProtocol->GetGenesisBlockHash() + && 2 == node.nodeCat) { Log("dpos node of MQ: [%s]", node.superNodeID.c_str()); } - else + else if (1 == node.nodeCat) { Log("fork node of MQ: [%s]", node.superNodeID.c_str()); } diff --git a/src/bigbang/rpcmod.cpp b/src/bigbang/rpcmod.cpp index c98df374..f3ced640 100644 --- a/src/bigbang/rpcmod.cpp +++ b/src/bigbang/rpcmod.cpp @@ -2423,6 +2423,7 @@ CRPCResultPtr CRPCMod::RPCEnrollSuperNode(rpc::CRPCParamPtr param) storage::CSuperNode node; node.superNodeID = std::move(id); node.vecOwnedForks = std::move(forks); + node.nodeCat = nNodeCat; if(!pService->AddSuperNode(node)) { diff --git a/src/storage/mqdb.cpp b/src/storage/mqdb.cpp index 8ba5482c..d5f2bd90 100644 --- a/src/storage/mqdb.cpp +++ b/src/storage/mqdb.cpp @@ -43,10 +43,16 @@ void CSuperNodeDB::Deinitialize() bool CSuperNodeDB::AddNewSuperNode(const CSuperNode& cli) { - vector nodes; - ListSuperNode(nodes); - bool ret = Write(cli.superNodeID, cli.vecOwnedForks, true); //overwrite - ListSuperNode(nodes); + bool ret = false; + if (1 == cli.nodeCat) + { + Clear(); + ret = Write(cli.superNodeID, cli.vecOwnedForks, false); + } + else if (2 == cli.nodeCat) + { + ret = Write(cli.superNodeID, cli.vecOwnedForks, true); + } return ret; } diff --git a/src/storage/mqdb.h b/src/storage/mqdb.h index 7d0b9cb4..5546344d 100644 --- a/src/storage/mqdb.h +++ b/src/storage/mqdb.h @@ -20,6 +20,7 @@ class CSuperNode public: std::string superNodeID; std::vector vecOwnedForks; + int8 nodeCat; public: CSuperNode(std::string id = std::string(), std::vector forks = std::vector()) @@ -31,6 +32,7 @@ class CSuperNode { s.Serialize(superNodeID, opt); s.Serialize(vecOwnedForks, opt); + s.Serialize(nodeCat, opt); } }; From c6ccf53afbf108645bbcc459f3f5d0565e702ed9 Mon Sep 17 00:00:00 2001 From: OuYun Date: Fri, 20 Mar 2020 16:12:32 +0800 Subject: [PATCH 035/181] add returned enrollment list when issueing RPC command successfully --- script/template/rpc.json | 29 +++++++++++++++++++++++++---- src/bigbang/base.h | 1 + src/bigbang/mqcluster.cpp | 10 +--------- src/bigbang/rpcmod.cpp | 30 +++++++++++++++++++++++------- src/bigbang/service.cpp | 11 +++++++++++ src/bigbang/service.h | 1 + 6 files changed, 62 insertions(+), 20 deletions(-) diff --git a/script/template/rpc.json b/script/template/rpc.json index 91402f59..a699be36 100755 --- a/script/template/rpc.json +++ b/script/template/rpc.json @@ -664,9 +664,29 @@ } }, "response": { - "type": "bool", - "name": "retflag", - "desc": "successful or not" + "type": "array", + "name": "node", + "desc": "enrolled nodes previously", + "content": { + "node": { + "type": "object", + "content": { + "clientid": { + "type": "string", + "desc": "client id for mq either dpos node or fork nodes" + }, + "forks": { + "type": "array", + "content": { + "fork": { + "type": "string", + "desc": "fork which will be produced by specific fork node" + } + } + } + } + } + } }, "example": [ { @@ -681,7 +701,8 @@ "error": [ "{\"code\":-32600,\"message\":\"Invalid super node\"}", "{\"code\":-6,\"message\":\"Unknown fork\"}", - "{\"code\":-32603,\"message\":\"Add super node enrollment failed\"}" + "{\"code\":-32603,\"message\":\"Add super node enrollment failed\"}", + "{\"code\":-32603,\"message\":\"List super node enrollment failed\"}" ] }, "listpeer": { diff --git a/src/bigbang/base.h b/src/bigbang/base.h index 8eae2782..518a6967 100644 --- a/src/bigbang/base.h +++ b/src/bigbang/base.h @@ -322,6 +322,7 @@ class IService : public xengine::IBase /* Util */ virtual bool GetTxSender(const uint256& txid, CAddress& sender) = 0; virtual bool AddSuperNode(const storage::CSuperNode& node) = 0; + virtual bool ListSuperNode(std::vector& nodes) = 0; }; class IDataStat : public xengine::IIOModule diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index c11e89c6..9ebb4fcb 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -50,15 +50,7 @@ bool CMQCluster::IsAuthenticated() bool CMQCluster::HandleInitialize() { - if (NODE_CATEGORY::FORKNODE == catNode) - { - // clientID = "FORKNODE-01"; - } - else if (NODE_CATEGORY::DPOSNODE == catNode) - { - // clientID = "DPOSNODE"; - } - else if (NODE_CATEGORY::BBCNODE == catNode) + if (NODE_CATEGORY::BBCNODE == catNode) { Log("CMQCluster::HandleInitialize(): bbc node so bypass"); return true; diff --git a/src/bigbang/rpcmod.cpp b/src/bigbang/rpcmod.cpp index f3ced640..2533971f 100644 --- a/src/bigbang/rpcmod.cpp +++ b/src/bigbang/rpcmod.cpp @@ -14,11 +14,11 @@ //#include #include "address.h" +#include "mqdb.h" #include "rpc/auto_protocol.h" #include "template/proof.h" #include "template/template.h" #include "version.h" -#include "mqdb.h" using namespace std; using namespace xengine; @@ -2420,18 +2420,34 @@ CRPCResultPtr CRPCMod::RPCEnrollSuperNode(rpc::CRPCParamPtr param) } forks.emplace_back(fork); } - storage::CSuperNode node; - node.superNodeID = std::move(id); - node.vecOwnedForks = std::move(forks); - node.nodeCat = nNodeCat; + storage::CSuperNode newNode; + newNode.superNodeID = std::move(id); + newNode.vecOwnedForks = std::move(forks); + newNode.nodeCat = nNodeCat; - if(!pService->AddSuperNode(node)) + if (!pService->AddSuperNode(newNode)) { throw CRPCException(RPC_INTERNAL_ERROR, "Enroll super node failed"); } + vector nodes; + if (!pService->ListSuperNode(nodes)) + { + throw CRPCException(RPC_INTERNAL_ERROR, "List super nodes failed"); + } + auto spResult = MakeCEnrollSuperNodeResultPtr(); - spResult->fRetflag = true; + for (const auto& it : nodes) + { + CEnrollSuperNodeResult::CNode node; + node.strClientid = it.superNodeID; + for (const auto& fork : it.vecOwnedForks) + { + node.vecForks.push_back(fork.ToString()); + } + spResult->vecNode.push_back(node); + } + return spResult; } diff --git a/src/bigbang/service.cpp b/src/bigbang/service.cpp index 77b6f164..759f2009 100644 --- a/src/bigbang/service.cpp +++ b/src/bigbang/service.cpp @@ -702,6 +702,17 @@ bool CService::AddSuperNode(const storage::CSuperNode& node) return true; } +bool CService::ListSuperNode(std::vector& nodes) +{ + if (!pBlockChain->ListSuperNode(nodes)) + { + StdError("CService::ListSuperNode", "list super nodes failed."); + return false; + } + + return true; +} + CAddress CService::GetBackSender(const uint256& txid) { CTransaction tx; diff --git a/src/bigbang/service.h b/src/bigbang/service.h index d7ec96ba..a9690317 100644 --- a/src/bigbang/service.h +++ b/src/bigbang/service.h @@ -83,6 +83,7 @@ class CService : public IService /* Util */ bool GetTxSender(const uint256& txid, CAddress& sender) override; bool AddSuperNode(const storage::CSuperNode& node) override; + bool ListSuperNode(std::vector& nodes) override; protected: bool HandleInitialize() override; From 7a30869559bbde57dcf50949f0896ff4a53b4083 Mon Sep 17 00:00:00 2001 From: OuYun Date: Fri, 20 Mar 2020 16:28:58 +0800 Subject: [PATCH 036/181] fix adjudgement of invalid fork hash --- src/bigbang/rpcmod.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/bigbang/rpcmod.cpp b/src/bigbang/rpcmod.cpp index 2533971f..e0931c17 100644 --- a/src/bigbang/rpcmod.cpp +++ b/src/bigbang/rpcmod.cpp @@ -2413,6 +2413,10 @@ CRPCResultPtr CRPCMod::RPCEnrollSuperNode(rpc::CRPCParamPtr param) std::vector forks; for (const auto& i : vFork) { + if (64 != i.size()) + { + throw CRPCException(RPC_INVALID_PARAMETER, "Invalid fork hash"); + } uint256 fork; if (fork.SetHex(i) != i.size()) { From b626dc3432ac4db9154e0554cbf923018816a791 Mon Sep 17 00:00:00 2001 From: OuYun Date: Fri, 20 Mar 2020 16:53:32 +0800 Subject: [PATCH 037/181] fix updating issue for fork node --- src/bigbang/mqcluster.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 9ebb4fcb..9ed76e9e 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -263,6 +263,7 @@ bool CMQCluster::HandleEvent(CEventMQEnrollUpdate& eventMqUpdateEnroll) { boost::unique_lock lock(mtxStatus); + mapSuperNode.clear(); mapSuperNode.insert(make_pair(id, forks)); } condStatus.notify_all(); From f6de8657b4ec97b5892ca97ce9c4bddf0db92a22 Mon Sep 17 00:00:00 2001 From: OuYun Date: Tue, 24 Mar 2020 17:45:33 +0800 Subject: [PATCH 038/181] bugfix of null topic issue --- src/bigbang/mqcluster.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 9ed76e9e..36b310f1 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -174,6 +174,7 @@ bool CMQCluster::HandleInvoke() { //dpos node clientID = node.first; topicReqBlk = "Cluster01/+/SyncBlockReq"; + topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; Log("CMQCluster::HandleInvoke(): dpos node clientid [%s] with topic [%s]", clientID.c_str(), topicReqBlk.c_str()); } @@ -214,6 +215,7 @@ bool CMQCluster::HandleEvent(CEventMQSyncBlock& eventMqSyncBlock) bool CMQCluster::HandleEvent(CEventMQChainUpdate& eventMqUpdateChain) { + Log("CMQCluster::HandleEvent(): entering forking event handler"); CMqRollbackUpdate& update = eventMqUpdateChain.data; if (catNode != NODE_CATEGORY::DPOSNODE) @@ -231,12 +233,17 @@ bool CMQCluster::HandleEvent(CEventMQChainUpdate& eventMqUpdateChain) CBufferPtr spRBC(new CBufStream); *spRBC.get() << rbc; + Log("CMQCluster::HandleEvent(): rollback-topic[%s]:" + "forkheight[%d] forkhash[%s] shortlen[%d]", + topicRbBlk.c_str(), rbc.rbHeight, rbc.rbHash.ToString().c_str(), rbc.rbSize); + { boost::unique_lock lock(mtxSend); deqSendBuff.emplace_back(make_pair(topicRbBlk, spRBC)); } condSend.notify_all(); + Log("CMQCluster::HandleEvent(): exiting forking event handler"); return true; } @@ -281,6 +288,7 @@ bool CMQCluster::HandleEvent(CEventMQEnrollUpdate& eventMqUpdateEnroll) { //dpos node clientID = id; topicReqBlk = "Cluster01/+/SyncBlockReq"; + topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; Log("CMQCluster::HandleEvent(): dpos node clientid [%s] with topic [%s]", clientID.c_str(), topicReqBlk.c_str()); } @@ -486,6 +494,8 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) if (rb.rbHeight < lastHeightResp) { + Log("CMQCluster::OnReceiveMessage(): rbheight[%d], lastheight[%d]", + rb.rbHeight, int(lastHeightResp)); //check hard fork point uint256 hash; if (!pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), rb.rbHeight, hash)) @@ -495,6 +505,8 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) } bool fMatch = true; int nSync = rb.rbHeight - 1; + Log("CMQCluster::OnReceiveMessage(): rbhash[%s], lasthash[%s]", + rb.rbHash.ToString().c_str(), hash.ToString().c_str()); if (hash != rb.rbHash) { Log("CMQCluster::OnReceiveMessage(): hard fork block hash does not match"); From 191710fe2a14e600c2c5f720d17cc9f56839c12c Mon Sep 17 00:00:00 2001 From: OuYun Date: Wed, 25 Mar 2020 15:23:19 +0800 Subject: [PATCH 039/181] bugfix of order issue for rollback short chain at dispatcher module --- src/bigbang/dispatcher.cpp | 19 ++++----- src/bigbang/mqcluster.cpp | 84 ++++++++++++++++++++++++++------------ 2 files changed, 66 insertions(+), 37 deletions(-) diff --git a/src/bigbang/dispatcher.cpp b/src/bigbang/dispatcher.cpp index 6dbdd60e..b569b190 100644 --- a/src/bigbang/dispatcher.cpp +++ b/src/bigbang/dispatcher.cpp @@ -369,19 +369,18 @@ void CDispatcher::UpdatePrimaryBlock(const CBlock& block, const CBlockChainUpdat { pMqChainUpdate->data.shortLen = updateBlockChain.vBlockRemove.size(); pMqChainUpdate->data.vShort.reserve(updateBlockChain.vBlockRemove.size()); - bool fFirst = true; for (const auto& rb : updateBlockChain.vBlockRemove) { - if (fFirst) - { - pMqChainUpdate->data.triHeight = rb.GetBlockHeight() - 1; - pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), - pMqChainUpdate->data.triHeight, - pMqChainUpdate->data.triHash); - fFirst = false; - } - pMqChainUpdate->data.vShort.push_back(rb.GetHash()); + Log("CDispatcher::UpdatePrimaryBlock: removed short link block [%s]", + rb.GetHash().ToString().c_str()); + pMqChainUpdate->data.vShort.emplace_back(rb.GetHash()); } + pMqChainUpdate->data.triHeight = pMqChainUpdate->data.vShort.back().Get32(7) - 1; + pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), + pMqChainUpdate->data.triHeight, + pMqChainUpdate->data.triHash); + Log("CDispatcher::UpdatePrimaryBlock: forked block hash [%s] with height [%d]", + pMqChainUpdate->data.triHash.ToString().c_str(), pMqChainUpdate->data.triHeight); pMQCluster->PostEvent(pMqChainUpdate); } diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 36b310f1..4fa739e2 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -323,6 +323,8 @@ bool CMQCluster::LogEvent(const string& info) bool CMQCluster::PostBlockRequest(int syncHeight) { + Log("CMQCluster::PostBlockRequest(): posting request for block #%d", syncHeight); + if (mapSuperNode.empty()) { Log("CMQCluster::PostBlockRequest(): enrollment is empty for this fork node"); @@ -355,6 +357,7 @@ bool CMQCluster::PostBlockRequest(int syncHeight) } height = syncHeight; } + Log("CMQCluster::PostBlockRequest(): posting request for block hash[%s]", hash.ToString().c_str()); CSyncBlockRequest req; req.ipAddr = 16777343; //127.0.0.1 @@ -429,7 +432,7 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) } if (-1 == resp.height) - { //has reached the best height for the first time communication, + { //has reached the best height for the first time communication, // then set timer to process the following business rather than req/resp model nReqBlkTimerID = SetTimer(1000 * 30, boost::bind(&CMQCluster::RequestBlockTimerFunc, this, _1)); @@ -496,47 +499,73 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) { Log("CMQCluster::OnReceiveMessage(): rbheight[%d], lastheight[%d]", rb.rbHeight, int(lastHeightResp)); + //check hard fork point uint256 hash; - if (!pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), rb.rbHeight, hash)) + if (!pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), rb.rbHeight, hash) + || hash != rb.rbHash) { - Error("CMQCluster::OnReceiveMessage(): failed to get hard fork block hash"); + Log("CMQCluster::OnReceiveMessage(): rbhash[%s], lasthash[%s]", + rb.rbHash.ToString().c_str(), hash.ToString().c_str()); + Error("CMQCluster::OnReceiveMessage(): failed to get hard fork block hash or dismatch" + "then re-synchronize block from genesis one"); + if (!PostBlockRequest(0)) //re-sync from genesis block + { + Error("CMQCluster::OnReceiveMessage(): failed to post request while re-sync"); + return; + } return; } - bool fMatch = true; - int nSync = rb.rbHeight - 1; + + bool fMatch = false; Log("CMQCluster::OnReceiveMessage(): rbhash[%s], lasthash[%s]", rb.rbHash.ToString().c_str(), hash.ToString().c_str()); if (hash != rb.rbHash) { - Log("CMQCluster::OnReceiveMessage(): hard fork block hash does not match"); - fMatch = false; + Error("CMQCluster::OnReceiveMessage(): hard fork block hash does not match"); + return; } //check blocks in rollback - if (fMatch) + int nShort = 0; + for (int i = rb.rbSize; i > 0; --i) { - for (int i = 0; i < rb.rbSize; ++i) + if (!pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), + rb.rbHeight + rb.rbSize - i + 1, hash)) { - if (!pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), rb.rbHeight + i + 1, hash)) + if (i != rb.rbSize) { - Error("CMQCluster::OnReceiveMessage(): failed to get rollback block hash"); - return; + Log("CMQCluster::OnReceiveMessage(): exceed to get rollback block hash"); + break; } - if (hash == rb.hashList[i]) + else { - Log("CMQCluster::OnReceiveMessage(): fork node has not been rolled back yet"); - fMatch = false; - break; + Error("CMQCluster::OnReceiveMessage(): short chain does not match for one on dpos node"); + return; } - ++nSync; + } + if (hash == rb.hashList[i - 1]) + { + Log("CMQCluster::OnReceiveMessage(): fork node has not been rolled back yet" + " with hash [%s]", + hash.ToString().c_str()); + fMatch = true; + ++nShort; + continue; + } + else + { + Error("CMQCluster::OnReceiveMessage(): short chain does not match for one on dpos node"); + return; } } + Log("CMQCluster::OnReceiveMessage(): fork node rb[%d] against dpos node rb[%d]", nShort, rb.rbSize); //re-request from hard fork point to sync the best chain - if (!fMatch) + if (fMatch) { - lastHeightResp = nSync; + lastHeightResp = rb.rbHeight; + Log("CMQCluster::OnReceiveMessage(): rb.rbHeight[%d] against lastHeightResp[%d]", rb.rbHeight, int(lastHeightResp)); if (!PostBlockRequest(lastHeightResp)) { Error("CMQCluster::OnReceiveMessage(): failed to post request"); @@ -600,13 +629,13 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) if (req.lastHeight > best) { Error("CMQCluster::OnReceiveMessage(): block height owned by fork node " - "should not be greater than the best one on dpos node"); + "should not be greater than the best one on dpos node"); return; } else if (req.lastHeight == best) { Log("CMQCluster::OnReceiveMessage(): block height owned by fork node " - "has reached the best one on dpos node, please wait..."); + "has reached the best one on dpos node, please wait..."); resp.height = -1; resp.hash = uint256(); resp.isBest = 1; @@ -619,17 +648,19 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) uint256 hash; if (!pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), req.lastHeight, hash)) { - Error("CMQCluster::OnReceiveMessage(): failed to get checking height and hash match"); + Error("CMQCluster::OnReceiveMessage(): failed to get checking height and hash match " + "at height of #%d", req.lastHeight); return; } if (hash != req.lastHash) { - Error("CMQCluster::OnReceiveMessage(): height and hash do not match"); + Error("CMQCluster::OnReceiveMessage(): height and hash do not match hash[%s] vs. req.lastHash[%s] " + "at height of [%d]", hash.ToString().c_str(), req.lastHash.ToString().c_str(), req.lastHeight); return; } if (!pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), req.lastHeight + 1, hash)) { - Error("CMQCluster::OnReceiveMessage(): failed to get next block hash"); + Error("CMQCluster::OnReceiveMessage(): failed to get next block hash at height of #%d", req.lastHeight + 1); return; } CBlock block; @@ -643,15 +674,14 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) resp.height = req.lastHeight + 1; resp.hash = hash; resp.isBest = req.lastHeight + 1 < best - ? 0 - : 1; + ? 0 + : 1; Log("CMQCluster::OnReceiveMessage(): request[%d] best[%d] isBest[%d]", req.lastHeight + 1, best, resp.isBest); resp.blockSize = xengine::GetSerializeSize(block); resp.block = move(block); } - CBufferPtr spSS(new CBufStream); *spSS.get() << resp; string topicRsp = "Cluster01/" + req.forkNodeId + "/SyncBlockResp"; From 4507499d2b0a749700a4fb96cc47ac157d65f060 Mon Sep 17 00:00:00 2001 From: OuYun Date: Wed, 25 Mar 2020 17:20:22 +0800 Subject: [PATCH 040/181] add facilities to correctly deal with request for long chain --- src/bigbang/mqcluster.cpp | 34 ++++++++++++++++++++++++++++++---- src/bigbang/mqcluster.h | 4 ++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 4fa739e2..8761f73e 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -27,7 +27,8 @@ CMQCluster::CMQCluster(int catNodeIn) fAuth(false), fAbort(false), srvAddr("tcp://localhost:1883"), - nReqBlkTimerID(0) + nReqBlkTimerID(0), + nRollNum(0) { switch (catNodeIn) { @@ -350,10 +351,18 @@ bool CMQCluster::PostBlockRequest(int syncHeight) } else { - if (!pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), syncHeight, hash)) + if (nRollNum) { - Error("CMQCluster::PostBlockRequest(): failed to get specific block"); - return false; + boost::unique_lock lock(mtxRoll); + hash = vLongFork.back(); + } + else + { + if (!pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), syncHeight, hash)) + { + Error("CMQCluster::PostBlockRequest(): failed to get specific block"); + return false; + } } height = syncHeight; } @@ -459,6 +468,21 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) } lastHeightResp = resp.height; + //check if there are rollbacked blocks + if (nRollNum) + { + boost::unique_lock lock(mtxRoll); + if (vLongFork.size() < nRollNum) + { + vLongFork.emplace_back(resp.hash); + } + else + { + vLongFork.clear(); + nRollNum = 0; + } + } + //iterate to retrieve next one if (resp.isBest) @@ -569,8 +593,10 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) if (!PostBlockRequest(lastHeightResp)) { Error("CMQCluster::OnReceiveMessage(): failed to post request"); + nRollNum = nShort; return; } + nRollNum = nShort; } } } diff --git a/src/bigbang/mqcluster.h b/src/bigbang/mqcluster.h index 2a95eed0..6d5112f9 100644 --- a/src/bigbang/mqcluster.h +++ b/src/bigbang/mqcluster.h @@ -178,6 +178,10 @@ class CMQCluster : public IMQCluster std::atomic lastHeightResp; uint32 nReqBlkTimerID; + + boost::mutex mtxRoll; + std::vector vLongFork; + std::atomic nRollNum; }; } // namespace bigbang From e38d8927cdc6e507a16a9ed207c7d9e4de7dd4c3 Mon Sep 17 00:00:00 2001 From: OuYun Date: Thu, 26 Mar 2020 16:08:10 +0800 Subject: [PATCH 041/181] typo correction --- src/xengine/http/httpget.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/xengine/http/httpget.cpp b/src/xengine/http/httpget.cpp index 8722edaa..cc12410b 100644 --- a/src/xengine/http/httpget.cpp +++ b/src/xengine/http/httpget.cpp @@ -9,7 +9,7 @@ #include "httputil.h" #include "netio/netio.h" -#define HTTPGET_CONNET_TIMEOUT 10 +#define HTTPGET_CONNECT_TIMEOUT 10 using namespace std; using boost::asio::ip::tcp; @@ -271,7 +271,7 @@ void CHttpGet::HostResolved(const CNetHost& host, const tcp::endpoint& ep) if (it != mapRequest.upper_bound(host)) { CHttpReqData& httpReqData = (*it).second.data; - if (SSLConnect(ep, HTTPGET_CONNET_TIMEOUT, GetSSLOption(httpReqData, host.strHost))) + if (SSLConnect(ep, HTTPGET_CONNECT_TIMEOUT, GetSSLOption(httpReqData, host.strHost))) { mapRequest.insert(make_pair(CNetHost(ep), (*it).second)); } @@ -413,7 +413,7 @@ bool CHttpGet::HandleEvent(CEventHttpGet& eventGet) tcp::endpoint ep = host.ToEndPoint(); if (ep != tcp::endpoint()) { - if (!SSLConnect(ep, HTTPGET_CONNET_TIMEOUT, GetSSLOption(httpReqData, host.strHost))) + if (!SSLConnect(ep, HTTPGET_CONNECT_TIMEOUT, GetSSLOption(httpReqData, host.strHost))) { PostError(httpReqData.strIOModule, nNonce, HTTPGET_CONNECT_FAILED); return true; From bc9770c5062cc815623c1e14e793903475ceb382 Mon Sep 17 00:00:00 2001 From: OuYun Date: Thu, 26 Mar 2020 17:46:01 +0800 Subject: [PATCH 042/181] restrict getwork/submitwork from/to fork nodes --- script/template/rpc.json | 7 ++++++- src/bigbang/miner.cpp | 2 +- src/bigbang/rpcmod.cpp | 14 +++++++++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/script/template/rpc.json b/script/template/rpc.json index a699be36..c7db0298 100755 --- a/script/template/rpc.json +++ b/script/template/rpc.json @@ -3079,7 +3079,11 @@ } ], "error": [ - "{\"code\" : -32603, \"message\" : \"The primary chain is invalid.\"}" + "{\"code\" : -32600, \"message\" : \"Invalid request for non-super-node.\"}", + "{\"code\" : -4, \"message\" : \"Invalid spent address.\"}", + "{\"code\" : -4, \"message\" : \"Invalid private key.\"}", + "{\"code\" : -4, \"message\" : \"Mining signature private key and spent address cannot be from the same.\"}", + "{\"code\" : -4, \"message\" : \"Invalid mint template.\"}" ] }, "submitwork": { @@ -3123,6 +3127,7 @@ } ], "error": [ + "{\"code\" : -32600, \"message\" : \"Invalid request for non-super-node.\"}", "{\"code\" : -4, \"message\" : \"Invalid spent address\"}", "{\"code\" : -4, \"message\" : \"Invalid private key\"}", "{\"code\" : -4, \"message\" : \"Invalid mint template\"}", diff --git a/src/bigbang/miner.cpp b/src/bigbang/miner.cpp index 338bbd7b..1335bee7 100644 --- a/src/bigbang/miner.cpp +++ b/src/bigbang/miner.cpp @@ -252,7 +252,7 @@ bool CMiner::SendRequest(uint64 nNonce, const string& content) CHttpReqData& httpReqData = eventHttpGet.data; httpReqData.strIOModule = GetOwnKey(); httpReqData.nTimeout = Config()->nRPCConnectTimeout; - ; + if (Config()->fRPCSSLEnable) { httpReqData.strProtocol = "https"; diff --git a/src/bigbang/rpcmod.cpp b/src/bigbang/rpcmod.cpp index e0931c17..7fe2db20 100644 --- a/src/bigbang/rpcmod.cpp +++ b/src/bigbang/rpcmod.cpp @@ -2458,6 +2458,12 @@ CRPCResultPtr CRPCMod::RPCEnrollSuperNode(rpc::CRPCParamPtr param) // /* Mint */ CRPCResultPtr CRPCMod::RPCGetWork(CRPCParamPtr param) { + int nNodeCat = dynamic_cast(Config())->nCatOfNode; + if (1 == nNodeCat) + { + throw CRPCException(RPC_INVALID_REQUEST, "This fork node has not feature of POW mining"); + } + //getwork <"spent"> <"privkey"> ("prev") auto spParam = CastParamPtr(param); @@ -2475,7 +2481,7 @@ CRPCResultPtr CRPCMod::RPCGetWork(CRPCParamPtr param) crypto::CPubKey pubkeySpent; if (addrSpent.GetPubKey(pubkeySpent) && pubkeySpent == key.GetPubKey()) { - throw CRPCException(RPC_INVALID_ADDRESS_OR_KEY, "Invalid spent address or private key"); + throw CRPCException(RPC_INVALID_ADDRESS_OR_KEY, "Mining signature private key and spent address cannot be from the same."); } CTemplateMintPtr ptr = CTemplateMint::CreateTemplatePtr(new CTemplateProof(key.GetPubKey(), static_cast(addrSpent))); if (ptr == nullptr) @@ -2510,6 +2516,12 @@ CRPCResultPtr CRPCMod::RPCGetWork(CRPCParamPtr param) CRPCResultPtr CRPCMod::RPCSubmitWork(CRPCParamPtr param) { + int nNodeCat = dynamic_cast(Config())->nCatOfNode; + if (1 == nNodeCat) + { + throw CRPCException(RPC_INVALID_REQUEST, "This fork node has not feature of POW mining"); + } + auto spParam = CastParamPtr(param); vector vchWorkData(ParseHexString(spParam->strData)); CAddress addrSpent(spParam->strSpent); From d13674614e27a60a7580c34cc6852e28307900ab Mon Sep 17 00:00:00 2001 From: OuYun Date: Thu, 26 Mar 2020 17:48:58 +0800 Subject: [PATCH 043/181] update rpc template --- script/template/rpc.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/template/rpc.json b/script/template/rpc.json index c7db0298..12f38a84 100755 --- a/script/template/rpc.json +++ b/script/template/rpc.json @@ -3079,7 +3079,7 @@ } ], "error": [ - "{\"code\" : -32600, \"message\" : \"Invalid request for non-super-node.\"}", + "{\"code\" : -32600, \"message\" : \"This fork node has not feature of POW mining.\"}", "{\"code\" : -4, \"message\" : \"Invalid spent address.\"}", "{\"code\" : -4, \"message\" : \"Invalid private key.\"}", "{\"code\" : -4, \"message\" : \"Mining signature private key and spent address cannot be from the same.\"}", @@ -3127,7 +3127,7 @@ } ], "error": [ - "{\"code\" : -32600, \"message\" : \"Invalid request for non-super-node.\"}", + "{\"code\" : -32600, \"message\" : \"This fork node has not feature of POW mining.\"}", "{\"code\" : -4, \"message\" : \"Invalid spent address\"}", "{\"code\" : -4, \"message\" : \"Invalid private key\"}", "{\"code\" : -4, \"message\" : \"Invalid mint template\"}", From 97f9abc642ea64db530ea1cd1a6710fb957d55e6 Mon Sep 17 00:00:00 2001 From: OuYun Date: Thu, 26 Mar 2020 18:13:27 +0800 Subject: [PATCH 044/181] disable POW mining on fork nodes --- src/bigbang/blockmaker.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/bigbang/blockmaker.cpp b/src/bigbang/blockmaker.cpp index 5f9b085b..c0faf123 100644 --- a/src/bigbang/blockmaker.cpp +++ b/src/bigbang/blockmaker.cpp @@ -130,7 +130,8 @@ bool CBlockMaker::HandleInitialize() // } // } - if (!MintConfig()->destCryptonight.IsNull() && MintConfig()->keyCryptonight != 0) + int nNodeCat = dynamic_cast(Config())->nCatOfNode; + if (!MintConfig()->destCryptonight.IsNull() && MintConfig()->keyCryptonight != 0 && 1 != nNodeCat) { CBlockMakerProfile profile(CM_CRYPTONIGHT, MintConfig()->destCryptonight, MintConfig()->keyCryptonight); if (profile.IsValid()) @@ -317,7 +318,7 @@ bool CBlockMaker::SignBlock(CBlock& block, const CBlockMakerProfile& profile) bool CBlockMaker::DispatchBlock(CBlock& block) { -/* int nWait = block.nTimeStamp - GetNetTime(); + /* int nWait = block.nTimeStamp - GetNetTime(); if (nWait > 0 && !Wait(nWait)) { StdTrace("blockmaker", "Wait failed nWait: %d", nWait); From e6083f5b8d873869351b7cea51338614497bee36 Mon Sep 17 00:00:00 2001 From: OuYun Date: Fri, 27 Mar 2020 19:24:46 +0800 Subject: [PATCH 045/181] modification for fork --- src/bigbang/dispatcher.cpp | 28 +++++++------- src/bigbang/forkmanager.cpp | 76 ++++++++++++++++++------------------- src/bigbang/struct.h | 4 +- src/common/CMakeLists.txt | 2 +- src/storage/blockbase.cpp | 2 +- 5 files changed, 56 insertions(+), 56 deletions(-) diff --git a/src/bigbang/dispatcher.cpp b/src/bigbang/dispatcher.cpp index b569b190..72e4857d 100644 --- a/src/bigbang/dispatcher.cpp +++ b/src/bigbang/dispatcher.cpp @@ -142,7 +142,7 @@ bool CDispatcher::HandleInvoke() vector vActive; if (!pForkManager->LoadForkContext(vActive)) { - Error("Failed to load for context"); + Error("Failed to load fork context"); return false; } @@ -224,21 +224,21 @@ Errno CDispatcher::AddNewBlock(const CBlock& block, uint64 nNonce) pService->NotifyBlockChainUpdate(updateBlockChain); - // if (!block.IsVacant()) - // { - // vector vActive, vDeactive; - // pForkManager->ForkUpdate(updateBlockChain, vActive, vDeactive); + if (!block.IsVacant()) + { + vector vActive, vDeactive; + pForkManager->ForkUpdate(updateBlockChain, vActive, vDeactive); - // for (const uint256 hashFork : vActive) - // { - // ActivateFork(hashFork, nNonce); - // } + for (const uint256 hashFork : vActive) + { + ActivateFork(hashFork, nNonce); + } - // for (const uint256 hashFork : vDeactive) - // { - // pNetChannel->UnsubscribeFork(hashFork); - // } - // } + for (const uint256 hashFork : vDeactive) + { + pNetChannel->UnsubscribeFork(hashFork); + } + } if (block.IsPrimary()) { diff --git a/src/bigbang/forkmanager.cpp b/src/bigbang/forkmanager.cpp index 98bc8f11..12304847 100644 --- a/src/bigbang/forkmanager.cpp +++ b/src/bigbang/forkmanager.cpp @@ -6,7 +6,7 @@ #include -// #include "template/fork.h" +#include "template/fork.h" using namespace std; using namespace xengine; @@ -136,43 +136,43 @@ bool CForkManager::LoadForkContext(vector& vActive) void CForkManager::ForkUpdate(const CBlockChainUpdate& update, vector& vActive, vector& vDeactive) { - // boost::unique_lock wlock(rwAccess); - - // CForkSchedule& sched = mapForkSched[update.hashFork]; - // if (!sched.IsJointEmpty()) - // { - // for (const CBlockEx& block : boost::adaptors::reverse(update.vBlockAddNew)) - // { - // if (!block.IsExtended() && !block.IsVacant()) - // { - // sched.RemoveJoint(block.GetHash(), vActive); - // if (sched.IsHalted()) - // { - // vDeactive.push_back(update.hashFork); - // } - // } - // } - // } - // if (update.hashFork == pCoreProtocol->GetGenesisBlockHash()) - // { - // for (const CBlockEx& block : boost::adaptors::reverse(update.vBlockAddNew)) - // { - // for (const CTransaction& tx : block.vtx) - // { - // CTemplateId tid; - // if (tx.sendTo.GetTemplateId(tid) && tid.GetType() == TEMPLATE_FORK - // && !tx.vchData.empty() - // && tx.nAmount >= CTemplateFork::LockedCoin(update.nLastBlockHeight)) - // { - // CForkContext ctxt; - // if (pBlockChain->AddNewForkContext(tx, ctxt) == OK) - // { - // AddNewForkContext(ctxt, vActive); - // } - // } - // } - // } - // } + boost::unique_lock wlock(rwAccess); + + CForkSchedule& sched = mapForkSched[update.hashFork]; + if (!sched.IsJointEmpty()) + { + for (const CBlockEx& block : boost::adaptors::reverse(update.vBlockAddNew)) + { + if (!block.IsExtended() && !block.IsVacant()) + { + sched.RemoveJoint(block.GetHash(), vActive); + if (sched.IsHalted()) + { + vDeactive.push_back(update.hashFork); + } + } + } + } + if (update.hashFork == pCoreProtocol->GetGenesisBlockHash()) + { + for (const CBlockEx& block : boost::adaptors::reverse(update.vBlockAddNew)) + { + for (const CTransaction& tx : block.vtx) + { + CTemplateId tid; + if (tx.sendTo.GetTemplateId(tid) && tid.GetType() == TEMPLATE_FORK + && !tx.vchData.empty() + && tx.nAmount >= CTemplateFork::LockedCoin(update.nLastBlockHeight)) + { + CForkContext ctxt; + if (pBlockChain->AddNewForkContext(tx, ctxt) == OK) + { + AddNewForkContext(ctxt, vActive); + } + } + } + } + } } bool CForkManager::AddNewForkContext(const CForkContext& ctxt, vector& vActive) diff --git a/src/bigbang/struct.h b/src/bigbang/struct.h index 5e4c0542..8694fc8e 100644 --- a/src/bigbang/struct.h +++ b/src/bigbang/struct.h @@ -117,10 +117,10 @@ class CBlockChainUpdate public: uint256 hashFork; uint256 hashParent; - int nOriginHeight; + int32 nOriginHeight; uint256 hashLastBlock; int64 nLastBlockTime; - int nLastBlockHeight; + int32 nLastBlockHeight; int64 nMoneySupply; std::set setTxUpdate; std::vector vBlockAddNew; diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 9b3ba40d..dfefd67e 100755 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -11,7 +11,7 @@ set(template template/template.h template/template.cpp template/weighted.h template/weighted.cpp template/multisig.h template/multisig.cpp - # template/fork.h template/fork.cpp + template/fork.h template/fork.cpp template/mint.h template/mint.cpp template/proof.h template/proof.cpp # template/delegate.h template/delegate.cpp diff --git a/src/storage/blockbase.cpp b/src/storage/blockbase.cpp index 6683c8de..246e352b 100644 --- a/src/storage/blockbase.cpp +++ b/src/storage/blockbase.cpp @@ -396,7 +396,7 @@ bool CBlockBase::Initiate(const uint256& hashGenesis, const CBlock& blockGenesis CForkContext ctxt(hashGenesis, uint64(0), uint64(0), profile); if (!dbBlock.AddNewForkContext(ctxt)) { - StdTrace("BlockBase", "Add New Fork COntext %s block failed", hashGenesis.ToString().c_str()); + StdTrace("BlockBase", "Add New Fork Context %s block failed", hashGenesis.ToString().c_str()); return false; } From 1bed360f46267ee196af83a5c61588b41bebf8cf Mon Sep 17 00:00:00 2001 From: OuYun Date: Tue, 31 Mar 2020 17:07:12 +0800 Subject: [PATCH 046/181] add ip info to supernode storage for global purpose and update enrollsupernode rpc command accordingly --- script/template/rpc.json | 6 ++++++ src/bigbang/mqcluster.cpp | 2 +- src/bigbang/rpcmod.cpp | 10 +++++++++- src/storage/mqdb.h | 32 +++++++++++++++++++++++++++++--- 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/script/template/rpc.json b/script/template/rpc.json index 12f38a84..c2364ea0 100755 --- a/script/template/rpc.json +++ b/script/template/rpc.json @@ -650,6 +650,11 @@ "type": "string", "desc": "client id for dpos node or fork nodes" }, + "clientip" : + { + "type": "string", + "desc": "client global ip address for dpos node or fork nodes" + }, "forks" : { "type" : "array", @@ -700,6 +705,7 @@ ], "error": [ "{\"code\":-32600,\"message\":\"Invalid super node\"}", + "{\"code\":-6,\"message\":\"Invalid ip address\"}", "{\"code\":-6,\"message\":\"Unknown fork\"}", "{\"code\":-32603,\"message\":\"Add super node enrollment failed\"}", "{\"code\":-32603,\"message\":\"List super node enrollment failed\"}" diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 8761f73e..77c697bf 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -644,7 +644,7 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) //add this requesting fork node to active list if (!mapActiveSuperNode.count(req.ipAddr)) { - mapActiveSuperNode[req.ipAddr] = storage::CSuperNode(req.forkNodeId, req.forkList); + mapActiveSuperNode[req.ipAddr] = storage::CSuperNode(req.forkNodeId, req.ipAddr, req.forkList); } //processing request from fork node diff --git a/src/bigbang/rpcmod.cpp b/src/bigbang/rpcmod.cpp index 7fe2db20..4d4aedc9 100644 --- a/src/bigbang/rpcmod.cpp +++ b/src/bigbang/rpcmod.cpp @@ -2407,8 +2407,15 @@ CRPCResultPtr CRPCMod::RPCEnrollSuperNode(rpc::CRPCParamPtr param) } auto spParam = CastParamPtr(param); - std::vector vFork = spParam->vecForks; + std::string ipStr = spParam->strClientip; + unsigned long ipNum = 0; + if (!storage::CSuperNode::Ip2Int(ipStr, ipNum)) + { + throw CRPCException(RPC_INVALID_PARAMETER, "Invalid ip address"); + } + std::string id = spParam->strClientid; + std::vector vFork = spParam->vecForks; std::vector forks; for (const auto& i : vFork) @@ -2426,6 +2433,7 @@ CRPCResultPtr CRPCMod::RPCEnrollSuperNode(rpc::CRPCParamPtr param) } storage::CSuperNode newNode; newNode.superNodeID = std::move(id); + newNode.ipAddr = ipNum; newNode.vecOwnedForks = std::move(forks); newNode.nodeCat = nNodeCat; diff --git a/src/storage/mqdb.h b/src/storage/mqdb.h index 5546344d..ee70bea0 100644 --- a/src/storage/mqdb.h +++ b/src/storage/mqdb.h @@ -7,6 +7,7 @@ #include "uint256.h" #include "xengine.h" +#include namespace bigbang { @@ -19,18 +20,43 @@ class CSuperNode public: std::string superNodeID; + uint32 ipAddr; std::vector vecOwnedForks; int8 nodeCat; public: - CSuperNode(std::string id = std::string(), std::vector forks = std::vector()) - : superNodeID(id), vecOwnedForks(forks) {} + CSuperNode(std::string id = std::string(), uint32 ip = 0, + std::vector forks = std::vector(), int8 cat = 0) + : superNodeID(id), + ipAddr(ip), + vecOwnedForks(forks), + nodeCat(cat) {} + + static bool Ip2Int(const std::string& ipStr, unsigned long& ipNum) + { + boost::system::error_code ec; + boost::asio::ip::address_v4 addr = boost::asio::ip::address_v4::from_string(ipStr, ec); + if (!ec) + { + ipNum = addr.to_ulong(); + return true; + } + return false; + } + + static bool Int2Ip(const unsigned long& ipNum, std::string& ipStr) + { + boost::asio::ip::address_v4 addr(ipNum); + ipStr = addr.to_string(); + return true; + } protected: template void Serialize(xengine::CStream& s, O& opt) { s.Serialize(superNodeID, opt); + s.Serialize(ipAddr, opt); s.Serialize(vecOwnedForks, opt); s.Serialize(nodeCat, opt); } @@ -50,7 +76,7 @@ class CSuperNodeDB : public xengine::CKVDB protected: bool LoadSuperNodeWalker(xengine::CBufStream& ssKey, xengine::CBufStream& ssValue, - std::map>& mapCli); + std::map>& mapCli); }; } // namespace storage From a57c1fef971ddbaff688b6e32f859dc8a191902b Mon Sep 17 00:00:00 2001 From: OuYun Date: Wed, 1 Apr 2020 10:12:53 +0800 Subject: [PATCH 047/181] update fork node ip address related facilities --- src/bigbang/mqcluster.cpp | 14 +++++++------- src/bigbang/mqcluster.h | 11 ++++++++--- src/bigbang/service.cpp | 1 + src/bigbang/struct.h | 1 + 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 77c697bf..5ed494f5 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -26,7 +26,7 @@ CMQCluster::CMQCluster(int catNodeIn) pService(nullptr), fAuth(false), fAbort(false), - srvAddr("tcp://localhost:1883"), + addrBroker("tcp://localhost:1883"), nReqBlkTimerID(0), nRollNum(0) { @@ -255,12 +255,13 @@ bool CMQCluster::HandleEvent(CEventMQEnrollUpdate& eventMqUpdateEnroll) if (NODE_CATEGORY::FORKNODE == catNode) { clientID = id; + ipAddr = eventMqUpdateEnroll.data.ipAddr; topicReqBlk = "Cluster01/" + clientID + "/SyncBlockReq"; topicRespBlk = "Cluster01/" + clientID + "/SyncBlockResp"; topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; - Log("CMQCluster::HandleEvent(): fork node clientid [%s] with topics:" + Log("CMQCluster::HandleEvent(): fork node clientid [%s] ip [%d] with topics:" "\n[%s]\n[%s]", - clientID.c_str(), + clientID.c_str(), ipAddr, topicRespBlk.c_str(), topicRbBlk.c_str()); for (const auto& fork : forks) { @@ -369,8 +370,7 @@ bool CMQCluster::PostBlockRequest(int syncHeight) Log("CMQCluster::PostBlockRequest(): posting request for block hash[%s]", hash.ToString().c_str()); CSyncBlockRequest req; - req.ipAddr = 16777343; //127.0.0.1 - //req.ipAddr = 1111638320; //"0ABB" + req.ipAddr = ipAddr; //16777343 - 127.0.0.1; 1111638320 - "0ABB" req.forkNodeIdLen = clientID.size(); req.forkNodeId = clientID; auto enroll = mapSuperNode.begin(); @@ -869,7 +869,7 @@ bool CMQCluster::ClientAgent(MQ_CLI_ACTION action) { try { - static mqtt::async_client client(srvAddr, clientID); + static mqtt::async_client client(addrBroker, clientID); static mqtt::connect_options connOpts; connOpts.set_keep_alive_interval(20); @@ -884,7 +884,7 @@ bool CMQCluster::ClientAgent(MQ_CLI_ACTION action) { case MQ_CLI_ACTION::CONN: { - cout << "Initializing for server '" << srvAddr << "'..." << endl; + cout << "Initializing for server '" << addrBroker << "'..." << endl; client.set_callback(cb); cout << " ...OK" << endl; diff --git a/src/bigbang/mqcluster.h b/src/bigbang/mqcluster.h index 6d5112f9..c0888d88 100644 --- a/src/bigbang/mqcluster.h +++ b/src/bigbang/mqcluster.h @@ -156,17 +156,22 @@ class CMQCluster : public IMQCluster void OnReceiveMessage(const std::string& topic, CBufStream& payload); bool ClientAgent(MQ_CLI_ACTION action); void MqttThreadFunc(); + bool fAuth; bool fAbort; - string srvAddr; - string clientID; + string addrBroker; + NODE_CATEGORY catNode; + string topicReqBlk; string topicRespBlk; string topicRbBlk; - NODE_CATEGORY catNode; + boost::mutex mtxStatus; boost::condition_variable condStatus; std::map> mapSuperNode; + string clientID; + uint32 ipAddr; + std::map mapActiveSuperNode; boost::mutex mtxSend; diff --git a/src/bigbang/service.cpp b/src/bigbang/service.cpp index 759f2009..b64d262c 100644 --- a/src/bigbang/service.cpp +++ b/src/bigbang/service.cpp @@ -695,6 +695,7 @@ bool CService::AddSuperNode(const storage::CSuperNode& node) if (nullptr != pEvent) { pEvent->data.superNodeClientID = node.superNodeID; + pEvent->data.ipAddr = node.ipAddr; pEvent->data.vecForksOwned = node.vecOwnedForks; pMQCluster->PostEvent(pEvent); } diff --git a/src/bigbang/struct.h b/src/bigbang/struct.h index 8694fc8e..e1b47339 100644 --- a/src/bigbang/struct.h +++ b/src/bigbang/struct.h @@ -269,6 +269,7 @@ class CMqSuperNodeUpdate { public: std::string superNodeClientID; + uint32 ipAddr; std::vector vecForksOwned; }; From fe93c62aaca550900a807bcb7102b409906ae033 Mon Sep 17 00:00:00 2001 From: OuYun Date: Wed, 1 Apr 2020 13:26:30 +0800 Subject: [PATCH 048/181] add notification to set fork filter as finishing enrolling a fork node in supernode cluster --- src/bigbang/base.h | 2 +- src/bigbang/forkmanager.cpp | 26 ++++++++++++++++++++++++++ src/bigbang/forkmanager.h | 1 + src/bigbang/service.cpp | 2 ++ 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/bigbang/base.h b/src/bigbang/base.h index 518a6967..6c9ad902 100644 --- a/src/bigbang/base.h +++ b/src/bigbang/base.h @@ -147,7 +147,7 @@ class IForkManager : public xengine::IBase virtual void ForkUpdate(const CBlockChainUpdate& update, std::vector& vActive, std::vector& vDeactive) = 0; virtual void GetForkList(std::vector& vFork) const = 0; virtual bool GetSubline(const uint256& hashFork, std::vector>& vSubline) const = 0; - const CForkConfig* ForkConfig() + virtual bool SetForkFilter(const std::vector& vFork = std::vector(), const std::vector& vGroup = std::vector()) = 0; const CForkConfig* ForkConfig() { return dynamic_cast(xengine::IBase::Config()); } diff --git a/src/bigbang/forkmanager.cpp b/src/bigbang/forkmanager.cpp index 12304847..510e7e27 100644 --- a/src/bigbang/forkmanager.cpp +++ b/src/bigbang/forkmanager.cpp @@ -258,6 +258,32 @@ bool CForkManager::GetSubline(const uint256& hashFork, vector return true; } +bool CForkManager::SetForkFilter(const std::vector& vFork, const std::vector& vGroup) +{ + boost::unique_lock wlock(rwAccess); + + if (!fAllowAnyFork) + { + for (auto const& fork : vFork) + { + if (fork != 0) + { + setForkAllowed.insert(fork); + } + } + for (auto const& group : vGroup) + { + if (group != 0) + { + setGroupAllowed.insert(group); + } + } + } + + return true; +} + + bool CForkManager::IsAllowedFork(const uint256& hashFork, const uint256& hashParent) const { if (fAllowAnyFork || setForkAllowed.count(hashFork) || setGroupAllowed.count(hashFork)) diff --git a/src/bigbang/forkmanager.h b/src/bigbang/forkmanager.h index 158c5e96..d956f773 100644 --- a/src/bigbang/forkmanager.h +++ b/src/bigbang/forkmanager.h @@ -106,6 +106,7 @@ class CForkManager : public IForkManager bool AddNewForkContext(const CForkContext& ctxt, std::vector& vActive); void GetForkList(std::vector& vFork) const override; bool GetSubline(const uint256& hashFork, std::vector>& vSubline) const override; + bool SetForkFilter(const std::vector& vFork = std::vector(), const std::vector& vGroup = std::vector()) override; protected: bool HandleInitialize() override; diff --git a/src/bigbang/service.cpp b/src/bigbang/service.cpp index b64d262c..c98ca494 100644 --- a/src/bigbang/service.cpp +++ b/src/bigbang/service.cpp @@ -700,6 +700,8 @@ bool CService::AddSuperNode(const storage::CSuperNode& node) pMQCluster->PostEvent(pEvent); } + pForkManager->SetForkFilter(node.vecOwnedForks); + return true; } From 4db69827382484ddeea9667d607be03dfe99671c Mon Sep 17 00:00:00 2001 From: OuYun Date: Wed, 1 Apr 2020 13:58:22 +0800 Subject: [PATCH 049/181] update RPC enrollsupernode to add output of fork node ip address --- script/template/rpc.json | 7 ++++++- src/bigbang/rpcmod.cpp | 6 ++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/script/template/rpc.json b/script/template/rpc.json index c2364ea0..4c799371 100755 --- a/script/template/rpc.json +++ b/script/template/rpc.json @@ -680,6 +680,10 @@ "type": "string", "desc": "client id for mq either dpos node or fork nodes" }, + "clientip": { + "type": "string", + "desc": "client ip address for fork nodes to engage in p2p communication" + }, "forks": { "type": "array", "content": { @@ -708,7 +712,8 @@ "{\"code\":-6,\"message\":\"Invalid ip address\"}", "{\"code\":-6,\"message\":\"Unknown fork\"}", "{\"code\":-32603,\"message\":\"Add super node enrollment failed\"}", - "{\"code\":-32603,\"message\":\"List super node enrollment failed\"}" + "{\"code\":-32603,\"message\":\"List super node enrollment failed\"}", + "{\"code\":-32603,\"message\":\"Failed to convert ip address\"}" ] }, "listpeer": { diff --git a/src/bigbang/rpcmod.cpp b/src/bigbang/rpcmod.cpp index 4d4aedc9..67af6381 100644 --- a/src/bigbang/rpcmod.cpp +++ b/src/bigbang/rpcmod.cpp @@ -2453,6 +2453,12 @@ CRPCResultPtr CRPCMod::RPCEnrollSuperNode(rpc::CRPCParamPtr param) { CEnrollSuperNodeResult::CNode node; node.strClientid = it.superNodeID; + string strAddr; + if (!storage::CSuperNode::Int2Ip(it.ipAddr, strAddr)) + { + throw CRPCException(RPC_INTERNAL_ERROR, "Failed to convert ip address"); + } + node.strClientip = strAddr; for (const auto& fork : it.vecOwnedForks) { node.vecForks.push_back(fork.ToString()); From 83175c16091fac4f3f64a2c19ccfb59427389a94 Mon Sep 17 00:00:00 2001 From: OuYun Date: Wed, 1 Apr 2020 16:49:57 +0800 Subject: [PATCH 050/181] add interface to storage component for supernode and update some related invoking --- src/bigbang/base.h | 1 + src/bigbang/blockchain.cpp | 10 ++++ src/bigbang/blockchain.h | 1 + src/bigbang/mqcluster.cpp | 7 ++- src/storage/blockbase.cpp | 23 ++++++++- src/storage/blockbase.h | 1 + src/storage/blockdb.cpp | 5 ++ src/storage/blockdb.h | 1 + src/storage/mqdb.cpp | 96 +++++++++++++++++++++++++++++++++----- src/storage/mqdb.h | 15 ++++-- 10 files changed, 137 insertions(+), 23 deletions(-) diff --git a/src/bigbang/base.h b/src/bigbang/base.h index 6c9ad902..0dadaf55 100644 --- a/src/bigbang/base.h +++ b/src/bigbang/base.h @@ -92,6 +92,7 @@ class IBlockChain : public xengine::IBase virtual Errno AddNewOrigin(const CBlock& block, CBlockChainUpdate& update) = 0; virtual Errno AddNewSuperNode(const storage::CSuperNode& node) = 0; virtual bool ListSuperNode(std::vector& nodes) = 0; + virtual bool FetchSuperNode(std::vector& nodes) = 0; virtual bool GetProofOfWorkTarget(const uint256& hashPrev, int nAlgo, int& nBits, int64& nReward) = 0; virtual bool GetBlockMintReward(const uint256& hashPrev, int64& nReward) = 0; virtual bool GetBlockLocator(const uint256& hashFork, CBlockLocator& locator, uint256& hashDepth, int nIncStep) = 0; diff --git a/src/bigbang/blockchain.cpp b/src/bigbang/blockchain.cpp index bae121ce..a8b0de19 100644 --- a/src/bigbang/blockchain.cpp +++ b/src/bigbang/blockchain.cpp @@ -660,6 +660,16 @@ bool CBlockChain::ListSuperNode(std::vector& nodes) return true; } +bool CBlockChain::FetchSuperNode(std::vector& nodes) +{ + if (!cntrBlock.FetchSuperNode(nodes)) + { + return false; + } + + return true; +} + // uint320 GetBlockTrust() const // { // if (IsVacant() && vchProof.empty()) diff --git a/src/bigbang/blockchain.h b/src/bigbang/blockchain.h index d3b4ff37..0995d760 100644 --- a/src/bigbang/blockchain.h +++ b/src/bigbang/blockchain.h @@ -46,6 +46,7 @@ class CBlockChain : public IBlockChain Errno AddNewOrigin(const CBlock& block, CBlockChainUpdate& update) override; Errno AddNewSuperNode(const storage::CSuperNode& node) override; bool ListSuperNode(std::vector& nodes) override; + bool FetchSuperNode(std::vector& nodes) override; bool GetProofOfWorkTarget(const uint256& hashPrev, int nAlgo, int& nBits, int64& nReward) override; bool GetBlockMintReward(const uint256& hashPrev, int64& nReward) override; bool GetBlockLocator(const uint256& hashFork, CBlockLocator& locator, uint256& hashDepth, int nIncStep) override; diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 5ed494f5..87d4a131 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -102,7 +102,7 @@ bool CMQCluster::HandleInvoke() } std::vector nodes; - if (!pBlockChain->ListSuperNode(nodes)) + if (!pBlockChain->FetchSuperNode(nodes)) { Log("CMQCluster::HandleInvoke(): list super node failed"); return false; @@ -111,12 +111,11 @@ bool CMQCluster::HandleInvoke() { mapSuperNode.insert(make_pair(node.superNodeID, node.vecOwnedForks)); if (1 == node.vecOwnedForks.size() - && node.vecOwnedForks[0] == pCoreProtocol->GetGenesisBlockHash() - && 2 == node.nodeCat) + && node.vecOwnedForks[0] == pCoreProtocol->GetGenesisBlockHash()) { Log("dpos node of MQ: [%s]", node.superNodeID.c_str()); } - else if (1 == node.nodeCat) + else if (0 != node.ipAddr) { Log("fork node of MQ: [%s]", node.superNodeID.c_str()); } diff --git a/src/storage/blockbase.cpp b/src/storage/blockbase.cpp index 246e352b..e0984a95 100644 --- a/src/storage/blockbase.cpp +++ b/src/storage/blockbase.cpp @@ -521,8 +521,27 @@ bool CBlockBase::ListSuperNode(vector& nodes) { for (const auto& fork : node.vecOwnedForks) { - Log("CBlockBase::ListSuperNode", "supernode client ID [%s] : fork [%s]", - node.superNodeID.c_str(), fork.ToString().c_str()); + Log("CBlockBase::ListSuperNode", "supernode client ID [%s] IP [%d]: fork [%s]", + node.superNodeID.c_str(), node.ipAddr, fork.ToString().c_str()); + } + } + return true; +} + +bool CBlockBase::FetchSuperNode(vector& nodes) +{ + if (!dbBlock.FetchSuperNode(nodes)) + { + Error("CBlockBase::FetchSuperNode", "Failed to fetch supernode"); + return false; + } + Log("CBlockBase::FetchSuperNode", "Fetch supernode successfully"); + for (const auto& node : nodes) + { + for (const auto& fork : node.vecOwnedForks) + { + Log("CBlockBase::FetchSuperNode", "supernode client ID [%s] IP [%d]: fork [%s]", + node.superNodeID.c_str(), node.ipAddr, fork.ToString().c_str()); } } return true; diff --git a/src/storage/blockbase.h b/src/storage/blockbase.h index 184185f3..7533f1fb 100644 --- a/src/storage/blockbase.h +++ b/src/storage/blockbase.h @@ -235,6 +235,7 @@ class CBlockBase bool AddNewForkContext(const CForkContext& ctxt); bool AddNewSuperNode(const CSuperNode& superNode); bool ListSuperNode(std::vector& nodes); + bool FetchSuperNode(std::vector& nodes); bool Retrieve(const uint256& hash, CBlock& block); bool Retrieve(const CBlockIndex* pIndex, CBlock& block); bool Retrieve(const uint256& hash, CBlockEx& block); diff --git a/src/storage/blockdb.cpp b/src/storage/blockdb.cpp index 54c21262..3050819c 100644 --- a/src/storage/blockdb.cpp +++ b/src/storage/blockdb.cpp @@ -261,5 +261,10 @@ bool CBlockDB::ListSuperNode(std::vector& nodes) return dbSuperNode.ListSuperNode(nodes); } +bool CBlockDB::FetchSuperNode(std::vector& nodes) +{ + return dbSuperNode.FetchSuperNode(nodes); +} + } // namespace storage } // namespace bigbang diff --git a/src/storage/blockdb.h b/src/storage/blockdb.h index 17780caa..ada154d0 100644 --- a/src/storage/blockdb.h +++ b/src/storage/blockdb.h @@ -50,6 +50,7 @@ class CBlockDB std::map& mapEnrollTxPos); bool AddNewSuperNode(const CSuperNode& superNode); bool ListSuperNode(std::vector& nodes); + bool FetchSuperNode(std::vector& nodes); protected: bool LoadFork(); diff --git a/src/storage/mqdb.cpp b/src/storage/mqdb.cpp index d5f2bd90..78ad9e97 100644 --- a/src/storage/mqdb.cpp +++ b/src/storage/mqdb.cpp @@ -14,6 +14,7 @@ namespace bigbang namespace storage { +const string CLIENT_ID_OUT_OF_MQ_CLUSTER = "OUTER-NODE"; ////////////////////////////// // CSuperNodeDB @@ -46,29 +47,40 @@ bool CSuperNodeDB::AddNewSuperNode(const CSuperNode& cli) bool ret = false; if (1 == cli.nodeCat) { - Clear(); - ret = Write(cli.superNodeID, cli.vecOwnedForks, false); + if (!ClearSuperNode()) + { + return false; + } + ret = Write(make_pair(cli.superNodeID, cli.ipAddr), cli.vecOwnedForks, true); } else if (2 == cli.nodeCat) { - ret = Write(cli.superNodeID, cli.vecOwnedForks, true); + if (!ClearSuperNode()) + { + return false; + } + ret = Write(make_pair(cli.superNodeID, cli.ipAddr), cli.vecOwnedForks, true); + } + else if (0 == cli.nodeCat) + { + ret = Write(make_pair(CLIENT_ID_OUT_OF_MQ_CLUSTER, cli.ipAddr), cli.vecOwnedForks, true); } return ret; } -bool CSuperNodeDB::RemoveSuperNode(const string& cliID) +bool CSuperNodeDB::RemoveSuperNode(const string& cliID, const int8& ipNum) { - return Erase(cliID); + return Erase(make_pair(cliID, ipNum)); } -bool CSuperNodeDB::RetrieveSuperNode(const string& cliID, CSuperNode& cli) +bool CSuperNodeDB::RetrieveSuperNode(const string& cliID, const int8& ipNum, CSuperNode& cli) { - return Read(cliID, cli); + return Read(make_pair(cliID, ipNum), cli); } bool CSuperNodeDB::ListSuperNode(std::vector& vCli) { - map> mapCli; + map, vector> mapCli; if (!WalkThrough(boost::bind(&CSuperNodeDB::LoadSuperNodeWalker, this, _1, _2, boost::ref(mapCli)))) { @@ -79,7 +91,8 @@ bool CSuperNodeDB::ListSuperNode(std::vector& vCli) for (const auto& it : mapCli) { CSuperNode node; - node.superNodeID = it.first; + node.superNodeID = it.first.first; + node.ipAddr = it.first.second; node.vecOwnedForks = it.second; vCli.emplace_back(node); } @@ -91,14 +104,73 @@ void CSuperNodeDB::Clear() RemoveAll(); } +bool CSuperNodeDB::FetchSuperNodeWalker(xengine::CBufStream& ssKey, xengine::CBufStream& ssValue, + map, vector>& mapCli) +{ + string strCliID; + uint32 nIP; + ssKey >> strCliID >> nIP; + + if (strCliID == CLIENT_ID_OUT_OF_MQ_CLUSTER) + { + return true; + } + + std::vector forks; + ssValue >> forks; + mapCli.insert(make_pair(make_pair(strCliID, nIP), forks)); + return true; +} + +bool CSuperNodeDB::FetchSuperNode(std::vector& vCli) +{ + map, vector> mapCli; + + if (!WalkThrough(boost::bind(&CSuperNodeDB::FetchSuperNodeWalker, this, _1, _2, boost::ref(mapCli)))) + { + return false; + } + + vCli.reserve(mapCli.size()); + for (const auto& it : mapCli) + { + CSuperNode node; + node.superNodeID = it.first.first; + node.ipAddr = it.first.second; + node.vecOwnedForks = it.second; + vCli.emplace_back(node); + } + return true; +} + +bool CSuperNodeDB::ClearSuperNode() +{ + vector vSuperNode; + if (!FetchSuperNode(vSuperNode)) + { + return false; + } + + for (auto const& supernode : vSuperNode) + { + if (!RemoveSuperNode(supernode.superNodeID, supernode.ipAddr)) + { + return false; + } + } + + return true; +} + bool CSuperNodeDB::LoadSuperNodeWalker(xengine::CBufStream& ssKey, xengine::CBufStream& ssValue, - map>& mapCli) + map, vector>& mapCli) { string strCliID; - ssKey >> strCliID; + uint32 nIP; + ssKey >> strCliID >> nIP; std::vector forks; ssValue >> forks; - mapCli.insert(make_pair(strCliID, forks)); + mapCli.insert(make_pair(make_pair(strCliID, nIP), forks)); return true; } diff --git a/src/storage/mqdb.h b/src/storage/mqdb.h index ee70bea0..17fffee0 100644 --- a/src/storage/mqdb.h +++ b/src/storage/mqdb.h @@ -5,9 +5,10 @@ #ifndef BIGBANG_MQDB_H #define BIGBANG_MQDB_H +#include + #include "uint256.h" #include "xengine.h" -#include namespace bigbang { @@ -69,14 +70,18 @@ class CSuperNodeDB : public xengine::CKVDB bool Initialize(const boost::filesystem::path& pathData); void Deinitialize(); bool AddNewSuperNode(const CSuperNode& cli); - bool RemoveSuperNode(const std::string& cliID); - bool RetrieveSuperNode(const std::string& superNodeID, CSuperNode& cli); - bool ListSuperNode(std::vector& vCli); + bool RemoveSuperNode(const std::string& cliID, const int8& ipNum); + bool RetrieveSuperNode(const std::string& superNodeID, const int8& ipNum, CSuperNode& cli); + bool ListSuperNode(std::vector& vCli); //return all nodes void Clear(); + bool ClearSuperNode(); + bool FetchSuperNode(std::vector& vCli); //only return super nodes including dpos and fork nodes protected: bool LoadSuperNodeWalker(xengine::CBufStream& ssKey, xengine::CBufStream& ssValue, - std::map>& mapCli); + std::map, std::vector>& mapCli); + bool FetchSuperNodeWalker(xengine::CBufStream& ssKey, xengine::CBufStream& ssValue, + std::map, std::vector>& mapCli); }; } // namespace storage From 3fd53ee18dc6bab0f43ac8989d75942fc95035cb Mon Sep 17 00:00:00 2001 From: OuYun Date: Wed, 1 Apr 2020 17:45:05 +0800 Subject: [PATCH 051/181] fix issue involved at storage --- src/storage/mqdb.cpp | 27 +++++++++++++++++++++------ src/storage/mqdb.h | 2 +- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/storage/mqdb.cpp b/src/storage/mqdb.cpp index 78ad9e97..3627d377 100644 --- a/src/storage/mqdb.cpp +++ b/src/storage/mqdb.cpp @@ -47,7 +47,7 @@ bool CSuperNodeDB::AddNewSuperNode(const CSuperNode& cli) bool ret = false; if (1 == cli.nodeCat) { - if (!ClearSuperNode()) + if (!ClearSuperNode(cli)) { return false; } @@ -55,7 +55,7 @@ bool CSuperNodeDB::AddNewSuperNode(const CSuperNode& cli) } else if (2 == cli.nodeCat) { - if (!ClearSuperNode()) + if (!ClearSuperNode(cli)) { return false; } @@ -143,7 +143,7 @@ bool CSuperNodeDB::FetchSuperNode(std::vector& vCli) return true; } -bool CSuperNodeDB::ClearSuperNode() +bool CSuperNodeDB::ClearSuperNode(const CSuperNode& cli) { vector vSuperNode; if (!FetchSuperNode(vSuperNode)) @@ -151,11 +151,26 @@ bool CSuperNodeDB::ClearSuperNode() return false; } - for (auto const& supernode : vSuperNode) + if (1 == cli.nodeCat) //fork node updates self by remove all supernode entries { - if (!RemoveSuperNode(supernode.superNodeID, supernode.ipAddr)) + for (auto const& supernode : vSuperNode) { - return false; + if (!RemoveSuperNode(supernode.superNodeID, supernode.ipAddr)) + { + return false; + } + } + return true; + } + + if (2 == cli.nodeCat && 0 == cli.ipAddr) //dpos node updates oneself + { + for (auto const& supernode : vSuperNode) + { + if (0 == supernode.ipAddr && !RemoveSuperNode(supernode.superNodeID, supernode.ipAddr)) //need to remove previous one first + { + return false; + } } } diff --git a/src/storage/mqdb.h b/src/storage/mqdb.h index 17fffee0..6a65152a 100644 --- a/src/storage/mqdb.h +++ b/src/storage/mqdb.h @@ -74,7 +74,7 @@ class CSuperNodeDB : public xengine::CKVDB bool RetrieveSuperNode(const std::string& superNodeID, const int8& ipNum, CSuperNode& cli); bool ListSuperNode(std::vector& vCli); //return all nodes void Clear(); - bool ClearSuperNode(); + bool ClearSuperNode(const CSuperNode& cli); bool FetchSuperNode(std::vector& vCli); //only return super nodes including dpos and fork nodes protected: From 3cb7a4894c9c969fa9e77248941ab0ff85a8a44e Mon Sep 17 00:00:00 2001 From: OuYun Date: Wed, 1 Apr 2020 18:24:05 +0800 Subject: [PATCH 052/181] fix defect of incorrect startup to connect mq broker while no correspond type entry entered with null topic --- src/bigbang/mqcluster.cpp | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 87d4a131..e980ef99 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -28,7 +28,8 @@ CMQCluster::CMQCluster(int catNodeIn) fAbort(false), addrBroker("tcp://localhost:1883"), nReqBlkTimerID(0), - nRollNum(0) + nRollNum(0), + clientID("") { switch (catNodeIn) { @@ -113,11 +114,11 @@ bool CMQCluster::HandleInvoke() if (1 == node.vecOwnedForks.size() && node.vecOwnedForks[0] == pCoreProtocol->GetGenesisBlockHash()) { - Log("dpos node of MQ: [%s]", node.superNodeID.c_str()); + Log("dpos node of MQ: [%s] [%d]", node.superNodeID.c_str(), node.ipAddr); } else if (0 != node.ipAddr) { - Log("fork node of MQ: [%s]", node.superNodeID.c_str()); + Log("fork node of MQ: [%s] [%d]", node.superNodeID.c_str(), node.ipAddr); } for (const auto& fork : node.vecOwnedForks) { @@ -284,7 +285,7 @@ bool CMQCluster::HandleEvent(CEventMQEnrollUpdate& eventMqUpdateEnroll) } else if (NODE_CATEGORY::DPOSNODE == catNode) { - if (1 == forks.size() + if (1 == forks.size() && 0 == eventMqUpdateEnroll.data.ipAddr && forks[0] == pCoreProtocol->GetGenesisBlockHash()) { //dpos node clientID = id; @@ -292,6 +293,12 @@ bool CMQCluster::HandleEvent(CEventMQEnrollUpdate& eventMqUpdateEnroll) topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; Log("CMQCluster::HandleEvent(): dpos node clientid [%s] with topic [%s]", clientID.c_str(), topicReqBlk.c_str()); + + { + boost::unique_lock lock(mtxStatus); + mapSuperNode.insert(make_pair(id, forks)); + } + condStatus.notify_all(); } else { //fork nodes either enrolled or p2p @@ -299,12 +306,6 @@ bool CMQCluster::HandleEvent(CEventMQEnrollUpdate& eventMqUpdateEnroll) Log("CMQCluster::HandleEvent(): dpos node register clientid [%s] with topic [%s]", clientID.c_str(), topicReqBlk.c_str()); } - - { - boost::unique_lock lock(mtxStatus); - mapSuperNode.insert(make_pair(id, forks)); - } - condStatus.notify_all(); } return true; @@ -956,7 +957,7 @@ void CMQCluster::MqttThreadFunc() if (!fAbort) { boost::unique_lock lock(mtxStatus); - while (mapSuperNode.empty()) + while ("" == clientID) { Log("there is no enrollment info, waiting for it coming..."); condStatus.wait(lock); From 15088e4f3c8c9250555a45a3d638da6a221584b7 Mon Sep 17 00:00:00 2001 From: OuYun Date: Thu, 2 Apr 2020 12:52:33 +0800 Subject: [PATCH 053/181] bugfix of mqdb storage functions --- src/storage/mqdb.cpp | 6 +++--- src/storage/mqdb.h | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/storage/mqdb.cpp b/src/storage/mqdb.cpp index 3627d377..2ebd3dba 100644 --- a/src/storage/mqdb.cpp +++ b/src/storage/mqdb.cpp @@ -68,14 +68,14 @@ bool CSuperNodeDB::AddNewSuperNode(const CSuperNode& cli) return ret; } -bool CSuperNodeDB::RemoveSuperNode(const string& cliID, const int8& ipNum) +bool CSuperNodeDB::RemoveSuperNode(const string& cliID, const uint32& ipNum) { return Erase(make_pair(cliID, ipNum)); } -bool CSuperNodeDB::RetrieveSuperNode(const string& cliID, const int8& ipNum, CSuperNode& cli) +bool CSuperNodeDB::RetrieveSuperNode(const string& cliID, const uint32& ipNum, vector& vFork) { - return Read(make_pair(cliID, ipNum), cli); + return Read(make_pair(cliID, ipNum), vFork); } bool CSuperNodeDB::ListSuperNode(std::vector& vCli) diff --git a/src/storage/mqdb.h b/src/storage/mqdb.h index 6a65152a..e717015e 100644 --- a/src/storage/mqdb.h +++ b/src/storage/mqdb.h @@ -70,8 +70,8 @@ class CSuperNodeDB : public xengine::CKVDB bool Initialize(const boost::filesystem::path& pathData); void Deinitialize(); bool AddNewSuperNode(const CSuperNode& cli); - bool RemoveSuperNode(const std::string& cliID, const int8& ipNum); - bool RetrieveSuperNode(const std::string& superNodeID, const int8& ipNum, CSuperNode& cli); + bool RemoveSuperNode(const std::string& cliID, const uint32& ipNum); + bool RetrieveSuperNode(const std::string& superNodeID, const uint32& ipNum, std::vector& vFork); bool ListSuperNode(std::vector& vCli); //return all nodes void Clear(); bool ClearSuperNode(const CSuperNode& cli); From 4bb8401452420479ff383f9d176f6ee02b458e1f Mon Sep 17 00:00:00 2001 From: OuYun Date: Thu, 2 Apr 2020 22:52:16 +0800 Subject: [PATCH 054/181] add supernode db interface and optimize code --- src/bigbang/netchn.cpp | 6 +++--- src/storage/mqdb.cpp | 21 +++++++++++---------- src/storage/mqdb.h | 1 + 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/bigbang/netchn.cpp b/src/bigbang/netchn.cpp index 63044cb5..06718dc8 100644 --- a/src/bigbang/netchn.cpp +++ b/src/bigbang/netchn.cpp @@ -310,11 +310,11 @@ void CNetChannel::SubscribeFork(const uint256& hashFork, const uint64& nNonce) eventSubscribe.data.push_back(hashFork); { boost::shared_lock rlock(rwNetPeer); - for (map::iterator it = mapPeer.begin(); it != mapPeer.end(); ++it) + for (const auto& p : mapPeer) { - eventSubscribe.nNonce = (*it).first; + eventSubscribe.nNonce = p.first; pPeerNet->DispatchEvent(&eventSubscribe); - DispatchGetBlocksEvent(it->first, hashFork); + DispatchGetBlocksEvent(p.first, hashFork); BroadcastTxInv(hashFork); } } diff --git a/src/storage/mqdb.cpp b/src/storage/mqdb.cpp index 2ebd3dba..fb068476 100644 --- a/src/storage/mqdb.cpp +++ b/src/storage/mqdb.cpp @@ -45,15 +45,14 @@ void CSuperNodeDB::Deinitialize() bool CSuperNodeDB::AddNewSuperNode(const CSuperNode& cli) { bool ret = false; - if (1 == cli.nodeCat) + switch(cli.nodeCat) { - if (!ClearSuperNode(cli)) - { - return false; - } - ret = Write(make_pair(cli.superNodeID, cli.ipAddr), cli.vecOwnedForks, true); + case 0: + { + ret = Write(make_pair(CLIENT_ID_OUT_OF_MQ_CLUSTER, cli.ipAddr), cli.vecOwnedForks, true); + break; } - else if (2 == cli.nodeCat) + case 1: case 2: { if (!ClearSuperNode(cli)) { @@ -61,9 +60,6 @@ bool CSuperNodeDB::AddNewSuperNode(const CSuperNode& cli) } ret = Write(make_pair(cli.superNodeID, cli.ipAddr), cli.vecOwnedForks, true); } - else if (0 == cli.nodeCat) - { - ret = Write(make_pair(CLIENT_ID_OUT_OF_MQ_CLUSTER, cli.ipAddr), cli.vecOwnedForks, true); } return ret; } @@ -78,6 +74,11 @@ bool CSuperNodeDB::RetrieveSuperNode(const string& cliID, const uint32& ipNum, v return Read(make_pair(cliID, ipNum), vFork); } +bool CSuperNodeDB::UpdateSuperNode(const string& cliID, const uint32& ipNum, const vector& vFork) +{ + return Write(make_pair(cliID, ipNum), vFork, true); +} + bool CSuperNodeDB::ListSuperNode(std::vector& vCli) { map, vector> mapCli; diff --git a/src/storage/mqdb.h b/src/storage/mqdb.h index e717015e..ff400812 100644 --- a/src/storage/mqdb.h +++ b/src/storage/mqdb.h @@ -72,6 +72,7 @@ class CSuperNodeDB : public xengine::CKVDB bool AddNewSuperNode(const CSuperNode& cli); bool RemoveSuperNode(const std::string& cliID, const uint32& ipNum); bool RetrieveSuperNode(const std::string& superNodeID, const uint32& ipNum, std::vector& vFork); + bool UpdateSuperNode(const std::string& cliID, const uint32& ipNum, const std::vector& vFork); bool ListSuperNode(std::vector& vCli); //return all nodes void Clear(); bool ClearSuperNode(const CSuperNode& cli); From 0e906a896577aa90c5875a67f2893c48e8071bc2 Mon Sep 17 00:00:00 2001 From: OuYun Date: Mon, 6 Apr 2020 23:02:51 +0800 Subject: [PATCH 055/181] fix the following defects: - revise length of long chain blocks needed to be rolled back; - additional processing such as clean sending buffer and reset timer of request for blocks after receiving rollback message; - handling with duplicated blocks --- src/bigbang/dispatcher.cpp | 5 +- src/bigbang/mqcluster.cpp | 95 +++++++++++++++++++++++++++++--------- src/bigbang/mqcluster.h | 5 +- src/bigbang/struct.h | 2 +- 4 files changed, 79 insertions(+), 28 deletions(-) diff --git a/src/bigbang/dispatcher.cpp b/src/bigbang/dispatcher.cpp index 72e4857d..1cfbb178 100644 --- a/src/bigbang/dispatcher.cpp +++ b/src/bigbang/dispatcher.cpp @@ -367,7 +367,7 @@ void CDispatcher::UpdatePrimaryBlock(const CBlock& block, const CBlockChainUpdat CEventMQChainUpdate* pMqChainUpdate = new CEventMQChainUpdate(0); if (pMqChainUpdate != nullptr) { - pMqChainUpdate->data.shortLen = updateBlockChain.vBlockRemove.size(); + pMqChainUpdate->data.actRollBackLen = updateBlockChain.vBlockAddNew.size(); pMqChainUpdate->data.vShort.reserve(updateBlockChain.vBlockRemove.size()); for (const auto& rb : updateBlockChain.vBlockRemove) { @@ -375,7 +375,8 @@ void CDispatcher::UpdatePrimaryBlock(const CBlock& block, const CBlockChainUpdat rb.GetHash().ToString().c_str()); pMqChainUpdate->data.vShort.emplace_back(rb.GetHash()); } - pMqChainUpdate->data.triHeight = pMqChainUpdate->data.vShort.back().Get32(7) - 1; + reverse(pMqChainUpdate->data.vShort.begin(), pMqChainUpdate->data.vShort.end()); + pMqChainUpdate->data.triHeight = pMqChainUpdate->data.vShort.front().Get32(7) - 1; pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), pMqChainUpdate->data.triHeight, pMqChainUpdate->data.triHash); diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index e980ef99..59977183 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -129,6 +129,8 @@ bool CMQCluster::HandleInvoke() if (NODE_CATEGORY::FORKNODE == catNode) { + lastHeightResp = pBlockChain->GetBlockCount(pCoreProtocol->GetGenesisBlockHash()) - 1; + if (mapSuperNode.size() == 0) { Log("CMQCluster::HandleInvoke(): this fork node has not enrolled " @@ -168,6 +170,7 @@ bool CMQCluster::HandleInvoke() } else if (NODE_CATEGORY::DPOSNODE == catNode) { + lastHeightResp = -1; for (const auto& node : mapSuperNode) { if (1 == node.second.size() @@ -228,7 +231,7 @@ bool CMQCluster::HandleEvent(CEventMQChainUpdate& eventMqUpdateChain) CRollbackBlock rbc; rbc.rbHeight = update.triHeight; rbc.rbHash = update.triHash; - rbc.rbSize = update.shortLen; + rbc.rbSize = update.actRollBackLen; rbc.hashList = update.vShort; CBufferPtr spRBC(new CBufStream); @@ -424,7 +427,7 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) return; case NODE_CATEGORY::FORKNODE: { - Log("CMQCluster::OnReceiveMessage(): current height is [%d]", int(lastHeightResp)); + Log("CMQCluster::OnReceiveMessage(): current sync height is [%d]", int(lastHeightResp)); if (topicRbBlk != topic) { //respond to request block of main chain //unpack payload @@ -463,7 +466,30 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) err = pDispatcher->AddNewBlock(resp.block); if (err != OK) { - Error("CMQCluster::OnReceiveMessage(): failed to validate block (%d) : %s\n", err, ErrorString(err)); + Error("CMQCluster::OnReceiveMessage(): failed to add new block (%d) : %s\n", err, ErrorString(err)); + if (ERR_ALREADY_HAVE == err) + { + lastHeightResp = resp.height; + //check if there are rollbacked blocks + if (nRollNum) + { + boost::unique_lock lock(mtxRoll); + if (vLongFork.size() < nRollNum) + { + vLongFork.emplace_back(resp.hash); + } + else + { + vLongFork.clear(); + nRollNum = 0; + } + } + if (!PostBlockRequest(lastHeightResp)) + { + Error("CMQCluster::OnReceiveMessage(): failed to post request on response due to duplication"); + return; + } + } return; } lastHeightResp = resp.height; @@ -489,6 +515,7 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) { //when reach best height, send request by timer nReqBlkTimerID = SetTimer(1000 * 60, boost::bind(&CMQCluster::RequestBlockTimerFunc, this, _1)); + return; } else { @@ -499,7 +526,7 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) } if (!PostBlockRequest(lastHeightResp)) { - Error("CMQCluster::OnReceiveMessage(): failed to post request"); + Error("CMQCluster::OnReceiveMessage(): failed to post request on response"); return; } } @@ -524,51 +551,69 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) Log("CMQCluster::OnReceiveMessage(): rbheight[%d], lastheight[%d]", rb.rbHeight, int(lastHeightResp)); + //cancel request sync timer + if (nReqBlkTimerID != 0) + { + CancelTimer(nReqBlkTimerID); + nReqBlkTimerID = 0; + } + + //empty sending buffer + { + boost::unique_lock lock(mtxSend); + deqSendBuff.clear(); + } + //check hard fork point uint256 hash; - if (!pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), rb.rbHeight, hash) - || hash != rb.rbHash) + if (!pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), rb.rbHeight, hash)) { - Log("CMQCluster::OnReceiveMessage(): rbhash[%s], lasthash[%s]", - rb.rbHash.ToString().c_str(), hash.ToString().c_str()); Error("CMQCluster::OnReceiveMessage(): failed to get hard fork block hash or dismatch" "then re-synchronize block from genesis one"); if (!PostBlockRequest(0)) //re-sync from genesis block { Error("CMQCluster::OnReceiveMessage(): failed to post request while re-sync"); - return; } return; } - bool fMatch = false; - Log("CMQCluster::OnReceiveMessage(): rbhash[%s], lasthash[%s]", - rb.rbHash.ToString().c_str(), hash.ToString().c_str()); if (hash != rb.rbHash) { - Error("CMQCluster::OnReceiveMessage(): hard fork block hash does not match"); + Error("CMQCluster::OnReceiveMessage(): hashes do not match - rbhash[%s], lasthash[%s]", + rb.rbHash.ToString().c_str(), hash.ToString().c_str()); + if (!PostBlockRequest(0)) //re-sync from genesis block + { + Error("CMQCluster::OnReceiveMessage(): failed to post request while re-sync"); + } return; } + bool fMatch = false; + Log("CMQCluster::OnReceiveMessage(): rbhash[%s], lasthash[%s]", + rb.rbHash.ToString().c_str(), hash.ToString().c_str()); + //check blocks in rollback int nShort = 0; - for (int i = rb.rbSize; i > 0; --i) + for (int i = 0; i < rb.rbSize; ++i) { if (!pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), - rb.rbHeight + rb.rbSize - i + 1, hash)) + rb.rbHeight + i + 1, hash)) { - if (i != rb.rbSize) + if (i != 0) { Log("CMQCluster::OnReceiveMessage(): exceed to get rollback block hash"); break; } else { - Error("CMQCluster::OnReceiveMessage(): short chain does not match for one on dpos node"); + Error("CMQCluster::OnReceiveMessage(): short chain does not match for one on dpos node:1"); return; } } - if (hash == rb.hashList[i - 1]) + Log("CMQCluster::OnReceiveMessage(): fork node blkhsh[%s] vs. dpos node blkhsh[%s] " + "at height of [%d]", rb.hashList[i].ToString().c_str(), hash.ToString().c_str(), + rb.rbHeight + i + 1); + if (hash == rb.hashList[i]) { Log("CMQCluster::OnReceiveMessage(): fork node has not been rolled back yet" " with hash [%s]", @@ -579,7 +624,11 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) } else { - Error("CMQCluster::OnReceiveMessage(): short chain does not match for one on dpos node"); + Error("CMQCluster::OnReceiveMessage(): short chain does not match for one on dpos node:2"); + if (!PostBlockRequest(0)) //re-sync from genesis block + { + Error("CMQCluster::OnReceiveMessage(): failed to post request while re-sync"); + } return; } } @@ -589,14 +638,14 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) if (fMatch) { lastHeightResp = rb.rbHeight; - Log("CMQCluster::OnReceiveMessage(): rb.rbHeight[%d] against lastHeightResp[%d]", rb.rbHeight, int(lastHeightResp)); + Log("CMQCluster::OnReceiveMessage(): match to prepare rollback: rb.rbHeight[%d] against lastHeightResp[%d]", rb.rbHeight, int(lastHeightResp)); if (!PostBlockRequest(lastHeightResp)) { - Error("CMQCluster::OnReceiveMessage(): failed to post request"); - nRollNum = nShort; + Error("CMQCluster::OnReceiveMessage(): failed to post request on rollback"); + nRollNum = rb.rbSize; return; } - nRollNum = nShort; + nRollNum = rb.rbSize; } } } diff --git a/src/bigbang/mqcluster.h b/src/bigbang/mqcluster.h index c0888d88..bb3f558b 100644 --- a/src/bigbang/mqcluster.h +++ b/src/bigbang/mqcluster.h @@ -74,7 +74,7 @@ class CRollbackBlock public: int32 rbHeight; uint256 rbHash; - uint8 rbSize; + int rbSize; std::vector hashList; protected: @@ -182,7 +182,8 @@ class CMQCluster : public IMQCluster std::deque> deqRecvBuff; //topic vs. payload std::atomic lastHeightResp; - uint32 nReqBlkTimerID; +// uint32 nReqBlkTimerID; + std::atomic nReqBlkTimerID; boost::mutex mtxRoll; std::vector vLongFork; diff --git a/src/bigbang/struct.h b/src/bigbang/struct.h index e1b47339..4e042bc6 100644 --- a/src/bigbang/struct.h +++ b/src/bigbang/struct.h @@ -261,7 +261,7 @@ class CMqRollbackUpdate public: int32 triHeight; uint256 triHash; - uint8 shortLen; + int actRollBackLen; std::vector vShort; }; From c4f53f946e756ea430b22816faeacd7bdac5aba5 Mon Sep 17 00:00:00 2001 From: OuYun Date: Tue, 7 Apr 2020 11:08:32 +0800 Subject: [PATCH 056/181] restrict dpos node to subscribe biz forks and/or get blocks on them --- src/bigbang/forkmanager.cpp | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/src/bigbang/forkmanager.cpp b/src/bigbang/forkmanager.cpp index 510e7e27..767afbdc 100644 --- a/src/bigbang/forkmanager.cpp +++ b/src/bigbang/forkmanager.cpp @@ -53,26 +53,34 @@ void CForkManager::HandleDeinitialize() bool CForkManager::HandleInvoke() { - boost::unique_lock wlock(rwAccess); + int8 nodeCat = dynamic_cast(Config())->nCatOfNode; + if (2 == nodeCat) + { + return true; + } - fAllowAnyFork = ForkConfig()->fAllowAnyFork; - if (!fAllowAnyFork) { - setForkAllowed.insert(pCoreProtocol->GetGenesisBlockHash()); - for (const string& strFork : ForkConfig()->vFork) + boost::unique_lock wlock(rwAccess); + + fAllowAnyFork = ForkConfig()->fAllowAnyFork; + if (!fAllowAnyFork) { - uint256 hashFork(strFork); - if (hashFork != 0) + setForkAllowed.insert(pCoreProtocol->GetGenesisBlockHash()); + for (const string& strFork : ForkConfig()->vFork) { - setForkAllowed.insert(hashFork); + uint256 hashFork(strFork); + if (hashFork != 0) + { + setForkAllowed.insert(hashFork); + } } - } - for (const string& strFork : ForkConfig()->vGroup) - { - uint256 hashFork(strFork); - if (hashFork != 0) + for (const string& strFork : ForkConfig()->vGroup) { - setGroupAllowed.insert(hashFork); + uint256 hashFork(strFork); + if (hashFork != 0) + { + setGroupAllowed.insert(hashFork); + } } } } From 943cbf077118e78fb4aaf1734921a6c57d3ee5cc Mon Sep 17 00:00:00 2001 From: OuYun Date: Tue, 7 Apr 2020 11:25:45 +0800 Subject: [PATCH 057/181] update previous commit for when invoking enroll supernode RPC --- src/bigbang/forkmanager.cpp | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/src/bigbang/forkmanager.cpp b/src/bigbang/forkmanager.cpp index 767afbdc..add7132a 100644 --- a/src/bigbang/forkmanager.cpp +++ b/src/bigbang/forkmanager.cpp @@ -56,6 +56,7 @@ bool CForkManager::HandleInvoke() int8 nodeCat = dynamic_cast(Config())->nCatOfNode; if (2 == nodeCat) { + setForkAllowed.insert(pCoreProtocol->GetGenesisBlockHash()); return true; } @@ -268,22 +269,30 @@ bool CForkManager::GetSubline(const uint256& hashFork, vector bool CForkManager::SetForkFilter(const std::vector& vFork, const std::vector& vGroup) { - boost::unique_lock wlock(rwAccess); + int8 nodeCat = dynamic_cast(Config())->nCatOfNode; + if (2 == nodeCat) + { + return true; + } - if (!fAllowAnyFork) { - for (auto const& fork : vFork) + boost::unique_lock wlock(rwAccess); + + if (!fAllowAnyFork) { - if (fork != 0) + for (auto const& fork : vFork) { - setForkAllowed.insert(fork); + if (fork != 0) + { + setForkAllowed.insert(fork); + } } - } - for (auto const& group : vGroup) - { - if (group != 0) + for (auto const& group : vGroup) { - setGroupAllowed.insert(group); + if (group != 0) + { + setGroupAllowed.insert(group); + } } } } From af45bb2ac4d7eec550a57676810fad2e4cf42a7e Mon Sep 17 00:00:00 2001 From: OuYun Date: Wed, 8 Apr 2020 09:31:43 +0800 Subject: [PATCH 058/181] restrict fork nodes to subscrib main chain and/or send getblocks for main chain blocks --- src/bigbang/netchn.cpp | 73 +++++++++++++++++++++++++++++------------- src/bigbang/netchn.h | 2 ++ 2 files changed, 52 insertions(+), 23 deletions(-) diff --git a/src/bigbang/netchn.cpp b/src/bigbang/netchn.cpp index 06718dc8..4d0c6e1e 100644 --- a/src/bigbang/netchn.cpp +++ b/src/bigbang/netchn.cpp @@ -57,7 +57,7 @@ void CNetChannelPeer::CNetChannelPeerFork::ClearExpiredTx() bool CNetChannelPeer::IsSynchronized(const uint256& hashFork) const { - map::const_iterator it = mapSubscribedFork.find(hashFork); + auto it = mapSubscribedFork.find(hashFork); if (it != mapSubscribedFork.end()) { return (*it).second.fSynchronized; @@ -67,7 +67,7 @@ bool CNetChannelPeer::IsSynchronized(const uint256& hashFork) const bool CNetChannelPeer::SetSyncStatus(const uint256& hashFork, bool fSync, bool& fInverted) { - map::iterator it = mapSubscribedFork.find(hashFork); + auto it = mapSubscribedFork.find(hashFork); if (it != mapSubscribedFork.end()) { fInverted = ((*it).second.fSynchronized != fSync); @@ -79,7 +79,7 @@ bool CNetChannelPeer::SetSyncStatus(const uint256& hashFork, bool fSync, bool& f void CNetChannelPeer::AddKnownTx(const uint256& hashFork, const vector& vTxHash, size_t nTotalSynTxCount) { - map::iterator it = mapSubscribedFork.find(hashFork); + auto it = mapSubscribedFork.find(hashFork); if (it != mapSubscribedFork.end()) { (*it).second.AddKnownTx(vTxHash, nTotalSynTxCount); @@ -88,7 +88,7 @@ void CNetChannelPeer::AddKnownTx(const uint256& hashFork, const vector& bool CNetChannelPeer::MakeTxInv(const uint256& hashFork, const vector& vTxPool, vector& vInv) { - map::iterator it = mapSubscribedFork.find(hashFork); + auto it = mapSubscribedFork.find(hashFork); if (it != mapSubscribedFork.end()) { CNetChannelPeerFork& peerFork = (*it).second; @@ -109,7 +109,7 @@ bool CNetChannelPeer::MakeTxInv(const uint256& hashFork, const vector& } else if (!(*it).second.IsKnownTx(txid)) { - vInv.push_back(network::CInv(network::CInv::MSG_TX, txid)); + vInv.emplace_back(network::CInv(network::CInv::MSG_TX, txid)); vTxHash.push_back(txid); } } @@ -140,6 +140,7 @@ CNetChannel::CNetChannel() pService = nullptr; pDispatcher = nullptr; fStartIdlePushTxTimer = false; + nNodeCat = 0; } CNetChannel::~CNetChannel() @@ -204,6 +205,7 @@ bool CNetChannel::HandleInvoke() nTimerPushTx = 0; fStartIdlePushTxTimer = false; } + nNodeCat = dynamic_cast(Config())->nCatOfNode; return network::INetChannel::HandleInvoke(); } @@ -242,7 +244,7 @@ int CNetChannel::GetPrimaryChainHeight() bool CNetChannel::IsForkSynchronized(const uint256& hashFork) const { boost::shared_lock rlock(rwNetPeer); - map>::const_iterator it = mapUnsync.find(hashFork); + auto it = mapUnsync.find(hashFork); return (it == mapUnsync.end() || (*it).second.empty()); } @@ -256,13 +258,13 @@ void CNetChannel::BroadcastBlockInv(const uint256& hashFork, const uint256& hash } network::CEventPeerInv eventInv(0, hashFork); - eventInv.data.push_back(network::CInv(network::CInv::MSG_BLOCK, hashBlock)); + eventInv.data.emplace_back(network::CInv(network::CInv::MSG_BLOCK, hashBlock)); { boost::shared_lock rlock(rwNetPeer); - for (map::iterator it = mapPeer.begin(); it != mapPeer.end(); ++it) + for (auto const& p : mapPeer) { - uint64 nNonce = (*it).first; - if (!setKnownPeer.count(nNonce) && (*it).second.IsSubscribed(hashFork)) + uint64 nNonce = p.first; + if (!setKnownPeer.count(nNonce) && p.second.IsSubscribed(hashFork)) { eventInv.nNonce = nNonce; pPeerNet->DispatchEvent(&eventInv); @@ -306,6 +308,12 @@ void CNetChannel::SubscribeFork(const uint256& hashFork, const uint64& nNonce) StdLog("NetChannel", "SubscribeFork: mapSched insert success, hashFork: %s", hashFork.GetHex().c_str()); } + if (hashFork == pCoreProtocol->GetGenesisBlockHash() && 1 == nNodeCat) + { + StdLog("NetChannel", "SubscribeFork: success in filtering main chain for fork node with nonce[%d]", nNonce); + return; + } + network::CEventPeerSubscribe eventSubscribe(0ULL, pCoreProtocol->GetGenesisBlockHash()); eventSubscribe.data.push_back(hashFork); { @@ -356,17 +364,25 @@ bool CNetChannel::HandleEvent(network::CEventPeerActive& eventActive) StdLog("NetChannel", "CEventPeerActive: peer: %s", GetPeerAddressInfo(nNonce).c_str()); if ((eventActive.data.nService & network::NODE_NETWORK)) { - DispatchGetBlocksEvent(nNonce, pCoreProtocol->GetGenesisBlockHash()); - BroadcastTxInv(pCoreProtocol->GetGenesisBlockHash()); + if (1 != nNodeCat) + { + DispatchGetBlocksEvent(nNonce, pCoreProtocol->GetGenesisBlockHash()); + BroadcastTxInv(pCoreProtocol->GetGenesisBlockHash()); + } + else + { + StdLog("NetChannel", "CEventPeerActive: succeed in filtering to " + "get main chain from peer[%s]", GetPeerAddressInfo(nNonce).c_str()); + } network::CEventPeerSubscribe eventSubscribe(nNonce, pCoreProtocol->GetGenesisBlockHash()); { boost::recursive_mutex::scoped_lock scoped_lock(mtxSched); - for (map::iterator it = mapSched.begin(); it != mapSched.end(); ++it) + for (auto const& sch : mapSched) { - if ((*it).first != pCoreProtocol->GetGenesisBlockHash()) + if (sch.first != pCoreProtocol->GetGenesisBlockHash()) { - eventSubscribe.data.push_back((*it).first); + eventSubscribe.data.push_back(sch.first); } } } @@ -385,22 +401,22 @@ bool CNetChannel::HandleEvent(network::CEventPeerDeactive& eventDeactive) StdLog("NetChannel", "CEventPeerDeactive: peer: %s", GetPeerAddressInfo(nNonce).c_str()); { boost::recursive_mutex::scoped_lock scoped_lock(mtxSched); - for (map::iterator it = mapSched.begin(); it != mapSched.end(); ++it) + for (auto& sch : mapSched) { - CSchedule& sched = (*it).second; + CSchedule& sched = sch.second; set setSchedPeer; sched.RemovePeer(nNonce, setSchedPeer); for (const uint64 nNonceSched : setSchedPeer) { - SchedulePeerInv(nNonceSched, (*it).first, sched); + SchedulePeerInv(nNonceSched, sch.first, sched); } } } { boost::unique_lock wlock(rwNetPeer); - map::iterator it = mapPeer.find(nNonce); + auto it = mapPeer.find(nNonce); if (it != mapPeer.end()) { for (auto& subFork : (*it).second.mapSubscribedFork) @@ -423,13 +439,24 @@ bool CNetChannel::HandleEvent(network::CEventPeerSubscribe& eventSubscribe) if (hashFork == pCoreProtocol->GetGenesisBlockHash()) { boost::unique_lock wlock(rwNetPeer); - map::iterator it = mapPeer.find(nNonce); + auto it = mapPeer.find(nNonce); if (it != mapPeer.end()) { for (const uint256& hash : eventSubscribe.data) { - (*it).second.Subscribe(hash); - mapUnsync[hash].insert(nNonce); + if (2 == nNodeCat) + { + if (hash == pCoreProtocol->GetGenesisBlockHash()) + { + (*it).second.Subscribe(hash); + mapUnsync[hash].insert(nNonce); + } + } + else + { + (*it).second.Subscribe(hash); + mapUnsync[hash].insert(nNonce); + } { boost::recursive_mutex::scoped_lock scoped_lock(mtxSched); @@ -457,7 +484,7 @@ bool CNetChannel::HandleEvent(network::CEventPeerUnsubscribe& eventUnsubscribe) if (hashFork == pCoreProtocol->GetGenesisBlockHash()) { boost::unique_lock wlock(rwNetPeer); - map::iterator it = mapPeer.find(nNonce); + auto it = mapPeer.find(nNonce); if (it != mapPeer.end()) { for (const uint256& hash : eventUnsubscribe.data) diff --git a/src/bigbang/netchn.h b/src/bigbang/netchn.h index 40f40b3f..24a57536 100644 --- a/src/bigbang/netchn.h +++ b/src/bigbang/netchn.h @@ -319,6 +319,8 @@ class CNetChannel : public network::INetChannel uint32 nTimerPushTx; bool fStartIdlePushTxTimer; std::set setPushTxFork; + + std::atomic nNodeCat; }; } // namespace bigbang From 90a73274596dc03682bc5ad60c118eb1b9643e74 Mon Sep 17 00:00:00 2001 From: OuYun Date: Wed, 8 Apr 2020 17:54:00 +0800 Subject: [PATCH 059/181] add allowed fork list for fork nodes when launching --- src/bigbang/mqcluster.cpp | 11 +++++++++++ src/bigbang/mqcluster.h | 1 + src/storage/mqdb.h | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 59977183..4e048148 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -82,6 +82,12 @@ bool CMQCluster::HandleInitialize() return false; } + if (!GetObject("forkmanager", pForkManager)) + { + Error("Failed to request forkmanager"); + return false; + } + Log("CMQCluster::HandleInitialize() successfully"); return true; } @@ -92,6 +98,7 @@ void CMQCluster::HandleDeinitialize() pBlockChain = nullptr; pDispatcher = nullptr; pService = nullptr; + pForkManager = nullptr; } bool CMQCluster::HandleInvoke() @@ -137,6 +144,7 @@ bool CMQCluster::HandleInvoke() "itself to dpos node yet[%d]", mapSuperNode.size()); } + if (mapSuperNode.size() > 1) { Error("CMQCluster::HandleInvoke(): fork node should have one " @@ -144,8 +152,11 @@ bool CMQCluster::HandleInvoke() mapSuperNode.size()); return false; } + if (1 == mapSuperNode.size()) { + pForkManager->SetForkFilter(mapSuperNode.begin()->second); + clientID = mapSuperNode.begin()->first; topicReqBlk = "Cluster01/" + clientID + "/SyncBlockReq"; topicRespBlk = "Cluster01/" + clientID + "/SyncBlockResp"; diff --git a/src/bigbang/mqcluster.h b/src/bigbang/mqcluster.h index bb3f558b..a5ba74ed 100644 --- a/src/bigbang/mqcluster.h +++ b/src/bigbang/mqcluster.h @@ -144,6 +144,7 @@ class CMQCluster : public IMQCluster IBlockChain* pBlockChain; IDispatcher* pDispatcher; IService* pService; + IForkManager* pForkManager; boost::mutex mutex; boost::condition_variable condMQ; diff --git a/src/storage/mqdb.h b/src/storage/mqdb.h index ff400812..5eb5d4ea 100644 --- a/src/storage/mqdb.h +++ b/src/storage/mqdb.h @@ -66,7 +66,7 @@ class CSuperNode class CSuperNodeDB : public xengine::CKVDB { public: - CSuperNodeDB() {} + CSuperNodeDB() = default; bool Initialize(const boost::filesystem::path& pathData); void Deinitialize(); bool AddNewSuperNode(const CSuperNode& cli); From c003a103a477f86ab58e56733afede63219de0de Mon Sep 17 00:00:00 2001 From: OuYun Date: Wed, 8 Apr 2020 22:37:05 +0800 Subject: [PATCH 060/181] set filters in netchannel for super nodes --- src/bigbang/forkmanager.cpp | 2 + src/bigbang/mqcluster.cpp | 8 ++-- src/bigbang/netchn.cpp | 79 +++++++++++++++++++++++++++++++++++-- 3 files changed, 82 insertions(+), 7 deletions(-) diff --git a/src/bigbang/forkmanager.cpp b/src/bigbang/forkmanager.cpp index add7132a..b51fd162 100644 --- a/src/bigbang/forkmanager.cpp +++ b/src/bigbang/forkmanager.cpp @@ -285,6 +285,7 @@ bool CForkManager::SetForkFilter(const std::vector& vFork, const std::v if (fork != 0) { setForkAllowed.insert(fork); + Log("CForkManager::SetForkFilter: set fork filter[%s]", fork.ToString().c_str()); } } for (auto const& group : vGroup) @@ -292,6 +293,7 @@ bool CForkManager::SetForkFilter(const std::vector& vFork, const std::v if (group != 0) { setGroupAllowed.insert(group); + Log("CForkManager::SetForkFilter: set fork group filter[%s]", group.ToString().c_str()); } } } diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 4e048148..603a1d67 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -147,8 +147,8 @@ bool CMQCluster::HandleInvoke() if (mapSuperNode.size() > 1) { - Error("CMQCluster::HandleInvoke(): fork node should have one " - "single enrollment[%d]", + Error("CMQCluster::HandleInvoke(): fork node should only have one " + "single enrollment but [%d]", mapSuperNode.size()); return false; } @@ -172,11 +172,11 @@ bool CMQCluster::HandleInvoke() fork.ToString().c_str(), clientID.c_str()); } - if (!PostBlockRequest(-1)) +/* if (!PostBlockRequest(-1)) { Error("CMQCluster::HandleInvoke(): failed to post requesting block"); return false; - } + }*/ } } else if (NODE_CATEGORY::DPOSNODE == catNode) diff --git a/src/bigbang/netchn.cpp b/src/bigbang/netchn.cpp index 4d0c6e1e..caf92cd5 100644 --- a/src/bigbang/netchn.cpp +++ b/src/bigbang/netchn.cpp @@ -310,7 +310,13 @@ void CNetChannel::SubscribeFork(const uint256& hashFork, const uint64& nNonce) if (hashFork == pCoreProtocol->GetGenesisBlockHash() && 1 == nNodeCat) { - StdLog("NetChannel", "SubscribeFork: success in filtering main chain for fork node with nonce[%d]", nNonce); + StdLog("NetChannel", "SubscribeFork: succeed in filtering the main chain for fork nodes with nonce[%d]", nNonce); + return; + } + + if (hashFork != pCoreProtocol->GetGenesisBlockHash() && 2 == nNodeCat) + { + StdLog("NetChannel", "SubscribeFork: succeed in filtering biz chains for dpos nodes with nonce[%d]", nNonce); return; } @@ -345,9 +351,9 @@ void CNetChannel::UnsubscribeFork(const uint256& hashFork) { boost::shared_lock rlock(rwNetPeer); - for (map::iterator it = mapPeer.begin(); it != mapPeer.end(); ++it) + for (const auto& p : mapPeer) { - eventUnsubscribe.nNonce = (*it).first; + eventUnsubscribe.nNonce = p.first; pPeerNet->DispatchEvent(&eventUnsubscribe); } } @@ -383,6 +389,7 @@ bool CNetChannel::HandleEvent(network::CEventPeerActive& eventActive) if (sch.first != pCoreProtocol->GetGenesisBlockHash()) { eventSubscribe.data.push_back(sch.first); + StdLog("NetChannel", "CEventPeerActive: prepare to subscribe fork[%s]", sch.first.ToString().c_str()); } } } @@ -687,6 +694,28 @@ bool CNetChannel::HandleEvent(network::CEventPeerGetBlocks& eventGetBlocks) { uint64 nNonce = eventGetBlocks.nNonce; uint256& hashFork = eventGetBlocks.hashFork; + + if (1 == nNodeCat) + { + if (hashFork == pCoreProtocol->GetGenesisBlockHash()) + { + StdTrace("NetChannel", "CEventPeerGetBlocks: peer[%s] is asking main chain blocks " + "from a fork node, just ignore it", + GetPeerAddressInfo(nNonce).c_str()); + return true; + } + } + if (2 == nNodeCat) + { + if (hashFork != pCoreProtocol->GetGenesisBlockHash()) + { + StdTrace("NetChannel", "CEventPeerGetBlocks: peer[%s] is asking biz chain blocks " + "from a dpos node, just ignore it", + GetPeerAddressInfo(nNonce).c_str()); + return true; + } + } + vector vBlockHash; StdTrace("NetChannel", "CEventPeerGetBlocks: peer: %s, fork: %s", @@ -741,6 +770,28 @@ bool CNetChannel::HandleEvent(network::CEventPeerTx& eventTx) { uint64 nNonce = eventTx.nNonce; uint256& hashFork = eventTx.hashFork; + + if (1 == nNodeCat) + { + if (hashFork == pCoreProtocol->GetGenesisBlockHash()) + { + StdTrace("NetChannel", "CEventPeerTx: peer[%s] is feeding a main chain tx " + "to a fork node, just ignore it", + GetPeerAddressInfo(nNonce).c_str()); + return true; + } + } + if (2 == nNodeCat) + { + if (hashFork != pCoreProtocol->GetGenesisBlockHash()) + { + StdTrace("NetChannel", "CEventPeerTx: peer[%s] is feeding a biz chain tx " + "to a dpos node, just ignore it", + GetPeerAddressInfo(nNonce).c_str()); + return true; + } + } + CTransaction& tx = eventTx.data; uint256 txid = tx.GetHash(); @@ -793,6 +844,28 @@ bool CNetChannel::HandleEvent(network::CEventPeerBlock& eventBlock) { uint64 nNonce = eventBlock.nNonce; uint256& hashFork = eventBlock.hashFork; + + if (1 == nNodeCat) + { + if (hashFork == pCoreProtocol->GetGenesisBlockHash()) + { + StdTrace("NetChannel", "CEventPeerBlock: peer[%s] is feeding a main chain block " + "to a fork node, just ignore it", + GetPeerAddressInfo(nNonce).c_str()); + return true; + } + } + if (2 == nNodeCat) + { + if (hashFork != pCoreProtocol->GetGenesisBlockHash()) + { + StdTrace("NetChannel", "CEventPeerBlock: peer[%s] is feeding a biz chain block " + "to a dpos node, just ignore it", + GetPeerAddressInfo(nNonce).c_str()); + return true; + } + } + CBlock& block = eventBlock.data; uint256 hash = block.GetHash(); From 07cff812fc2a679bc6b08998b87082c328a21470 Mon Sep 17 00:00:00 2001 From: OuYun Date: Thu, 9 Apr 2020 18:12:38 +0800 Subject: [PATCH 061/181] hybrid nodes interaction: built infrastructure --- src/bigbang/netchn.cpp | 148 +++++++++++++++++++++++++++++++++++++++- src/bigbang/netchn.h | 2 + src/network/peerevent.h | 6 ++ src/network/peernet.cpp | 38 +++++++++++ src/network/peernet.h | 2 + src/network/proto.h | 21 ++++++ src/storage/mqdb.cpp | 1 - src/storage/mqdb.h | 44 ++++++++++++ 8 files changed, 260 insertions(+), 2 deletions(-) diff --git a/src/bigbang/netchn.cpp b/src/bigbang/netchn.cpp index caf92cd5..a3dedc88 100644 --- a/src/bigbang/netchn.cpp +++ b/src/bigbang/netchn.cpp @@ -378,7 +378,8 @@ bool CNetChannel::HandleEvent(network::CEventPeerActive& eventActive) else { StdLog("NetChannel", "CEventPeerActive: succeed in filtering to " - "get main chain from peer[%s]", GetPeerAddressInfo(nNonce).c_str()); + "get main chain from peer[%s]", + GetPeerAddressInfo(nNonce).c_str()); } network::CEventPeerSubscribe eventSubscribe(nNonce, pCoreProtocol->GetGenesisBlockHash()); @@ -438,6 +439,151 @@ bool CNetChannel::HandleEvent(network::CEventPeerDeactive& eventDeactive) return true; } +bool CNetChannel::HandleEvent(network::CEventPeerGetBizForks& eventGetBizForks) +{ + uint64 nNonce = eventGetBizForks.nNonce; + vector& bizForks = eventGetBizForks.data; + StdLog("NetChannel", "CEventPeerGetBizForks: peer[%s] is asking for total [%d] biz forks", + GetPeerAddressInfo(nNonce).c_str(), bizForks.size()); + for (auto const& f : bizForks) + { + StdLog("NetChannel", "CEventPeerGetBizForks: peer[%s] is asking for fork[%s]", + GetPeerAddressInfo(nNonce).c_str(), f.ToString().c_str()); + } + + vector nodes; + if (!pBlockChain->ListSuperNode(nodes)) + { + StdError("NetChannel", "CEventPeerGetBizForks: ListSuperNode failed"); + return false; + } + + StdLog("NetChannel", "CEventPeerGetBizForks: there is/are [%d] node(s) all", nodes.size()); + for (auto const& node : nodes) + { + StdLog("NetChannel", "CEventPeerGetBizForks: biz fork as [%s]", node.ToString().c_str()); + } + + //filter out dpos node itself + for (auto it = nodes.begin(); it != nodes.end(); ++it) + { + if (0 == it->ipAddr) + { + nodes.erase(it); + break; + } + } + + StdLog("NetChannel", "CEventPeerGetBizForks: there is/are [%d] node(s) without dpos node", nodes.size()); + for (auto const& node : nodes) + { + StdLog("NetChannel", "CEventPeerGetBizForks: biz fork as [%s] w/o dpos node", node.ToString().c_str()); + } + + storage::CForkKnownIPSet setForkIp; + for (auto const& node : nodes) + { + for (auto const& fork : node.vecOwnedForks) + { + setForkIp.emplace(storage::CForkKnownIP(fork, node.ipAddr)); + } + } + + if (bizForks.empty()) + { //get all to peer + map> mapForkIp; + const storage::CForkKnownIpSetById& idxForkID = setForkIp.get<0>(); + for (auto const& it : idxForkID) + { + mapForkIp[it.forkID].emplace_back(it.nodeIP); + } + + network::CEventPeerBizForks eventFork(nNonce); + eventFork.data = mapForkIp; + pPeerNet->DispatchEvent(&eventFork); + + return true; + } + + for (auto const& fork : bizForks) + { + map> mapForkIp; + const storage::CForkKnownIpSetById& idxForkID = setForkIp.get<0>(); + auto itBegin = idxForkID.equal_range(fork).first; + auto itEnd = idxForkID.equal_range(fork).second; + + while (itBegin != itEnd) + { + mapForkIp[fork].emplace_back(itBegin->nodeIP); + ++itBegin; + } + + network::CEventPeerBizForks eventFork(nNonce); + eventFork.data = mapForkIp; + pPeerNet->DispatchEvent(&eventFork); + } + + return true; +} + +bool CNetChannel::HandleEvent(network::CEventPeerBizForks& eventBizForks) +{ + uint64 nNonce = eventBizForks.nNonce; + map> mapForkIp = eventBizForks.data.mapFork; + + StdLog("NetChannel", "CEventPeerBizForks: peer[%s] is feeding total [%d] biz forks", + GetPeerAddressInfo(nNonce).c_str(), mapForkIp.size()); + for (auto const& fork : mapForkIp) + { + StdLog("NetChannel", "CEventPeerBizForks: peer[%s] is feeding total [%d] IPs for fork[%s]", + fork.second.size(), GetPeerAddressInfo(nNonce).c_str()); + for (auto const& ip : fork.second) + { + StdLog("NetChannel", "CEventPeerBizForks: peer[%s] is feeding IP[%s] for fork[%s]", + storage::CSuperNode::Int2Ip(ip).c_str(), GetPeerAddressInfo(nNonce).c_str()); + } + } + + storage::CForkKnownIPSet setForkIp; + for (auto const& fork : mapForkIp) + { + for (auto const ip : fork.second) + { + setForkIp.emplace(storage::CForkKnownIP(fork.first, ip)); + } + } + + map> mapIpForks; + const storage::CForkKnownIpSetByIp& idxNodeID = setForkIp.get<1>(); + for (auto const& it : idxNodeID) + { + mapIpForks[it.nodeIP].emplace_back(it.forkID); + } + + vector nodes; + for (auto const& it : mapIpForks) + { + nodes.emplace_back(storage::CSuperNode(storage::CLIENT_ID_OUT_OF_MQ_CLUSTER, it.first, it.second, 0)); + } + + for (auto const& node : nodes) + { + if (!pBlockChain->AddNewSuperNode(node)) + { + StdError("NetChannel", "CEventPeerBizForks: AddNewSuperNode failed"); + return false; + } + } + + //if dpos node self, dispatch them to fork nodes by MQ + if (2 == nNodeCat) + { + ; + } + + return true; +} + bool CNetChannel::HandleEvent(network::CEventPeerSubscribe& eventSubscribe) { uint64 nNonce = eventSubscribe.nNonce; diff --git a/src/bigbang/netchn.h b/src/bigbang/netchn.h index 24a57536..efd60327 100644 --- a/src/bigbang/netchn.h +++ b/src/bigbang/netchn.h @@ -270,6 +270,8 @@ class CNetChannel : public network::INetChannel bool HandleEvent(network::CEventPeerActive& eventActive) override; bool HandleEvent(network::CEventPeerDeactive& eventDeactive) override; + bool HandleEvent(network::CEventPeerGetBizForks& eventGetBizForks) override; + bool HandleEvent(network::CEventPeerBizForks& eventBizForks) override; bool HandleEvent(network::CEventPeerSubscribe& eventSubscribe) override; bool HandleEvent(network::CEventPeerUnsubscribe& eventUnsubscribe) override; bool HandleEvent(network::CEventPeerInv& eventInv) override; diff --git a/src/network/peerevent.h b/src/network/peerevent.h index defb3c6d..c55929fe 100644 --- a/src/network/peerevent.h +++ b/src/network/peerevent.h @@ -23,6 +23,8 @@ enum //PEER EVENT_PEER_ACTIVE, EVENT_PEER_DEACTIVE, + EVENT_PEER_GETBIZFORKS, + EVENT_PEER_BIZFORKS, EVENT_PEER_SUBSCRIBE, EVENT_PEER_UNSUBSCRIBE, EVENT_PEER_INV, @@ -209,6 +211,8 @@ class CBbPeerEventListener; typedef TYPE_PEEREVENT(EVENT_PEER_ACTIVE, CAddress) CEventPeerActive; typedef TYPE_PEEREVENT(EVENT_PEER_DEACTIVE, CAddress) CEventPeerDeactive; +typedef TYPE_PEEREVENT(EVENT_PEER_GETBIZFORKS, std::vector) CEventPeerGetBizForks; +typedef TYPE_PEEREVENT(EVENT_PEER_BIZFORKS, CBizFork) CEventPeerBizForks; typedef TYPE_PEERDATAEVENT(EVENT_PEER_SUBSCRIBE, std::vector) CEventPeerSubscribe; typedef TYPE_PEERDATAEVENT(EVENT_PEER_UNSUBSCRIBE, std::vector) CEventPeerUnsubscribe; typedef TYPE_PEERDATAEVENT(EVENT_PEER_INV, std::vector) CEventPeerInv; @@ -230,6 +234,8 @@ class CBbPeerEventListener : virtual public xengine::CEventListener virtual ~CBbPeerEventListener() {} DECLARE_EVENTHANDLER(CEventPeerActive); DECLARE_EVENTHANDLER(CEventPeerDeactive); + DECLARE_EVENTHANDLER(CEventPeerGetBizForks); + DECLARE_EVENTHANDLER(CEventPeerBizForks); DECLARE_EVENTHANDLER(CEventPeerSubscribe); DECLARE_EVENTHANDLER(CEventPeerUnsubscribe); DECLARE_EVENTHANDLER(CEventPeerInv); diff --git a/src/network/peernet.cpp b/src/network/peernet.cpp index 1b25e78a..62fcd268 100644 --- a/src/network/peernet.cpp +++ b/src/network/peernet.cpp @@ -70,6 +70,20 @@ void CBbPeerNet::HandleDeinitialize() // pDelegatedChannel = nullptr; } +bool CBbPeerNet::HandleEvent(CEventPeerGetBizForks& eventGetBizForks) +{ + CBufStream ssPayload; + ssPayload << eventGetBizForks; + return SendDataMessage(eventGetBizForks.nNonce, PROTO_CMD_GETBIZFORK, ssPayload); +} + +bool CBbPeerNet::HandleEvent(CEventPeerBizForks& eventBizForks) +{ + CBufStream ssPayload; + ssPayload << eventBizForks; + return SendDataMessage(eventBizForks.nNonce, PROTO_CMD_BIZFORK, ssPayload); +} + bool CBbPeerNet::HandleEvent(CEventPeerSubscribe& eventSubscribe) { CBufStream ssPayload; @@ -425,6 +439,7 @@ bool CBbPeerNet::HandlePeerRecvMessage(CPeer* pPeer, int nChannel, int nCommand, return pBbPeer->SendMessage(PROTO_CHN_NETWORK, PROTO_CMD_ADDRESS, ss); } case PROTO_CMD_ADDRESS: + { if (!fEnclosed) { vector vAddr; @@ -467,6 +482,7 @@ bool CBbPeerNet::HandlePeerRecvMessage(CPeer* pPeer, int nChannel, int nCommand, return true; } break; + } case PROTO_CMD_PING: { uint32 nSeq; @@ -495,6 +511,28 @@ bool CBbPeerNet::HandlePeerRecvMessage(CPeer* pPeer, int nChannel, int nCommand, } return true; } + case PROTO_CMD_GETBIZFORK: + { + CEventPeerGetBizForks* pEvent = new CEventPeerGetBizForks(pBbPeer->GetNonce()); + if (pEvent != nullptr) + { + ssPayload >> pEvent->data; + pNetChannel->PostEvent(pEvent); + return true; + } + break; + } + case PROTO_CMD_BIZFORK: + { + CEventPeerBizForks* pEvent = new CEventPeerBizForks(pBbPeer->GetNonce()); + if (pEvent != nullptr) + { + ssPayload >> pEvent->data; + pNetChannel->PostEvent(pEvent); + return true; + } + break; + } default: break; } diff --git a/src/network/peernet.h b/src/network/peernet.h index 87ad7914..868e1c7c 100644 --- a/src/network/peernet.h +++ b/src/network/peernet.h @@ -55,6 +55,8 @@ class CBbPeerNet : public xengine::CPeerNet, virtual public CBbPeerEventListener protected: bool HandleInitialize() override; void HandleDeinitialize() override; + bool HandleEvent(CEventPeerGetBizForks& eventGetBizForks) override; + bool HandleEvent(CEventPeerBizForks& eventBizForks) override; bool HandleEvent(CEventPeerSubscribe& eventSubscribe) override; bool HandleEvent(CEventPeerUnsubscribe& eventUnsubscribe) override; bool HandleEvent(CEventPeerInv& eventInv) override; diff --git a/src/network/proto.h b/src/network/proto.h index bf742bbd..4f98a5b6 100644 --- a/src/network/proto.h +++ b/src/network/proto.h @@ -37,6 +37,8 @@ enum PROTO_CMD_ADDRESS = 4, PROTO_CMD_PING = 5, PROTO_CMD_PONG = 6, + PROTO_CMD_GETBIZFORK = 7, + PROTO_CMD_BIZFORK = 8, }; enum @@ -130,6 +132,25 @@ class CPeerMessageHeader } }; +class CBizFork +{ + friend class xengine::CStream; + +public: + CBizFork() {} + CBizFork(const std::map>& rForkIn) : mapFork(rForkIn){} + +protected: + template + void Serialize(xengine::CStream& s, O& opt) + { + s.Serialize(mapFork, opt); + } + +public: + std::map> mapFork; +}; + class CMsgRsp { friend class xengine::CStream; diff --git a/src/storage/mqdb.cpp b/src/storage/mqdb.cpp index fb068476..4b49816b 100644 --- a/src/storage/mqdb.cpp +++ b/src/storage/mqdb.cpp @@ -14,7 +14,6 @@ namespace bigbang namespace storage { -const string CLIENT_ID_OUT_OF_MQ_CLUSTER = "OUTER-NODE"; ////////////////////////////// // CSuperNodeDB diff --git a/src/storage/mqdb.h b/src/storage/mqdb.h index 5eb5d4ea..34b55fe4 100644 --- a/src/storage/mqdb.h +++ b/src/storage/mqdb.h @@ -15,6 +15,31 @@ namespace bigbang namespace storage { +const std::string CLIENT_ID_OUT_OF_MQ_CLUSTER = "OUTER-NODE"; + +class CForkKnownIP +{ +public: + CForkKnownIP() {} + CForkKnownIP(const uint256& forkidIn, const uint32& nodeipIn) + : forkID(forkidIn), nodeIP(nodeipIn) {} + +public: + uint256 forkID; + uint32 nodeIP; +}; + +typedef boost::multi_index_container< + CForkKnownIP, + boost::multi_index::indexed_by< + boost::multi_index::ordered_non_unique>, + boost::multi_index::ordered_non_unique> + > +> CForkKnownIPSet; + +typedef CForkKnownIPSet::nth_index<0>::type CForkKnownIpSetById; +typedef CForkKnownIPSet::nth_index<1>::type CForkKnownIpSetByIp; + class CSuperNode { friend class xengine::CStream; @@ -52,6 +77,25 @@ class CSuperNode return true; } + static std::string Int2Ip(const unsigned long& ipNum) + { + boost::asio::ip::address_v4 addr(ipNum); + return addr.to_string(); + } + + std::string ToString() const + { + std::ostringstream oss; + oss << "CSuperNode : superNodeID=" << superNodeID + << " ipAddr=" << Int2Ip(ipAddr) << "\n"; + for (auto const& f : vecOwnedForks) + { + oss << " ownedFork=" << f.ToString() << "\n"; + } + oss << " nodeCat=" << nodeCat << "\n"; + return oss.str(); + } + protected: template void Serialize(xengine::CStream& s, O& opt) From bca1a995ddf219e5de77d12ffa97b83f7904a360 Mon Sep 17 00:00:00 2001 From: OuYun Date: Fri, 10 Apr 2020 10:32:02 +0800 Subject: [PATCH 062/181] code optimize: const value to enum --- src/bigbang/dispatcher.cpp | 3 ++- src/bigbang/netchn.cpp | 27 ++++++++++++++------------- src/bigbang/rpcmod.cpp | 3 ++- src/common/defs.h | 6 ++++++ src/storage/mqdb.cpp | 17 +++++++++-------- 5 files changed, 33 insertions(+), 23 deletions(-) diff --git a/src/bigbang/dispatcher.cpp b/src/bigbang/dispatcher.cpp index 1cfbb178..36e7562e 100644 --- a/src/bigbang/dispatcher.cpp +++ b/src/bigbang/dispatcher.cpp @@ -8,6 +8,7 @@ #include #include +#include "defs.h" #include "event.h" using namespace std; @@ -362,7 +363,7 @@ void CDispatcher::UpdatePrimaryBlock(const CBlock& block, const CBlockChainUpdat // SyncForkHeight(updateBlockChain.nLastBlockHeight); - if (2 == nNodeCat && !updateBlockChain.vBlockRemove.empty()) + if (NODE_CAT_DPOSNODE == nNodeCat && !updateBlockChain.vBlockRemove.empty()) { CEventMQChainUpdate* pMqChainUpdate = new CEventMQChainUpdate(0); if (pMqChainUpdate != nullptr) diff --git a/src/bigbang/netchn.cpp b/src/bigbang/netchn.cpp index a3dedc88..c235ede1 100644 --- a/src/bigbang/netchn.cpp +++ b/src/bigbang/netchn.cpp @@ -5,6 +5,7 @@ #include "netchn.h" #include +#include #include "schedule.h" @@ -140,7 +141,7 @@ CNetChannel::CNetChannel() pService = nullptr; pDispatcher = nullptr; fStartIdlePushTxTimer = false; - nNodeCat = 0; + nNodeCat = NODE_CAT_BBCNODE; } CNetChannel::~CNetChannel() @@ -308,13 +309,13 @@ void CNetChannel::SubscribeFork(const uint256& hashFork, const uint64& nNonce) StdLog("NetChannel", "SubscribeFork: mapSched insert success, hashFork: %s", hashFork.GetHex().c_str()); } - if (hashFork == pCoreProtocol->GetGenesisBlockHash() && 1 == nNodeCat) + if (hashFork == pCoreProtocol->GetGenesisBlockHash() && NODE_CAT_FORKNODE == nNodeCat) { StdLog("NetChannel", "SubscribeFork: succeed in filtering the main chain for fork nodes with nonce[%d]", nNonce); return; } - if (hashFork != pCoreProtocol->GetGenesisBlockHash() && 2 == nNodeCat) + if (hashFork != pCoreProtocol->GetGenesisBlockHash() && NODE_CAT_DPOSNODE == nNodeCat) { StdLog("NetChannel", "SubscribeFork: succeed in filtering biz chains for dpos nodes with nonce[%d]", nNonce); return; @@ -370,7 +371,7 @@ bool CNetChannel::HandleEvent(network::CEventPeerActive& eventActive) StdLog("NetChannel", "CEventPeerActive: peer: %s", GetPeerAddressInfo(nNonce).c_str()); if ((eventActive.data.nService & network::NODE_NETWORK)) { - if (1 != nNodeCat) + if (NODE_CAT_FORKNODE != nNodeCat) { DispatchGetBlocksEvent(nNonce, pCoreProtocol->GetGenesisBlockHash()); BroadcastTxInv(pCoreProtocol->GetGenesisBlockHash()); @@ -490,7 +491,7 @@ bool CNetChannel::HandleEvent(network::CEventPeerGetBizForks& eventGetBizForks) } if (bizForks.empty()) - { //get all to peer + { //get all to peer map> mapForkIp; const storage::CForkKnownIpSetById& idxForkID = setForkIp.get<0>(); for (auto const& it : idxForkID) @@ -576,7 +577,7 @@ bool CNetChannel::HandleEvent(network::CEventPeerBizForks& eventBizForks) } //if dpos node self, dispatch them to fork nodes by MQ - if (2 == nNodeCat) + if (NODE_CAT_DPOSNODE == nNodeCat) { ; } @@ -597,7 +598,7 @@ bool CNetChannel::HandleEvent(network::CEventPeerSubscribe& eventSubscribe) { for (const uint256& hash : eventSubscribe.data) { - if (2 == nNodeCat) + if (NODE_CAT_DPOSNODE == nNodeCat) { if (hash == pCoreProtocol->GetGenesisBlockHash()) { @@ -841,7 +842,7 @@ bool CNetChannel::HandleEvent(network::CEventPeerGetBlocks& eventGetBlocks) uint64 nNonce = eventGetBlocks.nNonce; uint256& hashFork = eventGetBlocks.hashFork; - if (1 == nNodeCat) + if (NODE_CAT_FORKNODE == nNodeCat) { if (hashFork == pCoreProtocol->GetGenesisBlockHash()) { @@ -851,7 +852,7 @@ bool CNetChannel::HandleEvent(network::CEventPeerGetBlocks& eventGetBlocks) return true; } } - if (2 == nNodeCat) + if (NODE_CAT_DPOSNODE == nNodeCat) { if (hashFork != pCoreProtocol->GetGenesisBlockHash()) { @@ -917,7 +918,7 @@ bool CNetChannel::HandleEvent(network::CEventPeerTx& eventTx) uint64 nNonce = eventTx.nNonce; uint256& hashFork = eventTx.hashFork; - if (1 == nNodeCat) + if (NODE_CAT_FORKNODE == nNodeCat) { if (hashFork == pCoreProtocol->GetGenesisBlockHash()) { @@ -927,7 +928,7 @@ bool CNetChannel::HandleEvent(network::CEventPeerTx& eventTx) return true; } } - if (2 == nNodeCat) + if (NODE_CAT_DPOSNODE == nNodeCat) { if (hashFork != pCoreProtocol->GetGenesisBlockHash()) { @@ -991,7 +992,7 @@ bool CNetChannel::HandleEvent(network::CEventPeerBlock& eventBlock) uint64 nNonce = eventBlock.nNonce; uint256& hashFork = eventBlock.hashFork; - if (1 == nNodeCat) + if (NODE_CAT_FORKNODE == nNodeCat) { if (hashFork == pCoreProtocol->GetGenesisBlockHash()) { @@ -1001,7 +1002,7 @@ bool CNetChannel::HandleEvent(network::CEventPeerBlock& eventBlock) return true; } } - if (2 == nNodeCat) + if (NODE_CAT_DPOSNODE == nNodeCat) { if (hashFork != pCoreProtocol->GetGenesisBlockHash()) { diff --git a/src/bigbang/rpcmod.cpp b/src/bigbang/rpcmod.cpp index 67af6381..e02fb198 100644 --- a/src/bigbang/rpcmod.cpp +++ b/src/bigbang/rpcmod.cpp @@ -14,6 +14,7 @@ //#include #include "address.h" +#include "defs.h" #include "mqdb.h" #include "rpc/auto_protocol.h" #include "template/proof.h" @@ -2401,7 +2402,7 @@ CRPCResultPtr CRPCMod::RPCListUnspent(CRPCParamPtr param) CRPCResultPtr CRPCMod::RPCEnrollSuperNode(rpc::CRPCParamPtr param) { int nNodeCat = dynamic_cast(Config())->nCatOfNode; - if (1 != nNodeCat && 2 != nNodeCat) + if (NODE_CAT_FORKNODE != nNodeCat && NODE_CAT_DPOSNODE != nNodeCat) { throw CRPCException(RPC_INVALID_REQUEST, "Only super node has the feature"); } diff --git a/src/common/defs.h b/src/common/defs.h index 6c3611cb..e8598434 100644 --- a/src/common/defs.h +++ b/src/common/defs.h @@ -15,5 +15,11 @@ static const unsigned int HEIGHT_HASH_TX_DATA_MAINNET = 133060; static const unsigned int HEIGHT_HASH_TX_DATA_TESTNET = 40; extern unsigned int HEIGHT_HASH_TX_DATA; +enum +{ + NODE_CAT_BBCNODE = 0, + NODE_CAT_FORKNODE = 1, + NODE_CAT_DPOSNODE = 2 +}; #endif //BIGBANG_DEFS_H diff --git a/src/storage/mqdb.cpp b/src/storage/mqdb.cpp index 4b49816b..8ac87a39 100644 --- a/src/storage/mqdb.cpp +++ b/src/storage/mqdb.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "mqdb.h" - +#include "defs.h" #include "leveldbeng.h" using namespace std; @@ -44,14 +44,15 @@ void CSuperNodeDB::Deinitialize() bool CSuperNodeDB::AddNewSuperNode(const CSuperNode& cli) { bool ret = false; - switch(cli.nodeCat) + switch (cli.nodeCat) { - case 0: + case NODE_CAT_BBCNODE: { ret = Write(make_pair(CLIENT_ID_OUT_OF_MQ_CLUSTER, cli.ipAddr), cli.vecOwnedForks, true); break; } - case 1: case 2: + case NODE_CAT_FORKNODE: + case NODE_CAT_DPOSNODE: { if (!ClearSuperNode(cli)) { @@ -105,7 +106,7 @@ void CSuperNodeDB::Clear() } bool CSuperNodeDB::FetchSuperNodeWalker(xengine::CBufStream& ssKey, xengine::CBufStream& ssValue, - map, vector>& mapCli) + map, vector>& mapCli) { string strCliID; uint32 nIP; @@ -151,7 +152,7 @@ bool CSuperNodeDB::ClearSuperNode(const CSuperNode& cli) return false; } - if (1 == cli.nodeCat) //fork node updates self by remove all supernode entries + if (NODE_CAT_FORKNODE == cli.nodeCat) //fork node updates self by remove all supernode entries { for (auto const& supernode : vSuperNode) { @@ -163,7 +164,7 @@ bool CSuperNodeDB::ClearSuperNode(const CSuperNode& cli) return true; } - if (2 == cli.nodeCat && 0 == cli.ipAddr) //dpos node updates oneself + if (NODE_CAT_DPOSNODE == cli.nodeCat && 0 == cli.ipAddr) //dpos node updates oneself { for (auto const& supernode : vSuperNode) { @@ -178,7 +179,7 @@ bool CSuperNodeDB::ClearSuperNode(const CSuperNode& cli) } bool CSuperNodeDB::LoadSuperNodeWalker(xengine::CBufStream& ssKey, xengine::CBufStream& ssValue, - map, vector>& mapCli) + map, vector>& mapCli) { string strCliID; uint32 nIP; From baee898ea23556b0422fedd90caef3ef96ee52bb Mon Sep 17 00:00:00 2001 From: OuYun Date: Fri, 10 Apr 2020 11:35:25 +0800 Subject: [PATCH 063/181] add new handler to deal with biz forks --- src/bigbang/mqcluster.cpp | 11 +++++++++++ src/bigbang/mqcluster.h | 1 + src/bigbang/mqevent.h | 6 +++++- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 603a1d67..4756e31d 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -325,6 +325,17 @@ bool CMQCluster::HandleEvent(CEventMQEnrollUpdate& eventMqUpdateEnroll) return true; } +bool CMQCluster::HandleEvent(CEventMQBizForkUpdate& eventMqBizFork) +{ + Log("CMQCluster::HandleEvent(): biz forks payload is coming"); + if (NODE_CATEGORY::DPOSNODE == catNode) + { + ; + } + + return true; +} + bool CMQCluster::HandleEvent(CEventMQAgreement& eventMqAgreement) { return true; diff --git a/src/bigbang/mqcluster.h b/src/bigbang/mqcluster.h index a5ba74ed..3f2184e8 100644 --- a/src/bigbang/mqcluster.h +++ b/src/bigbang/mqcluster.h @@ -135,6 +135,7 @@ class CMQCluster : public IMQCluster bool HandleEvent(CEventMQChainUpdate& eventMqUpdateChain) override; bool HandleEvent(CEventMQEnrollUpdate& eventMqUpdateEnroll) override; bool HandleEvent(CEventMQAgreement& eventMqAgreement) override; + bool HandleEvent(CEventMQBizForkUpdate& eventMqBizFork) override; bool IsAuthenticated() override; diff --git a/src/bigbang/mqevent.h b/src/bigbang/mqevent.h index d3b80ce6..c2b62555 100644 --- a/src/bigbang/mqevent.h +++ b/src/bigbang/mqevent.h @@ -19,7 +19,8 @@ enum EVENT_MQ_SYNCBLOCK, EVENT_MQ_UPDATEBLOCK, //dpos node deliver rollback block EVENT_MQ_ENROLLUPDATE, - EVENT_MQ_AGREEMENT + EVENT_MQ_AGREEMENT, + EVENT_MQ_UPDATEBIZFORK }; class CMQEventListener; @@ -30,6 +31,8 @@ typedef TYPE_MQEVENT(EVENT_MQ_SYNCBLOCK, std::string) CEventMQSyncBlock; typedef TYPE_MQEVENT(EVENT_MQ_UPDATEBLOCK, CMqRollbackUpdate) CEventMQChainUpdate; typedef TYPE_MQEVENT(EVENT_MQ_ENROLLUPDATE, CMqSuperNodeUpdate) CEventMQEnrollUpdate; typedef TYPE_MQEVENT(EVENT_MQ_AGREEMENT, CDelegateAgreement) CEventMQAgreement; +//typedef TYPE_MQEVENT(EVENT_MQ_UPDATEBIZFORK, std::vector) CEventMQBizForkUpdate; +typedef TYPE_MQEVENT(EVENT_MQ_UPDATEBIZFORK, storage::CForkKnownIPSet) CEventMQBizForkUpdate; class CMQEventListener : virtual public CEventListener { @@ -39,6 +42,7 @@ class CMQEventListener : virtual public CEventListener DECLARE_EVENTHANDLER(CEventMQChainUpdate); DECLARE_EVENTHANDLER(CEventMQEnrollUpdate); DECLARE_EVENTHANDLER(CEventMQAgreement); + DECLARE_EVENTHANDLER(CEventMQBizForkUpdate); }; } // namespace bigbang From 4b6d08cd6c4a1d58deddcbae9cd7226df446d01e Mon Sep 17 00:00:00 2001 From: OuYun Date: Fri, 10 Apr 2020 12:10:26 +0800 Subject: [PATCH 064/181] - add restriction to RPC enrollsupernode - update mq db --- script/template/rpc.json | 4 +++- src/bigbang/rpcmod.cpp | 10 ++++++++++ src/storage/mqdb.cpp | 5 +++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/script/template/rpc.json b/script/template/rpc.json index 4c799371..933be5df 100755 --- a/script/template/rpc.json +++ b/script/template/rpc.json @@ -710,7 +710,9 @@ "error": [ "{\"code\":-32600,\"message\":\"Invalid super node\"}", "{\"code\":-6,\"message\":\"Invalid ip address\"}", - "{\"code\":-6,\"message\":\"Unknown fork\"}", + "{\"code\":-6,\"message\":\"Failed: IP of fork node must not be 0.0.0.0\"}", + "{\"code\":-6,\"message\":\"Failed: IP of dpos node must be 0.0.0.0\"}", + "{\"code\":-6,\"message\":\"Invalid fork(s)\"}", "{\"code\":-32603,\"message\":\"Add super node enrollment failed\"}", "{\"code\":-32603,\"message\":\"List super node enrollment failed\"}", "{\"code\":-32603,\"message\":\"Failed to convert ip address\"}" diff --git a/src/bigbang/rpcmod.cpp b/src/bigbang/rpcmod.cpp index e02fb198..ad140066 100644 --- a/src/bigbang/rpcmod.cpp +++ b/src/bigbang/rpcmod.cpp @@ -2415,6 +2415,16 @@ CRPCResultPtr CRPCMod::RPCEnrollSuperNode(rpc::CRPCParamPtr param) throw CRPCException(RPC_INVALID_PARAMETER, "Invalid ip address"); } + if (NODE_CAT_FORKNODE == nNodeCat && 0 == ipNum) + { + throw CRPCException(RPC_INVALID_PARAMETER, "Failed: IP of fork node must not be 0.0.0.0"); + } + + if (NODE_CAT_DPOSNODE == nNodeCat && 0 != ipNum) + { + throw CRPCException(RPC_INVALID_PARAMETER, "Failed: IP of dpos node must be 0.0.0.0"); + } + std::string id = spParam->strClientid; std::vector vFork = spParam->vecForks; diff --git a/src/storage/mqdb.cpp b/src/storage/mqdb.cpp index 8ac87a39..e9877cb9 100644 --- a/src/storage/mqdb.cpp +++ b/src/storage/mqdb.cpp @@ -152,6 +152,11 @@ bool CSuperNodeDB::ClearSuperNode(const CSuperNode& cli) return false; } + if (vSuperNode.empty()) + { + return true; + } + if (NODE_CAT_FORKNODE == cli.nodeCat) //fork node updates self by remove all supernode entries { for (auto const& supernode : vSuperNode) From e8cfa8cbf848754a75a23e1ec8157e563c39bc40 Mon Sep 17 00:00:00 2001 From: OuYun Date: Fri, 10 Apr 2020 14:03:49 +0800 Subject: [PATCH 065/181] add condition for enrollsupernode RPC and update relative rpc json template --- script/template/rpc.json | 4 ++++ src/bigbang/rpcmod.cpp | 27 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/script/template/rpc.json b/script/template/rpc.json index 933be5df..ab78457c 100755 --- a/script/template/rpc.json +++ b/script/template/rpc.json @@ -713,6 +713,10 @@ "{\"code\":-6,\"message\":\"Failed: IP of fork node must not be 0.0.0.0\"}", "{\"code\":-6,\"message\":\"Failed: IP of dpos node must be 0.0.0.0\"}", "{\"code\":-6,\"message\":\"Invalid fork(s)\"}", + "{\"code\":-6,\"message\":\"No fork hash parameter\"}", + "{\"code\":-6,\"message\":\"Dpos node must have only main chain to enroll\"}", + "{\"code\":-6,\"message\":\"The main fork dpos node enrolls does not match\"}", + "{\"code\":-6,\"message\":\"The main fork fork node watches should not be the main fork\"}", "{\"code\":-32603,\"message\":\"Add super node enrollment failed\"}", "{\"code\":-32603,\"message\":\"List super node enrollment failed\"}", "{\"code\":-32603,\"message\":\"Failed to convert ip address\"}" diff --git a/src/bigbang/rpcmod.cpp b/src/bigbang/rpcmod.cpp index ad140066..36059ef7 100644 --- a/src/bigbang/rpcmod.cpp +++ b/src/bigbang/rpcmod.cpp @@ -2442,6 +2442,33 @@ CRPCResultPtr CRPCMod::RPCEnrollSuperNode(rpc::CRPCParamPtr param) } forks.emplace_back(fork); } + + if (forks.empty()) + { + throw CRPCException(RPC_INVALID_PARAMETER, "No fork hash parameter"); + } + + uint256 hashGenesis = pCoreProtocol->GetGenesisBlockHash(); + if (NODE_CAT_DPOSNODE == nNodeCat) + { + if (forks.size() > 1) + { + throw CRPCException(RPC_INVALID_PARAMETER, "Dpos node must have only main chain to enroll"); + } + else if (forks[0] != hashGenesis) + { + throw CRPCException(RPC_INVALID_PARAMETER, "The main fork dpos node enrolls does not match:[ " + hashGenesis.ToString() + " ]"); + } + } + + if (NODE_CAT_DPOSNODE == nNodeCat) + { + if (forks.end() != find(forks.begin(), forks.end(), hashGenesis)) + { + throw CRPCException(RPC_INVALID_PARAMETER, "The main fork fork node watches should not be the main fork:[ " + hashGenesis.ToString() + " ]"); + } + } + storage::CSuperNode newNode; newNode.superNodeID = std::move(id); newNode.ipAddr = ipNum; From c92a66d3c3e1b2c7fc5e3093113c6f44ed3a7c68 Mon Sep 17 00:00:00 2001 From: OuYun Date: Fri, 10 Apr 2020 18:14:34 +0800 Subject: [PATCH 066/181] refactor the following modules: - mqcluster - mqdb --- src/bigbang/base.h | 2 +- src/bigbang/blockchain.cpp | 4 +- src/bigbang/blockchain.h | 2 +- src/bigbang/dispatcher.cpp | 2 +- src/bigbang/mqcluster.cpp | 215 +++++++++++++++++++++---------------- src/bigbang/mqcluster.h | 6 +- src/bigbang/struct.h | 1 + src/storage/blockbase.cpp | 4 +- src/storage/blockbase.h | 2 +- src/storage/blockdb.cpp | 4 +- src/storage/blockdb.h | 2 +- src/storage/mqdb.cpp | 20 +++- src/storage/mqdb.h | 4 +- 13 files changed, 153 insertions(+), 115 deletions(-) diff --git a/src/bigbang/base.h b/src/bigbang/base.h index 0dadaf55..5f42014b 100644 --- a/src/bigbang/base.h +++ b/src/bigbang/base.h @@ -92,7 +92,7 @@ class IBlockChain : public xengine::IBase virtual Errno AddNewOrigin(const CBlock& block, CBlockChainUpdate& update) = 0; virtual Errno AddNewSuperNode(const storage::CSuperNode& node) = 0; virtual bool ListSuperNode(std::vector& nodes) = 0; - virtual bool FetchSuperNode(std::vector& nodes) = 0; + virtual bool FetchSuperNode(std::vector& nodes, const uint8 mask) = 0; virtual bool GetProofOfWorkTarget(const uint256& hashPrev, int nAlgo, int& nBits, int64& nReward) = 0; virtual bool GetBlockMintReward(const uint256& hashPrev, int64& nReward) = 0; virtual bool GetBlockLocator(const uint256& hashFork, CBlockLocator& locator, uint256& hashDepth, int nIncStep) = 0; diff --git a/src/bigbang/blockchain.cpp b/src/bigbang/blockchain.cpp index a8b0de19..389cfd91 100644 --- a/src/bigbang/blockchain.cpp +++ b/src/bigbang/blockchain.cpp @@ -660,9 +660,9 @@ bool CBlockChain::ListSuperNode(std::vector& nodes) return true; } -bool CBlockChain::FetchSuperNode(std::vector& nodes) +bool CBlockChain::FetchSuperNode(std::vector& nodes, const uint8 mask) { - if (!cntrBlock.FetchSuperNode(nodes)) + if (!cntrBlock.FetchSuperNode(nodes, mask)) { return false; } diff --git a/src/bigbang/blockchain.h b/src/bigbang/blockchain.h index 0995d760..ae0a5137 100644 --- a/src/bigbang/blockchain.h +++ b/src/bigbang/blockchain.h @@ -46,7 +46,7 @@ class CBlockChain : public IBlockChain Errno AddNewOrigin(const CBlock& block, CBlockChainUpdate& update) override; Errno AddNewSuperNode(const storage::CSuperNode& node) override; bool ListSuperNode(std::vector& nodes) override; - bool FetchSuperNode(std::vector& nodes) override; + bool FetchSuperNode(std::vector& nodes, const uint8 mask) override; bool GetProofOfWorkTarget(const uint256& hashPrev, int nAlgo, int& nBits, int64& nReward) override; bool GetBlockMintReward(const uint256& hashPrev, int64& nReward) override; bool GetBlockLocator(const uint256& hashFork, CBlockLocator& locator, uint256& hashDepth, int nIncStep) override; diff --git a/src/bigbang/dispatcher.cpp b/src/bigbang/dispatcher.cpp index 36e7562e..c1618146 100644 --- a/src/bigbang/dispatcher.cpp +++ b/src/bigbang/dispatcher.cpp @@ -110,7 +110,7 @@ bool CDispatcher::HandleInitialize() strCmd = dynamic_cast(Config())->strBlocknotify; nNodeCat = dynamic_cast(Config())->nCatOfNode; - if (2 == nNodeCat) + if (NODE_CAT_DPOSNODE == nNodeCat) { if (!GetObject("mqcluster", pMQCluster)) { diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 4756e31d..b7ed1575 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -109,93 +109,107 @@ bool CMQCluster::HandleInvoke() return true; } + uint8 mask = 0; + if (NODE_CATEGORY::FORKNODE == catNode) + { + mask = 2; + } + if (NODE_CATEGORY::DPOSNODE == catNode) + { + mask = 4; + } std::vector nodes; - if (!pBlockChain->FetchSuperNode(nodes)) + if (!pBlockChain->FetchSuperNode(nodes, mask)) { - Log("CMQCluster::HandleInvoke(): list super node failed"); + Error("CMQCluster::HandleInvoke(): fetch super node failed"); return false; } - for (const auto& node : nodes) + if (nodes.size() > 1) { - mapSuperNode.insert(make_pair(node.superNodeID, node.vecOwnedForks)); - if (1 == node.vecOwnedForks.size() - && node.vecOwnedForks[0] == pCoreProtocol->GetGenesisBlockHash()) - { - Log("dpos node of MQ: [%s] [%d]", node.superNodeID.c_str(), node.ipAddr); - } - else if (0 != node.ipAddr) - { - Log("fork node of MQ: [%s] [%d]", node.superNodeID.c_str(), node.ipAddr); - } - for (const auto& fork : node.vecOwnedForks) - { - Log("CMQCluster::HandleInvoke(): list fork/dpos node [%s] with fork [%s]", - node.superNodeID.c_str(), fork.ToString().c_str()); - } + Error("CMQCluster::HandleInvoke(): there are nodes enrollment info greater than one"); + return false; } - - if (NODE_CATEGORY::FORKNODE == catNode) + if (nodes.empty()) + { + Log("CMQCluster::HandleInvoke(): this super node has not enrolled yet"); + } + else { - lastHeightResp = pBlockChain->GetBlockCount(pCoreProtocol->GetGenesisBlockHash()) - 1; - - if (mapSuperNode.size() == 0) - { - Log("CMQCluster::HandleInvoke(): this fork node has not enrolled " - "itself to dpos node yet[%d]", - mapSuperNode.size()); - } - - if (mapSuperNode.size() > 1) { - Error("CMQCluster::HandleInvoke(): fork node should only have one " - "single enrollment but [%d]", - mapSuperNode.size()); - return false; + boost::unique_lock lock(mtxStatus); + clientID = nodes[0].superNodeID; + ipAddr = nodes[0].ipAddr; + for (auto const& fork : nodes[0].vecOwnedForks) + { + setBizFork.emplace(fork); + Log("CMQCluster::HandleInvoke(): super node enrolled fork:[%s]", fork.ToString().c_str()); + } } + Log("CMQCluster::HandleInvoke(): load super node status info " + "successfully[%s]", + nodes[0].ToString().c_str()); - if (1 == mapSuperNode.size()) + if (NODE_CATEGORY::FORKNODE == catNode) { - pForkManager->SetForkFilter(mapSuperNode.begin()->second); + lastHeightResp = pBlockChain->GetBlockCount(pCoreProtocol->GetGenesisBlockHash()) - 1; + pForkManager->SetForkFilter(nodes[0].vecOwnedForks); - clientID = mapSuperNode.begin()->first; topicReqBlk = "Cluster01/" + clientID + "/SyncBlockReq"; topicRespBlk = "Cluster01/" + clientID + "/SyncBlockResp"; topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; Log("CMQCluster::HandleInvoke(): fork node clientid [%s] with topics:" "\t[%s]\n\t[%s]", - clientID.c_str(), - topicRespBlk.c_str(), topicRbBlk.c_str()); - for (const auto& fork : mapSuperNode.begin()->second) - { - Log("CMQCluster::HandleInvoke(): fork [%s] intended to be produced " - "by this node [%s]:", - fork.ToString().c_str(), clientID.c_str()); - } + clientID.c_str(), topicRespBlk.c_str(), topicRbBlk.c_str()); -/* if (!PostBlockRequest(-1)) + if (!PostBlockRequest(-1)) { Error("CMQCluster::HandleInvoke(): failed to post requesting block"); return false; - }*/ + } + } + else if (NODE_CATEGORY::DPOSNODE == catNode) + { + lastHeightResp = -1; + topicReqBlk = "Cluster01/+/SyncBlockReq"; + topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; + Log("CMQCluster::HandleInvoke(): dpos node clientid [%s] with topic [%s]", + clientID.c_str(), topicReqBlk.c_str()); } } - else if (NODE_CATEGORY::DPOSNODE == catNode) + + if (NODE_CATEGORY::DPOSNODE == catNode) { - lastHeightResp = -1; - for (const auto& node : mapSuperNode) + nodes.clear(); + if (!pBlockChain->FetchSuperNode(nodes, 1 << 1)) { - if (1 == node.second.size() - && node.second[0] == pCoreProtocol->GetGenesisBlockHash()) - { //dpos node - clientID = node.first; - topicReqBlk = "Cluster01/+/SyncBlockReq"; - topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; - Log("CMQCluster::HandleInvoke(): dpos node clientid [%s] with topic [%s]", - clientID.c_str(), topicReqBlk.c_str()); - } + Error("CMQCluster::HandleInvoke(): list all mq fork nodes failed"); + return false; + } + for (auto const& node : nodes) + { + mapActiveMQForkNode.emplace(make_pair(node.superNodeID, + storage::CSuperNode(node.superNodeID, node.ipAddr, + node.vecOwnedForks, + static_cast(NODE_CATEGORY::FORKNODE)))); + Log("CMQCluster::HandleInvoke(): load fork node [%s]", node.ToString().c_str()); } } + nodes.clear(); + if (!pBlockChain->FetchSuperNode(nodes, 1 << 0)) + { + Error("CMQCluster::HandleInvoke(): list all other outer nodes failed"); + return false; + } + for (auto const& node : nodes) + { + mapOuterNode.emplace(make_pair(node.ipAddr, + storage::CSuperNode(node.superNodeID, node.ipAddr, + node.vecOwnedForks, + static_cast(NODE_CATEGORY::BBCNODE)))); + Log("CMQCluster::HandleInvoke(): load outer node [%s]", node.ToString().c_str()); + } + if (!ThreadStart(thrMqttClient)) { return false; @@ -268,14 +282,12 @@ bool CMQCluster::HandleEvent(CEventMQEnrollUpdate& eventMqUpdateEnroll) vector forks = eventMqUpdateEnroll.data.vecForksOwned; if (NODE_CATEGORY::FORKNODE == catNode) { - clientID = id; - ipAddr = eventMqUpdateEnroll.data.ipAddr; topicReqBlk = "Cluster01/" + clientID + "/SyncBlockReq"; topicRespBlk = "Cluster01/" + clientID + "/SyncBlockResp"; topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; Log("CMQCluster::HandleEvent(): fork node clientid [%s] ip [%d] with topics:" "\n[%s]\n[%s]", - clientID.c_str(), ipAddr, + id.c_str(), eventMqUpdateEnroll.data.ipAddr, topicRespBlk.c_str(), topicRbBlk.c_str()); for (const auto& fork : forks) { @@ -286,8 +298,12 @@ bool CMQCluster::HandleEvent(CEventMQEnrollUpdate& eventMqUpdateEnroll) { boost::unique_lock lock(mtxStatus); - mapSuperNode.clear(); - mapSuperNode.insert(make_pair(id, forks)); + clientID = id; + ipAddr = eventMqUpdateEnroll.data.ipAddr; + for (auto const& fork : forks) + { + setBizFork.emplace(fork); + } } condStatus.notify_all(); @@ -301,25 +317,37 @@ bool CMQCluster::HandleEvent(CEventMQEnrollUpdate& eventMqUpdateEnroll) { if (1 == forks.size() && 0 == eventMqUpdateEnroll.data.ipAddr && forks[0] == pCoreProtocol->GetGenesisBlockHash()) - { //dpos node - clientID = id; + { //dpos node enrolled by self topicReqBlk = "Cluster01/+/SyncBlockReq"; topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; - Log("CMQCluster::HandleEvent(): dpos node clientid [%s] with topic [%s]", - clientID.c_str(), topicReqBlk.c_str()); + Log("CMQCluster::HandleEvent(): dpos node clientid [%s] with topic [%s][%s]", + id.c_str(), topicReqBlk.c_str(), topicRbBlk.c_str()); { boost::unique_lock lock(mtxStatus); - mapSuperNode.insert(make_pair(id, forks)); + clientID = id; + ipAddr = 0; + setBizFork.emplace(forks[0]); } condStatus.notify_all(); + return true; } - else - { //fork nodes either enrolled or p2p - //mapActiveSuperNode[ip] = storage::CSuperNode(); - Log("CMQCluster::HandleEvent(): dpos node register clientid [%s] with topic [%s]", - clientID.c_str(), topicReqBlk.c_str()); + + if (storage::CLIENT_ID_OUT_OF_MQ_CLUSTER != id) + { //fork node enrolled by dpos node + mapActiveMQForkNode[id] = storage::CSuperNode(id, eventMqUpdateEnroll.data.ipAddr, forks); + Log("CMQCluster::HandleEvent(): dpos node register clientid [%s]", + clientID.c_str()); + return true; } + + //biz fork nodes by p2p + mapOuterNode.insert(make_pair(eventMqUpdateEnroll.data.ipAddr, + storage::CSuperNode(id, eventMqUpdateEnroll.data.ipAddr, forks))); + Log("CMQCluster::HandleEvent(): dpos node register biz fork nodes by p2p [%s - %s]", + clientID.c_str(), storage::CSuperNode::Int2Ip(eventMqUpdateEnroll.data.ipAddr).c_str()); + + return true; } return true; @@ -352,18 +380,12 @@ bool CMQCluster::PostBlockRequest(int syncHeight) { Log("CMQCluster::PostBlockRequest(): posting request for block #%d", syncHeight); - if (mapSuperNode.empty()) + if ("" == clientID) { Log("CMQCluster::PostBlockRequest(): enrollment is empty for this fork node"); return true; } - if (mapSuperNode.size() > 1) - { - Error("CMQCluster::PostBlockRequest(): enrollment is incorrect for this fork node"); - return false; - } - uint256 hash; int height; if (-1 == syncHeight) @@ -398,9 +420,11 @@ bool CMQCluster::PostBlockRequest(int syncHeight) req.ipAddr = ipAddr; //16777343 - 127.0.0.1; 1111638320 - "0ABB" req.forkNodeIdLen = clientID.size(); req.forkNodeId = clientID; - auto enroll = mapSuperNode.begin(); - req.forkNum = (*enroll).second.size(); - req.forkList = (*enroll).second; + req.forkNum = setBizFork.size(); + for (auto const& fork : setBizFork) + { + req.forkList.emplace_back(fork); + } req.lastHeight = height; req.lastHash = hash; req.tsRequest = GetTime(); @@ -633,7 +657,8 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) } } Log("CMQCluster::OnReceiveMessage(): fork node blkhsh[%s] vs. dpos node blkhsh[%s] " - "at height of [%d]", rb.hashList[i].ToString().c_str(), hash.ToString().c_str(), + "at height of [%d]", + rb.hashList[i].ToString().c_str(), hash.ToString().c_str(), rb.rbHeight + i + 1); if (hash == rb.hashList[i]) { @@ -690,22 +715,22 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) } //check if requesting fork node has been enrolled - auto node = mapSuperNode.find(req.forkNodeId); - if (node == mapSuperNode.end()) + auto node = mapActiveMQForkNode.find(req.forkNodeId); + if (node == mapActiveMQForkNode.end()) { Error("CMQCluster::OnReceiveMessage(): requesting fork node has not enrolled yet"); return; } //check if requesting fork node matches the corresponding one enrolled - if (node->second.size() != req.forkNum) + if (node->second.vecOwnedForks.size() != req.forkNum) { Error("CMQCluster::OnReceiveMessage(): requesting fork node number does not match"); return; } for (const auto& fork : req.forkList) { - auto pos = find(node->second.begin(), node->second.end(), fork); - if (pos == node->second.end()) + auto pos = find(node->second.vecOwnedForks.begin(), node->second.vecOwnedForks.end(), fork); + if (pos == node->second.vecOwnedForks.end()) { Error("CMQCluster::OnReceiveMessage(): requesting fork node detailed forks does not match"); return; @@ -713,9 +738,9 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) } //add this requesting fork node to active list - if (!mapActiveSuperNode.count(req.ipAddr)) + if (!mapActiveMQForkNode.count(req.forkNodeId)) { - mapActiveSuperNode[req.ipAddr] = storage::CSuperNode(req.forkNodeId, req.ipAddr, req.forkList); + mapActiveMQForkNode[req.forkNodeId] = storage::CSuperNode(req.forkNodeId, req.ipAddr, req.forkList); } //processing request from fork node @@ -746,13 +771,15 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) if (!pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), req.lastHeight, hash)) { Error("CMQCluster::OnReceiveMessage(): failed to get checking height and hash match " - "at height of #%d", req.lastHeight); + "at height of #%d", + req.lastHeight); return; } if (hash != req.lastHash) { Error("CMQCluster::OnReceiveMessage(): height and hash do not match hash[%s] vs. req.lastHash[%s] " - "at height of [%d]", hash.ToString().c_str(), req.lastHash.ToString().c_str(), req.lastHeight); + "at height of [%d]", + hash.ToString().c_str(), req.lastHash.ToString().c_str(), req.lastHeight); return; } if (!pBlockChain->GetBlockHash(pCoreProtocol->GetGenesisBlockHash(), req.lastHeight + 1, hash)) diff --git a/src/bigbang/mqcluster.h b/src/bigbang/mqcluster.h index 3f2184e8..2877eb05 100644 --- a/src/bigbang/mqcluster.h +++ b/src/bigbang/mqcluster.h @@ -170,11 +170,12 @@ class CMQCluster : public IMQCluster boost::mutex mtxStatus; boost::condition_variable condStatus; - std::map> mapSuperNode; string clientID; uint32 ipAddr; + std::set setBizFork; - std::map mapActiveSuperNode; + std::map mapActiveMQForkNode; //nil if bbc or fork nodes + std::map mapOuterNode; //nil if bbc nodes boost::mutex mtxSend; boost::condition_variable condSend; @@ -184,7 +185,6 @@ class CMQCluster : public IMQCluster std::deque> deqRecvBuff; //topic vs. payload std::atomic lastHeightResp; -// uint32 nReqBlkTimerID; std::atomic nReqBlkTimerID; boost::mutex mtxRoll; diff --git a/src/bigbang/struct.h b/src/bigbang/struct.h index 4e042bc6..58ab55b3 100644 --- a/src/bigbang/struct.h +++ b/src/bigbang/struct.h @@ -256,6 +256,7 @@ class CBlockMakerUpdate uint16 nMintType; }; +/* Super node */ class CMqRollbackUpdate { public: diff --git a/src/storage/blockbase.cpp b/src/storage/blockbase.cpp index e0984a95..4e9c0e75 100644 --- a/src/storage/blockbase.cpp +++ b/src/storage/blockbase.cpp @@ -528,9 +528,9 @@ bool CBlockBase::ListSuperNode(vector& nodes) return true; } -bool CBlockBase::FetchSuperNode(vector& nodes) +bool CBlockBase::FetchSuperNode(vector& nodes, const uint8 mask) { - if (!dbBlock.FetchSuperNode(nodes)) + if (!dbBlock.FetchSuperNode(nodes, mask)) { Error("CBlockBase::FetchSuperNode", "Failed to fetch supernode"); return false; diff --git a/src/storage/blockbase.h b/src/storage/blockbase.h index 7533f1fb..12981d9d 100644 --- a/src/storage/blockbase.h +++ b/src/storage/blockbase.h @@ -235,7 +235,7 @@ class CBlockBase bool AddNewForkContext(const CForkContext& ctxt); bool AddNewSuperNode(const CSuperNode& superNode); bool ListSuperNode(std::vector& nodes); - bool FetchSuperNode(std::vector& nodes); + bool FetchSuperNode(std::vector& nodes, const uint8 mask); bool Retrieve(const uint256& hash, CBlock& block); bool Retrieve(const CBlockIndex* pIndex, CBlock& block); bool Retrieve(const uint256& hash, CBlockEx& block); diff --git a/src/storage/blockdb.cpp b/src/storage/blockdb.cpp index 3050819c..f44ede5a 100644 --- a/src/storage/blockdb.cpp +++ b/src/storage/blockdb.cpp @@ -261,9 +261,9 @@ bool CBlockDB::ListSuperNode(std::vector& nodes) return dbSuperNode.ListSuperNode(nodes); } -bool CBlockDB::FetchSuperNode(std::vector& nodes) +bool CBlockDB::FetchSuperNode(std::vector& nodes, const uint8 mask) { - return dbSuperNode.FetchSuperNode(nodes); + return dbSuperNode.FetchSuperNode(nodes, mask); } } // namespace storage diff --git a/src/storage/blockdb.h b/src/storage/blockdb.h index ada154d0..7b14926a 100644 --- a/src/storage/blockdb.h +++ b/src/storage/blockdb.h @@ -50,7 +50,7 @@ class CBlockDB std::map& mapEnrollTxPos); bool AddNewSuperNode(const CSuperNode& superNode); bool ListSuperNode(std::vector& nodes); - bool FetchSuperNode(std::vector& nodes); + bool FetchSuperNode(std::vector& nodes, const uint8 mask); protected: bool LoadFork(); diff --git a/src/storage/mqdb.cpp b/src/storage/mqdb.cpp index e9877cb9..ffe6734e 100644 --- a/src/storage/mqdb.cpp +++ b/src/storage/mqdb.cpp @@ -106,13 +106,23 @@ void CSuperNodeDB::Clear() } bool CSuperNodeDB::FetchSuperNodeWalker(xengine::CBufStream& ssKey, xengine::CBufStream& ssValue, - map, vector>& mapCli) + map, vector>& mapCli, const uint8& mask) { string strCliID; uint32 nIP; ssKey >> strCliID >> nIP; - if (strCliID == CLIENT_ID_OUT_OF_MQ_CLUSTER) + if (((mask & 0x1 << 0) == 0) && strCliID == CLIENT_ID_OUT_OF_MQ_CLUSTER) + { + return true; + } + + if (((mask & 0x1 << 1) == 0) && strCliID != CLIENT_ID_OUT_OF_MQ_CLUSTER && 0 != nIP) + { + return true; + } + + if (((mask & 0x1 << 2) == 0) && strCliID != CLIENT_ID_OUT_OF_MQ_CLUSTER && 0 == nIP) { return true; } @@ -123,11 +133,11 @@ bool CSuperNodeDB::FetchSuperNodeWalker(xengine::CBufStream& ssKey, xengine::CBu return true; } -bool CSuperNodeDB::FetchSuperNode(std::vector& vCli) +bool CSuperNodeDB::FetchSuperNode(std::vector& vCli, const uint8& mask) { map, vector> mapCli; - if (!WalkThrough(boost::bind(&CSuperNodeDB::FetchSuperNodeWalker, this, _1, _2, boost::ref(mapCli)))) + if (!WalkThrough(boost::bind(&CSuperNodeDB::FetchSuperNodeWalker, this, _1, _2, boost::ref(mapCli), mask))) { return false; } @@ -147,7 +157,7 @@ bool CSuperNodeDB::FetchSuperNode(std::vector& vCli) bool CSuperNodeDB::ClearSuperNode(const CSuperNode& cli) { vector vSuperNode; - if (!FetchSuperNode(vSuperNode)) + if (!FetchSuperNode(vSuperNode, 6)) { return false; } diff --git a/src/storage/mqdb.h b/src/storage/mqdb.h index 34b55fe4..471d9894 100644 --- a/src/storage/mqdb.h +++ b/src/storage/mqdb.h @@ -120,13 +120,13 @@ class CSuperNodeDB : public xengine::CKVDB bool ListSuperNode(std::vector& vCli); //return all nodes void Clear(); bool ClearSuperNode(const CSuperNode& cli); - bool FetchSuperNode(std::vector& vCli); //only return super nodes including dpos and fork nodes + bool FetchSuperNode(std::vector& vCli, const uint8& mask); //only return super nodes on request protected: bool LoadSuperNodeWalker(xengine::CBufStream& ssKey, xengine::CBufStream& ssValue, std::map, std::vector>& mapCli); bool FetchSuperNodeWalker(xengine::CBufStream& ssKey, xengine::CBufStream& ssValue, - std::map, std::vector>& mapCli); + std::map, std::vector>& mapCli, const uint8& mask); }; } // namespace storage From f23b773fd1a5aec75013613aacec4d3aea377e57 Mon Sep 17 00:00:00 2001 From: OuYun Date: Sun, 12 Apr 2020 21:54:56 +0800 Subject: [PATCH 067/181] update log info to debug for convenience --- src/storage/blockbase.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/storage/blockbase.cpp b/src/storage/blockbase.cpp index 4e9c0e75..d3c5f28d 100644 --- a/src/storage/blockbase.cpp +++ b/src/storage/blockbase.cpp @@ -521,8 +521,8 @@ bool CBlockBase::ListSuperNode(vector& nodes) { for (const auto& fork : node.vecOwnedForks) { - Log("CBlockBase::ListSuperNode", "supernode client ID [%s] IP [%d]: fork [%s]", - node.superNodeID.c_str(), node.ipAddr, fork.ToString().c_str()); + Log("CBlockBase::ListSuperNode", "supernode client ID [%s] IP [%s]: fork [%s]", + node.superNodeID.c_str(), CSuperNode::Int2Ip(node.ipAddr).c_str(), fork.ToString().c_str()); } } return true; @@ -540,8 +540,8 @@ bool CBlockBase::FetchSuperNode(vector& nodes, const uint8 { for (const auto& fork : node.vecOwnedForks) { - Log("CBlockBase::FetchSuperNode", "supernode client ID [%s] IP [%d]: fork [%s]", - node.superNodeID.c_str(), node.ipAddr, fork.ToString().c_str()); + Log("CBlockBase::FetchSuperNode", "supernode client ID [%s] IP [%s]: fork [%s]", + node.superNodeID.c_str(), CSuperNode::Int2Ip(node.ipAddr).c_str(), fork.ToString().c_str()); } } return true; From 0160471f10744cea6463ae28f496627effee3f73 Mon Sep 17 00:00:00 2001 From: OuYun Date: Mon, 13 Apr 2020 09:58:28 +0800 Subject: [PATCH 068/181] connect mqcluster to netchannel --- src/bigbang/netchn.cpp | 20 ++++++++++++++++++-- src/bigbang/netchn.h | 2 ++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/bigbang/netchn.cpp b/src/bigbang/netchn.cpp index c235ede1..44e5651f 100644 --- a/src/bigbang/netchn.cpp +++ b/src/bigbang/netchn.cpp @@ -140,6 +140,7 @@ CNetChannel::CNetChannel() pTxPool = nullptr; pService = nullptr; pDispatcher = nullptr; + pMQCluster = nullptr; fStartIdlePushTxTimer = false; nNodeCat = NODE_CAT_BBCNODE; } @@ -186,6 +187,12 @@ bool CNetChannel::HandleInitialize() return false; } + if (!GetObject("mqcluster", pMQCluster)) + { + Error("Failed to request mqcluster"); + return false; + } + return true; } @@ -197,6 +204,7 @@ void CNetChannel::HandleDeinitialize() pTxPool = nullptr; pService = nullptr; pDispatcher = nullptr; + pMQCluster = nullptr; } bool CNetChannel::HandleInvoke() @@ -576,10 +584,18 @@ bool CNetChannel::HandleEvent(network::CEventPeerBizForks& eventBizForks) } } - //if dpos node self, dispatch them to fork nodes by MQ + //if dpos node self, dispatch them to fork nodes by MQ later if (NODE_CAT_DPOSNODE == nNodeCat) { - ; + CEventMQBizForkUpdate* pEvent = new CEventMQBizForkUpdate(nNonce); + if (pEvent != nullptr) + { + pEvent->data = setForkIp; + pMQCluster->PostEvent(pEvent); + StdLog("NetChannel", "CEventPeerBizForks: post biz forks by peer[%s] " + "with total [%d] ips, [%d] forks", + GetPeerAddressInfo(nNonce).c_str(), mapIpForks.size(), mapForkIp.size()); + } } return true; diff --git a/src/bigbang/netchn.h b/src/bigbang/netchn.h index efd60327..cdd4f211 100644 --- a/src/bigbang/netchn.h +++ b/src/bigbang/netchn.h @@ -6,6 +6,7 @@ #define BIGBANG_NETCHN_H #include "base.h" +#include "mqcluster.h" #include "peernet.h" #include "schedule.h" @@ -309,6 +310,7 @@ class CNetChannel : public network::INetChannel ITxPool* pTxPool; IDispatcher* pDispatcher; IService* pService; + IMQCluster* pMQCluster; mutable boost::recursive_mutex mtxSched; std::map mapSched; From cdbaffd96ccf8f4326b9047da523d828339dbed4 Mon Sep 17 00:00:00 2001 From: OuYun Date: Mon, 13 Apr 2020 10:54:05 +0800 Subject: [PATCH 069/181] adjust connector netchn to mq and add new nodes to p2p connection pool in the meantime --- src/bigbang/netchn.cpp | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/bigbang/netchn.cpp b/src/bigbang/netchn.cpp index 44e5651f..9a90cd4d 100644 --- a/src/bigbang/netchn.cpp +++ b/src/bigbang/netchn.cpp @@ -585,7 +585,7 @@ bool CNetChannel::HandleEvent(network::CEventPeerBizForks& eventBizForks) } //if dpos node self, dispatch them to fork nodes by MQ later - if (NODE_CAT_DPOSNODE == nNodeCat) + if (NODE_CAT_DPOSNODE == nNodeCat || NODE_CAT_FORKNODE == nNodeCat) { CEventMQBizForkUpdate* pEvent = new CEventMQBizForkUpdate(nNonce); if (pEvent != nullptr) @@ -598,6 +598,25 @@ bool CNetChannel::HandleEvent(network::CEventPeerBizForks& eventBizForks) } } + for (auto const& node : nodes) + { + string ip = storage::CSuperNode::Int2Ip(node.ipAddr); + CNetHost host(ip, DEFAULT_P2PPORT); + CEventPeerNetAddNode eventAddNode(0); + eventAddNode.data = host; + if (!pPeerNet->DispatchEvent(&eventAddNode)) + { + StdError("NetChannel", "CEventPeerBizForks: Add peer node[%s] failed", + storage::CSuperNode::Int2Ip(node.ipAddr).c_str()); + return false; + } + else + { + StdLog("NetChannel", "CEventPeerBizForks: Add peer node[%s] succeeded", + storage::CSuperNode::Int2Ip(node.ipAddr).c_str()); + } + } + return true; } From c6a85f6cbfd7aa642f80e057624380dbcd7c01ec Mon Sep 17 00:00:00 2001 From: OuYun Date: Mon, 13 Apr 2020 14:27:32 +0800 Subject: [PATCH 070/181] bugfix of incorrect node type --- src/bigbang/rpcmod.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bigbang/rpcmod.cpp b/src/bigbang/rpcmod.cpp index 36059ef7..b0d3918f 100644 --- a/src/bigbang/rpcmod.cpp +++ b/src/bigbang/rpcmod.cpp @@ -2461,11 +2461,11 @@ CRPCResultPtr CRPCMod::RPCEnrollSuperNode(rpc::CRPCParamPtr param) } } - if (NODE_CAT_DPOSNODE == nNodeCat) + if (NODE_CAT_FORKNODE == nNodeCat) { if (forks.end() != find(forks.begin(), forks.end(), hashGenesis)) { - throw CRPCException(RPC_INVALID_PARAMETER, "The main fork fork node watches should not be the main fork:[ " + hashGenesis.ToString() + " ]"); + throw CRPCException(RPC_INVALID_PARAMETER, "The fork fork node enrolls should not be the main fork:[ " + hashGenesis.ToString() + " ]"); } } From ff432d7676f7090677c5b17e240a1209d0677965 Mon Sep 17 00:00:00 2001 From: OuYun Date: Mon, 13 Apr 2020 16:48:36 +0800 Subject: [PATCH 071/181] set constraint of only bbc nodes put new peers to p2p connection pool according to biz forks scheduled --- src/bigbang/netchn.cpp | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/bigbang/netchn.cpp b/src/bigbang/netchn.cpp index 9a90cd4d..3c9520bb 100644 --- a/src/bigbang/netchn.cpp +++ b/src/bigbang/netchn.cpp @@ -596,25 +596,36 @@ bool CNetChannel::HandleEvent(network::CEventPeerBizForks& eventBizForks) "with total [%d] ips, [%d] forks", GetPeerAddressInfo(nNonce).c_str(), mapIpForks.size(), mapForkIp.size()); } + return true; } - for (auto const& node : nodes) + vector vIP; + for (auto const& it : mapForkIp) + { + bool fOwned = false; + { + boost::recursive_mutex::scoped_lock scoped_lock(mtxSched); + fOwned = mapSched.count(it.first); + } + if (fOwned) + { + vIP.insert(vIP.end(), it.second.begin(), it.second.end()); + StdLog("NetChannel", "CEventPeerBizForks: Add [%d] peer node(s) with biz fork[%s]", + it.second.size(), it.first.ToString().c_str()); + } + } + for (auto const& nIP : vIP) { - string ip = storage::CSuperNode::Int2Ip(node.ipAddr); - CNetHost host(ip, DEFAULT_P2PPORT); + string strIP = storage::CSuperNode::Int2Ip(nIP); + CNetHost host(strIP, DEFAULT_P2PPORT); CEventPeerNetAddNode eventAddNode(0); eventAddNode.data = host; if (!pPeerNet->DispatchEvent(&eventAddNode)) { - StdError("NetChannel", "CEventPeerBizForks: Add peer node[%s] failed", - storage::CSuperNode::Int2Ip(node.ipAddr).c_str()); + StdError("NetChannel", "CEventPeerBizForks: Add peer node[%s] failed", strIP.c_str()); return false; } - else - { - StdLog("NetChannel", "CEventPeerBizForks: Add peer node[%s] succeeded", - storage::CSuperNode::Int2Ip(node.ipAddr).c_str()); - } + StdLog("NetChannel", "CEventPeerBizForks: Add peer node[%s] succeeded", strIP.c_str()); } return true; From 62b2f3aec9765411173358ec61654e38200dd5e3 Mon Sep 17 00:00:00 2001 From: OuYun Date: Tue, 14 Apr 2020 12:12:33 +0800 Subject: [PATCH 072/181] bugfix of incorrect operation when enrollment has not occured yet at invoke routine --- src/bigbang/mqcluster.cpp | 68 ++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 36 deletions(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index b7ed1575..d9b101eb 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -132,53 +132,49 @@ bool CMQCluster::HandleInvoke() if (nodes.empty()) { Log("CMQCluster::HandleInvoke(): this super node has not enrolled yet"); + return true; } - else + { + boost::unique_lock lock(mtxStatus); + clientID = nodes[0].superNodeID; + ipAddr = nodes[0].ipAddr; + for (auto const& fork : nodes[0].vecOwnedForks) { - boost::unique_lock lock(mtxStatus); - clientID = nodes[0].superNodeID; - ipAddr = nodes[0].ipAddr; - for (auto const& fork : nodes[0].vecOwnedForks) - { - setBizFork.emplace(fork); - Log("CMQCluster::HandleInvoke(): super node enrolled fork:[%s]", fork.ToString().c_str()); - } + setBizFork.emplace(fork); + Log("CMQCluster::HandleInvoke(): super node enrolled fork:[%s]", fork.ToString().c_str()); } - Log("CMQCluster::HandleInvoke(): load super node status info " - "successfully[%s]", - nodes[0].ToString().c_str()); + } + Log("CMQCluster::HandleInvoke(): load super node status info " + "successfully[%s]", + nodes[0].ToString().c_str()); - if (NODE_CATEGORY::FORKNODE == catNode) - { - lastHeightResp = pBlockChain->GetBlockCount(pCoreProtocol->GetGenesisBlockHash()) - 1; - pForkManager->SetForkFilter(nodes[0].vecOwnedForks); + if (NODE_CATEGORY::FORKNODE == catNode) + { + lastHeightResp = pBlockChain->GetBlockCount(pCoreProtocol->GetGenesisBlockHash()) - 1; + pForkManager->SetForkFilter(nodes[0].vecOwnedForks); - topicReqBlk = "Cluster01/" + clientID + "/SyncBlockReq"; - topicRespBlk = "Cluster01/" + clientID + "/SyncBlockResp"; - topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; - Log("CMQCluster::HandleInvoke(): fork node clientid [%s] with topics:" - "\t[%s]\n\t[%s]", - clientID.c_str(), topicRespBlk.c_str(), topicRbBlk.c_str()); + topicReqBlk = "Cluster01/" + clientID + "/SyncBlockReq"; + topicRespBlk = "Cluster01/" + clientID + "/SyncBlockResp"; + topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; // todo: should configure DPOSNODE + Log("CMQCluster::HandleInvoke(): fork node clientid [%s] with topics:" + "\t[%s]\n\t[%s]", + clientID.c_str(), topicRespBlk.c_str(), topicRbBlk.c_str()); - if (!PostBlockRequest(-1)) - { - Error("CMQCluster::HandleInvoke(): failed to post requesting block"); - return false; - } - } - else if (NODE_CATEGORY::DPOSNODE == catNode) + if (!PostBlockRequest(-1)) { - lastHeightResp = -1; - topicReqBlk = "Cluster01/+/SyncBlockReq"; - topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; - Log("CMQCluster::HandleInvoke(): dpos node clientid [%s] with topic [%s]", - clientID.c_str(), topicReqBlk.c_str()); + Error("CMQCluster::HandleInvoke(): failed to post requesting block"); + return false; } } - - if (NODE_CATEGORY::DPOSNODE == catNode) + else if (NODE_CATEGORY::DPOSNODE == catNode) { + lastHeightResp = -1; + topicReqBlk = "Cluster01/+/SyncBlockReq"; + topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; // todo: DPOSNODE should come from storage + Log("CMQCluster::HandleInvoke(): dpos node clientid [%s] with topic [%s]", + clientID.c_str(), topicReqBlk.c_str()); + nodes.clear(); if (!pBlockChain->FetchSuperNode(nodes, 1 << 1)) { From 5d45b4f372bd709f9dc653c3bed97b69127d7bf6 Mon Sep 17 00:00:00 2001 From: OuYun Date: Tue, 14 Apr 2020 12:38:56 +0800 Subject: [PATCH 073/181] add constraint for dpos nodes --- src/bigbang/mqcluster.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index d9b101eb..f5650eb9 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -169,6 +169,15 @@ bool CMQCluster::HandleInvoke() } else if (NODE_CATEGORY::DPOSNODE == catNode) { + { + boost::unique_lock lock(mtxStatus); + if (setBizFork.size() != 1 || *(setBizFork.begin()) != pCoreProtocol->GetGenesisBlockHash()) + { + Error("CMQCluster::HandleInvoke(): dpos node enrollment info " + "invalid [%d] for self", setBizFork.size()); + return false; + } + } lastHeightResp = -1; topicReqBlk = "Cluster01/+/SyncBlockReq"; topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; // todo: DPOSNODE should come from storage From 668a6ef4d78d6382e7101542418731ffb809f1f6 Mon Sep 17 00:00:00 2001 From: OuYun Date: Tue, 14 Apr 2020 15:32:42 +0800 Subject: [PATCH 074/181] update --- src/bigbang/base.h | 3 ++- src/bigbang/forkmanager.cpp | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/bigbang/base.h b/src/bigbang/base.h index 5f42014b..5df9f003 100644 --- a/src/bigbang/base.h +++ b/src/bigbang/base.h @@ -148,7 +148,8 @@ class IForkManager : public xengine::IBase virtual void ForkUpdate(const CBlockChainUpdate& update, std::vector& vActive, std::vector& vDeactive) = 0; virtual void GetForkList(std::vector& vFork) const = 0; virtual bool GetSubline(const uint256& hashFork, std::vector>& vSubline) const = 0; - virtual bool SetForkFilter(const std::vector& vFork = std::vector(), const std::vector& vGroup = std::vector()) = 0; const CForkConfig* ForkConfig() + virtual bool SetForkFilter(const std::vector& vFork = std::vector(), const std::vector& vGroup = std::vector()) = 0; + const CForkConfig* ForkConfig() { return dynamic_cast(xengine::IBase::Config()); } diff --git a/src/bigbang/forkmanager.cpp b/src/bigbang/forkmanager.cpp index b51fd162..a7902df4 100644 --- a/src/bigbang/forkmanager.cpp +++ b/src/bigbang/forkmanager.cpp @@ -6,6 +6,7 @@ #include +#include "defs.h" #include "template/fork.h" using namespace std; @@ -54,7 +55,7 @@ void CForkManager::HandleDeinitialize() bool CForkManager::HandleInvoke() { int8 nodeCat = dynamic_cast(Config())->nCatOfNode; - if (2 == nodeCat) + if (NODE_CAT_DPOSNODE == nodeCat) { setForkAllowed.insert(pCoreProtocol->GetGenesisBlockHash()); return true; @@ -270,7 +271,7 @@ bool CForkManager::GetSubline(const uint256& hashFork, vector bool CForkManager::SetForkFilter(const std::vector& vFork, const std::vector& vGroup) { int8 nodeCat = dynamic_cast(Config())->nCatOfNode; - if (2 == nodeCat) + if (NODE_CAT_DPOSNODE == nodeCat) { return true; } @@ -302,7 +303,6 @@ bool CForkManager::SetForkFilter(const std::vector& vFork, const std::v return true; } - bool CForkManager::IsAllowedFork(const uint256& hashFork, const uint256& hashParent) const { if (fAllowAnyFork || setForkAllowed.count(hashFork) || setGroupAllowed.count(hashFork)) From b16ac42f6fa17c76340021b19bbf5c7e0edd391e Mon Sep 17 00:00:00 2001 From: OuYun Date: Tue, 14 Apr 2020 16:07:46 +0800 Subject: [PATCH 075/181] remove dead code --- src/bigbang/mqcluster.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index f5650eb9..1045379d 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -742,12 +742,6 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) } } - //add this requesting fork node to active list - if (!mapActiveMQForkNode.count(req.forkNodeId)) - { - mapActiveMQForkNode[req.forkNodeId] = storage::CSuperNode(req.forkNodeId, req.ipAddr, req.forkList); - } - //processing request from fork node //check height requested From ae62bcdc5acacac0006a47887f2357b1e4ac4a5b Mon Sep 17 00:00:00 2001 From: OuYun Date: Tue, 14 Apr 2020 18:52:16 +0800 Subject: [PATCH 076/181] add a mq protocol of assign biz fork: dpos node -> fork node --- src/bigbang/mqcluster.cpp | 63 +++++++++++++++++++++++++++++++++------ src/bigbang/mqcluster.h | 30 +++++++++++++++---- src/bigbang/netchn.cpp | 2 ++ 3 files changed, 81 insertions(+), 14 deletions(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 1045379d..0e334fc1 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -52,7 +52,7 @@ bool CMQCluster::IsAuthenticated() bool CMQCluster::HandleInitialize() { - if (NODE_CATEGORY::BBCNODE == catNode) + if (NODE_CATEGORY::BBCNODE == catNode) // todo: { Log("CMQCluster::HandleInitialize(): bbc node so bypass"); return true; @@ -103,7 +103,7 @@ void CMQCluster::HandleDeinitialize() bool CMQCluster::HandleInvoke() { - if (NODE_CATEGORY::BBCNODE == catNode) + if (NODE_CATEGORY::BBCNODE == catNode) // todo: { Log("CMQCluster::HandleInvoke(): bbc node so bypass"); return true; @@ -206,7 +206,7 @@ bool CMQCluster::HandleInvoke() Error("CMQCluster::HandleInvoke(): list all other outer nodes failed"); return false; } - for (auto const& node : nodes) + for (auto const& node : nodes) // todo: should send event to add peers to netchannel module if fork node itself { mapOuterNode.emplace(make_pair(node.ipAddr, storage::CSuperNode(node.superNodeID, node.ipAddr, @@ -361,9 +361,39 @@ bool CMQCluster::HandleEvent(CEventMQEnrollUpdate& eventMqUpdateEnroll) bool CMQCluster::HandleEvent(CEventMQBizForkUpdate& eventMqBizFork) { Log("CMQCluster::HandleEvent(): biz forks payload is coming"); - if (NODE_CATEGORY::DPOSNODE == catNode) + + const storage::CForkKnownIpSetByIp& idxIP = eventMqBizFork.data.get<1>(); + for (auto const& i : idxIP) { - ; + auto& node = mapOuterNode[i.nodeIP]; + node.ipAddr = i.nodeIP; + node.vecOwnedForks.push_back(i.forkID); + } + + if (NODE_CATEGORY::DPOSNODE == catNode) + { //spread ip/fork through mq broker + const storage::CForkKnownIpSetById& idxID = eventMqBizFork.data.get<0>(); + for (auto const& cli : mapActiveMQForkNode) + { + CAssignBizFork biz; + auto& it = biz.mapBizForkIP; + for (auto const& fork : cli.second.vecOwnedForks) + { + auto itBegin = idxID.equal_range(fork).first; + auto itEnd = idxID.equal_range(fork).second; + + while (itBegin != itEnd) + { + it[itBegin->forkID].push_back(itBegin->nodeIP); + ++itBegin; + } + } + if (!it.empty()) + { + string topic = "Cluster01/" + cli.second.superNodeID + "/AssignBizFork"; + PostBizForkAssign(topic, biz); + } + } } return true; @@ -442,8 +472,16 @@ bool CMQCluster::PostBlockRequest(int syncHeight) return true; } -bool CMQCluster::AppendSendQueue(const std::string& topic, - CBufferPtr payload) +bool CMQCluster::PostBizForkAssign(const std::string& topic, CAssignBizFork assign) +{ + CBufferPtr spSS(new CBufStream); + *spSS.get() << assign; + + AppendSendQueue(topic, spSS); + return true; +} + +bool CMQCluster::AppendSendQueue(const std::string& topic, CBufferPtr payload) { { boost::unique_lock lock(mtxSend); @@ -479,7 +517,7 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) case NODE_CATEGORY::FORKNODE: { Log("CMQCluster::OnReceiveMessage(): current sync height is [%d]", int(lastHeightResp)); - if (topicRbBlk != topic) + if (topicRbBlk != topic) //todo: to refactor { //respond to request block of main chain //unpack payload CSyncBlockResponse resp; @@ -1009,7 +1047,14 @@ bool CMQCluster::ClientAgent(MQ_CLI_ACTION action) mqtt::message_ptr pubmsg = mqtt::make_message( buf.first, buf.second->GetData(), buf.second->GetSize()); pubmsg->set_qos(QOS1); - pubmsg->set_retained(mqtt::message::DFLT_RETAINED); + if (buf.first.find("AssignBizFork")) + { + pubmsg->set_retained(true); + } + else + { + pubmsg->set_retained(mqtt::message::DFLT_RETAINED); + } delitok = client.publish(pubmsg, nullptr, cb); delitok->wait_for(100); cout << "_._._OK" << endl; diff --git a/src/bigbang/mqcluster.h b/src/bigbang/mqcluster.h index 2877eb05..f4398705 100644 --- a/src/bigbang/mqcluster.h +++ b/src/bigbang/mqcluster.h @@ -88,6 +88,21 @@ class CRollbackBlock } }; +class CAssignBizFork +{ + friend xengine::CStream; + +public: + std::map> mapBizForkIP; + +protected: + template + void Serialize(xengine::CStream& s, O& opt) + { + s.Serialize(mapBizForkIP, opt); + } +}; + class IMQCluster : public xengine::IIOModule, virtual public CMQEventListener { public: @@ -102,18 +117,21 @@ class CMQCluster : public IMQCluster typedef std::shared_ptr CBufferPtr; - enum class NODE_CATEGORY : char { + enum class NODE_CATEGORY : char + { BBCNODE = 0, FORKNODE, DPOSNODE }; - enum class MQ_CLI_ACTION : char { + enum class MQ_CLI_ACTION : char + { CONN = 0, SUB, PUB, DISCONN }; - enum { + enum + { QOS0 = 0, QOS1, QOS2 @@ -153,6 +171,7 @@ class CMQCluster : public IMQCluster private: bool PostBlockRequest(int syncHeight = -1); + bool PostBizForkAssign(const std::string& topic, CAssignBizFork assign); bool AppendSendQueue(const std::string& topic, CBufferPtr payload); void RequestBlockTimerFunc(uint32 nTimer); void OnReceiveMessage(const std::string& topic, CBufStream& payload); @@ -174,8 +193,9 @@ class CMQCluster : public IMQCluster uint32 ipAddr; std::set setBizFork; - std::map mapActiveMQForkNode; //nil if bbc or fork nodes - std::map mapOuterNode; //nil if bbc nodes + std::map mapActiveMQForkNode; //only for dpos node + std::map mapOuterNode; //for dpos/fork node + std::map mapBizForkUpdateTopic; boost::mutex mtxSend; boost::condition_variable condSend; diff --git a/src/bigbang/netchn.cpp b/src/bigbang/netchn.cpp index 3c9520bb..79d3ca76 100644 --- a/src/bigbang/netchn.cpp +++ b/src/bigbang/netchn.cpp @@ -599,6 +599,8 @@ bool CNetChannel::HandleEvent(network::CEventPeerBizForks& eventBizForks) return true; } + // todo: here should send event to add peers to netchannel module if fork nodes + vector vIP; for (auto const& it : mapForkIp) { From 54bb0e841332a64ee20183c10edbf3becda054ab Mon Sep 17 00:00:00 2001 From: OuYun Date: Wed, 15 Apr 2020 14:56:41 +0800 Subject: [PATCH 077/181] reuse existed code --- src/bigbang/mqcluster.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 0e334fc1..e1fd7c11 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -846,11 +846,7 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) CBufferPtr spSS(new CBufStream); *spSS.get() << resp; string topicRsp = "Cluster01/" + req.forkNodeId + "/SyncBlockResp"; - { - boost::unique_lock lock(mtxSend); - deqSendBuff.emplace_back(make_pair(topicRsp, spSS)); - } - condSend.notify_all(); + AppendSendQueue(topicRsp, spSS); break; } From 0c31e5189ef49961aa4017986d674fa5cddf3420 Mon Sep 17 00:00:00 2001 From: OuYun Date: Thu, 16 Apr 2020 11:31:19 +0800 Subject: [PATCH 078/181] add one topic for biz fork and refactor topics --- src/bigbang/mqcluster.cpp | 68 ++++++++++++++++++++++----------------- src/bigbang/mqcluster.h | 28 ++++++++++++---- 2 files changed, 61 insertions(+), 35 deletions(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index e1fd7c11..357b2156 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -52,7 +52,7 @@ bool CMQCluster::IsAuthenticated() bool CMQCluster::HandleInitialize() { - if (NODE_CATEGORY::BBCNODE == catNode) // todo: + if (NODE_CATEGORY::BBCNODE == catNode) { Log("CMQCluster::HandleInitialize(): bbc node so bypass"); return true; @@ -154,12 +154,11 @@ bool CMQCluster::HandleInvoke() lastHeightResp = pBlockChain->GetBlockCount(pCoreProtocol->GetGenesisBlockHash()) - 1; pForkManager->SetForkFilter(nodes[0].vecOwnedForks); - topicReqBlk = "Cluster01/" + clientID + "/SyncBlockReq"; - topicRespBlk = "Cluster01/" + clientID + "/SyncBlockResp"; - topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; // todo: should configure DPOSNODE - Log("CMQCluster::HandleInvoke(): fork node clientid [%s] with topics:" - "\t[%s]\n\t[%s]", - clientID.c_str(), topicRespBlk.c_str(), topicRbBlk.c_str()); + vecTopic[TOPIC_SUFFIX_REQ_BLOCK] = prefixTopic + clientID + vecSuffixTopic[TOPIC_SUFFIX_REQ_BLOCK]; + vecTopic[TOPIC_SUFFIX_RESP_BLOCK] = prefixTopic + clientID + vecSuffixTopic[TOPIC_SUFFIX_RESP_BLOCK]; + vecTopic[TOPIC_SUFFIX_UPDATE_BLOCK] = prefixTopic + dposNodeCliID + vecSuffixTopic[TOPIC_SUFFIX_UPDATE_BLOCK]; + Log("CMQCluster::HandleInvoke(): fork node clientid [%s] with topics:\t[%s]\n\t[%s]", + clientID.c_str(), vecTopic[TOPIC_SUFFIX_RESP_BLOCK].c_str(), vecTopic[TOPIC_SUFFIX_UPDATE_BLOCK].c_str()); if (!PostBlockRequest(-1)) { @@ -179,10 +178,10 @@ bool CMQCluster::HandleInvoke() } } lastHeightResp = -1; - topicReqBlk = "Cluster01/+/SyncBlockReq"; - topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; // todo: DPOSNODE should come from storage - Log("CMQCluster::HandleInvoke(): dpos node clientid [%s] with topic [%s]", - clientID.c_str(), topicReqBlk.c_str()); + vecTopic[TOPIC_SUFFIX_REQ_BLOCK] = prefixTopic + "+" + vecSuffixTopic[TOPIC_SUFFIX_REQ_BLOCK]; + vecTopic[TOPIC_SUFFIX_UPDATE_BLOCK] = prefixTopic + clientID + vecSuffixTopic[TOPIC_SUFFIX_UPDATE_BLOCK]; + Log("CMQCluster::HandleInvoke(): dpos node clientid [%s] with topic [%s][%s]", + clientID.c_str(), vecTopic[TOPIC_SUFFIX_REQ_BLOCK].c_str(), vecTopic[TOPIC_SUFFIX_UPDATE_BLOCK].c_str()); nodes.clear(); if (!pBlockChain->FetchSuperNode(nodes, 1 << 1)) @@ -269,11 +268,11 @@ bool CMQCluster::HandleEvent(CEventMQChainUpdate& eventMqUpdateChain) Log("CMQCluster::HandleEvent(): rollback-topic[%s]:" "forkheight[%d] forkhash[%s] shortlen[%d]", - topicRbBlk.c_str(), rbc.rbHeight, rbc.rbHash.ToString().c_str(), rbc.rbSize); + vecTopic[TOPIC_SUFFIX_UPDATE_BLOCK].c_str(), rbc.rbHeight, rbc.rbHash.ToString().c_str(), rbc.rbSize); { boost::unique_lock lock(mtxSend); - deqSendBuff.emplace_back(make_pair(topicRbBlk, spRBC)); + deqSendBuff.emplace_back(make_pair(vecTopic[TOPIC_SUFFIX_UPDATE_BLOCK], spRBC)); } condSend.notify_all(); @@ -287,13 +286,13 @@ bool CMQCluster::HandleEvent(CEventMQEnrollUpdate& eventMqUpdateEnroll) vector forks = eventMqUpdateEnroll.data.vecForksOwned; if (NODE_CATEGORY::FORKNODE == catNode) { - topicReqBlk = "Cluster01/" + clientID + "/SyncBlockReq"; - topicRespBlk = "Cluster01/" + clientID + "/SyncBlockResp"; - topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; + vecTopic[TOPIC_SUFFIX_REQ_BLOCK] = prefixTopic + clientID + vecSuffixTopic[TOPIC_SUFFIX_REQ_BLOCK]; + vecTopic[TOPIC_SUFFIX_RESP_BLOCK] = prefixTopic + clientID + vecSuffixTopic[TOPIC_SUFFIX_RESP_BLOCK]; + vecTopic[TOPIC_SUFFIX_UPDATE_BLOCK] = prefixTopic + dposNodeCliID + vecSuffixTopic[TOPIC_SUFFIX_UPDATE_BLOCK]; Log("CMQCluster::HandleEvent(): fork node clientid [%s] ip [%d] with topics:" "\n[%s]\n[%s]", id.c_str(), eventMqUpdateEnroll.data.ipAddr, - topicRespBlk.c_str(), topicRbBlk.c_str()); + vecTopic[TOPIC_SUFFIX_RESP_BLOCK].c_str(), vecTopic[TOPIC_SUFFIX_UPDATE_BLOCK].c_str()); for (const auto& fork : forks) { Log("CMQCluster::HandleEvent(): fork [%s] intended to be produced " @@ -323,10 +322,10 @@ bool CMQCluster::HandleEvent(CEventMQEnrollUpdate& eventMqUpdateEnroll) if (1 == forks.size() && 0 == eventMqUpdateEnroll.data.ipAddr && forks[0] == pCoreProtocol->GetGenesisBlockHash()) { //dpos node enrolled by self - topicReqBlk = "Cluster01/+/SyncBlockReq"; - topicRbBlk = "Cluster01/DPOSNODE/UpdateBlock"; + vecTopic[TOPIC_SUFFIX_REQ_BLOCK] = prefixTopic + "+" + vecSuffixTopic[TOPIC_SUFFIX_REQ_BLOCK]; + vecTopic[TOPIC_SUFFIX_UPDATE_BLOCK] = prefixTopic + clientID + vecSuffixTopic[TOPIC_SUFFIX_UPDATE_BLOCK]; Log("CMQCluster::HandleEvent(): dpos node clientid [%s] with topic [%s][%s]", - id.c_str(), topicReqBlk.c_str(), topicRbBlk.c_str()); + id.c_str(), vecTopic[TOPIC_SUFFIX_REQ_BLOCK].c_str(), vecTopic[TOPIC_SUFFIX_UPDATE_BLOCK].c_str()); { boost::unique_lock lock(mtxStatus); @@ -468,7 +467,7 @@ bool CMQCluster::PostBlockRequest(int syncHeight) CBufferPtr spSS(new CBufStream); *spSS.get() << req; - AppendSendQueue(topicReqBlk, spSS); + AppendSendQueue(vecTopic[TOPIC_SUFFIX_REQ_BLOCK], spSS); return true; } @@ -517,7 +516,7 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) case NODE_CATEGORY::FORKNODE: { Log("CMQCluster::OnReceiveMessage(): current sync height is [%d]", int(lastHeightResp)); - if (topicRbBlk != topic) //todo: to refactor + if (topic.find(vecSuffixTopic[TOPIC_SUFFIX_RESP_BLOCK]) && topic.find(prefixTopic)) { //respond to request block of main chain //unpack payload CSyncBlockResponse resp; @@ -619,8 +618,11 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) return; } } - } - else + + return; + } // end of dealing with response of requesting block + + if (topic.find(vecSuffixTopic[TOPIC_SUFFIX_UPDATE_BLOCK]) && topic.find(prefixTopic)) { //roll back blocks on main chain //unpack payload CRollbackBlock rb; @@ -738,7 +740,9 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) nRollNum = rb.rbSize; } } - } + + return; + } // end of dealing with rollback of main chain break; } @@ -941,8 +945,14 @@ class CMQCallback : mqCluster.LogEvent("[connected]"); if (CMQCluster::NODE_CATEGORY::FORKNODE == mqCluster.catNode) { - asynCli.subscribe(mqCluster.topicRespBlk, CMQCluster::QOS1, nullptr, subListener); - cout << "\nSubscribing to topic '" << mqCluster.topicRespBlk << "'\n" + asynCli.subscribe(mqCluster.vecTopic[CMQCluster::TOPIC_SUFFIX_RESP_BLOCK], CMQCluster::QOS1, nullptr, subListener); + cout << "\nSubscribing to topic '" << mqCluster.vecTopic[CMQCluster::TOPIC_SUFFIX_RESP_BLOCK] << "'\n" + << "\tfor client " << mqCluster.clientID + << " using QoS" << CMQCluster::QOS1 << endl; + + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + asynCli.subscribe(mqCluster.vecTopic[CMQCluster::TOPIC_SUFFIX_UPDATE_BLOCK], CMQCluster::QOS1, nullptr, subListener); + cout << "\nSubscribing to topic '" << mqCluster.vecTopic[CMQCluster::TOPIC_SUFFIX_UPDATE_BLOCK] << "'\n" << "\tfor client " << mqCluster.clientID << " using QoS" << CMQCluster::QOS1 << endl; @@ -954,8 +964,8 @@ class CMQCallback : } else if (CMQCluster::NODE_CATEGORY::DPOSNODE == mqCluster.catNode) { - asynCli.subscribe(mqCluster.topicReqBlk, CMQCluster::QOS1, nullptr, subListener); - cout << "\nSubscribing to topic '" << mqCluster.topicReqBlk << "'\n" + asynCli.subscribe(mqCluster.vecTopic[CMQCluster::TOPIC_SUFFIX_REQ_BLOCK], CMQCluster::QOS1, nullptr, subListener); + cout << "\nSubscribing to topic '" << mqCluster.vecTopic[CMQCluster::TOPIC_SUFFIX_REQ_BLOCK] << "'\n" << "\tfor client " << mqCluster.clientID << " using QoS" << CMQCluster::QOS1 << endl; } diff --git a/src/bigbang/mqcluster.h b/src/bigbang/mqcluster.h index f4398705..8337b6eb 100644 --- a/src/bigbang/mqcluster.h +++ b/src/bigbang/mqcluster.h @@ -20,7 +20,7 @@ class CSyncBlockRequest public: uint32 ipAddr; uint8 forkNodeIdLen; - string forkNodeId; + std::string forkNodeId; uint8 forkNum; std::vector forkList; int32 lastHeight; @@ -180,16 +180,32 @@ class CMQCluster : public IMQCluster bool fAuth; bool fAbort; - string addrBroker; + std::string addrBroker; + const std::string dposNodeCliID = "DPOSNODE"; + const std::string prefixTopic = "Cluster01/"; + enum + { + TOPIC_SUFFIX_REQ_BLOCK = 0, + TOPIC_SUFFIX_RESP_BLOCK = 1, + TOPIC_SUFFIX_UPDATE_BLOCK = 2, + TOPIC_SUFFIX_ASGN_BIZFORK = 3, + TOPIC_SUFFIX_MAX + }; + const std::vector vecSuffixTopic = { + "/SyncBlockReq", + "/SyncBlockResp", + "/UpdateBlock", + "/AssignBizFork" + }; + NODE_CATEGORY catNode; - string topicReqBlk; - string topicRespBlk; - string topicRbBlk; + std::vector vecTopic; //shared by both dpos and fork node + std::map mapBizForkTopic; //only for dpos node boost::mutex mtxStatus; boost::condition_variable condStatus; - string clientID; + std::string clientID; uint32 ipAddr; std::set setBizFork; From 179a5aca675c7d62bf0f0ea9a90b2a000dc52fb2 Mon Sep 17 00:00:00 2001 From: OuYun Date: Thu, 16 Apr 2020 16:32:18 +0800 Subject: [PATCH 079/181] add handling of biz fork assignment by dpos node through mq broker --- src/bigbang/mqcluster.cpp | 99 +++++++++++++++++++++++++++++++++------ src/bigbang/mqcluster.h | 7 ++- src/bigbang/mqevent.h | 1 - src/bigbang/netchn.cpp | 23 ++++++++- src/bigbang/netchn.h | 1 + src/network/peernet.h | 1 + 6 files changed, 113 insertions(+), 19 deletions(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 357b2156..0fb921fe 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -24,6 +24,7 @@ CMQCluster::CMQCluster(int catNodeIn) pCoreProtocol(nullptr), pBlockChain(nullptr), pService(nullptr), + pNetChannel(nullptr), fAuth(false), fAbort(false), addrBroker("tcp://localhost:1883"), @@ -43,6 +44,7 @@ CMQCluster::CMQCluster(int catNodeIn) catNode = NODE_CATEGORY::DPOSNODE; break; } + std::atomic_init(&isMainChainBlockBest, false); } bool CMQCluster::IsAuthenticated() @@ -88,6 +90,12 @@ bool CMQCluster::HandleInitialize() return false; } + if (!GetObject("netchannel", pNetChannel)) + { + Error("Failed to request peer net datachannel\n"); + return false; + } + Log("CMQCluster::HandleInitialize() successfully"); return true; } @@ -99,6 +107,7 @@ void CMQCluster::HandleDeinitialize() pDispatcher = nullptr; pService = nullptr; pForkManager = nullptr; + pNetChannel = nullptr; } bool CMQCluster::HandleInvoke() @@ -173,7 +182,8 @@ bool CMQCluster::HandleInvoke() if (setBizFork.size() != 1 || *(setBizFork.begin()) != pCoreProtocol->GetGenesisBlockHash()) { Error("CMQCluster::HandleInvoke(): dpos node enrollment info " - "invalid [%d] for self", setBizFork.size()); + "invalid [%d] for self", + setBizFork.size()); return false; } } @@ -357,6 +367,11 @@ bool CMQCluster::HandleEvent(CEventMQEnrollUpdate& eventMqUpdateEnroll) return true; } +bool CMQCluster::HandleEvent(CEventMQAgreement& eventMqAgreement) +{ + return true; +} + bool CMQCluster::HandleEvent(CEventMQBizForkUpdate& eventMqBizFork) { Log("CMQCluster::HandleEvent(): biz forks payload is coming"); @@ -370,12 +385,12 @@ bool CMQCluster::HandleEvent(CEventMQBizForkUpdate& eventMqBizFork) } if (NODE_CATEGORY::DPOSNODE == catNode) - { //spread ip/fork through mq broker + { //spread ip/fork through mq broker const storage::CForkKnownIpSetById& idxID = eventMqBizFork.data.get<0>(); for (auto const& cli : mapActiveMQForkNode) { CAssignBizFork biz; - auto& it = biz.mapBizForkIP; + auto& it = biz.mapIpBizFork; for (auto const& fork : cli.second.vecOwnedForks) { auto itBegin = idxID.equal_range(fork).first; @@ -383,13 +398,13 @@ bool CMQCluster::HandleEvent(CEventMQBizForkUpdate& eventMqBizFork) while (itBegin != itEnd) { - it[itBegin->forkID].push_back(itBegin->nodeIP); + it[itBegin->nodeIP].push_back(itBegin->forkID); ++itBegin; } } if (!it.empty()) { - string topic = "Cluster01/" + cli.second.superNodeID + "/AssignBizFork"; + string topic = prefixTopic + cli.second.superNodeID + vecSuffixTopic[TOPIC_SUFFIX_ASGN_BIZFORK]; PostBizForkAssign(topic, biz); } } @@ -398,11 +413,6 @@ bool CMQCluster::HandleEvent(CEventMQBizForkUpdate& eventMqBizFork) return true; } -bool CMQCluster::HandleEvent(CEventMQAgreement& eventMqAgreement) -{ - return true; -} - bool CMQCluster::LogEvent(const string& info) { cout << "callback to CMQCluster when MQ-EVENT" << info << endl; @@ -601,12 +611,14 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) if (resp.isBest) { //when reach best height, send request by timer + std::atomic_store(&isMainChainBlockBest, true); nReqBlkTimerID = SetTimer(1000 * 60, boost::bind(&CMQCluster::RequestBlockTimerFunc, this, _1)); return; } else { + std::atomic_store(&isMainChainBlockBest, false); if (0 != nReqBlkTimerID) { CancelTimer(nReqBlkTimerID); @@ -620,7 +632,7 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) } return; - } // end of dealing with response of requesting block + } // end of dealing with response of requesting block if (topic.find(vecSuffixTopic[TOPIC_SUFFIX_UPDATE_BLOCK]) && topic.find(prefixTopic)) { //roll back blocks on main chain @@ -742,7 +754,66 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) } return; - } // end of dealing with rollback of main chain + } // end of dealing with rollback of main chain + + if (topic.find(vecSuffixTopic[TOPIC_SUFFIX_ASGN_BIZFORK]) && topic.find(prefixTopic)) + { //receive biz fork list from dpos node by MQ broker + //unpack payload + CAssignBizFork biz; + try + { + payload >> biz; + } + catch (exception& e) + { + StdError(__PRETTY_FUNCTION__, e.what()); + Error("CMQCluster::OnReceiveMessage(): failed to unpack bizfork msg"); + return; + } + + //populate to memory + for (auto const& it : biz.mapIpBizFork) + { + mapOuterNode[it.first].ipAddr = it.first; + mapOuterNode[it.first].nodeCat = 0; + mapOuterNode[it.first].vecOwnedForks = it.second; + Log("CMQCluster::OnReceiveMessage(): adding new outer nodes from mq broker " + "by dpos node succeeded [%s]", + mapOuterNode[it.first].ToString().c_str()); + } + + //save to db storage + for (auto const& it : mapOuterNode) + { + if (!pBlockChain->AddNewSuperNode(it.second)) + { + Error("CMQCluster::OnReceiveMessage(): failed to add new outer nodes from mq broker by dpos node"); + return; + } + Log("CMQCluster::OnReceiveMessage(): adding new outer nodes from mq broker by dpos node succeeded"); + } + + //launch connecting those outer nodes if main chain has been best block + if (std::atomic_load(&isMainChainBlockBest)) + { + vector ips; + for (auto const& node : mapOuterNode) + { + ips.push_back(node.first); + } + if (!ips.empty()) + { + if (!pNetChannel->AddBizForkNodes(ips)) + { + Error("CMQCluster::OnReceiveMessage(): failed to post add new node msg"); + return; + } + Log("CMQCluster::OnReceiveMessage(): posting add new node msg succeeded"); + } + } + + return; + } // end of dealing with biz fork assignment break; } @@ -957,8 +1028,8 @@ class CMQCallback : << " using QoS" << CMQCluster::QOS1 << endl; std::this_thread::sleep_for(std::chrono::milliseconds(100)); - asynCli.subscribe(mqCluster.topicRbBlk, CMQCluster::QOS1, nullptr, subListener); - cout << "\nSubscribing to topic '" << mqCluster.topicRbBlk << "'\n" + asynCli.subscribe(mqCluster.vecTopic[CMQCluster::TOPIC_SUFFIX_ASGN_BIZFORK], CMQCluster::QOS1, nullptr, subListener); + cout << "\nSubscribing to topic '" << mqCluster.vecTopic[CMQCluster::TOPIC_SUFFIX_ASGN_BIZFORK] << "'\n" << "\tfor client " << mqCluster.clientID << " using QoS" << CMQCluster::QOS1 << endl; } diff --git a/src/bigbang/mqcluster.h b/src/bigbang/mqcluster.h index 8337b6eb..8c324af4 100644 --- a/src/bigbang/mqcluster.h +++ b/src/bigbang/mqcluster.h @@ -9,6 +9,7 @@ #include "mqdb.h" #include "mqevent.h" #include "xengine.h" +#include "netchn.h" namespace bigbang { @@ -93,13 +94,13 @@ class CAssignBizFork friend xengine::CStream; public: - std::map> mapBizForkIP; + std::map> mapIpBizFork; protected: template void Serialize(xengine::CStream& s, O& opt) { - s.Serialize(mapBizForkIP, opt); + s.Serialize(mapIpBizFork, opt); } }; @@ -164,6 +165,7 @@ class CMQCluster : public IMQCluster IDispatcher* pDispatcher; IService* pService; IForkManager* pForkManager; + INetChannel* pNetChannel; boost::mutex mutex; boost::condition_variable condMQ; @@ -208,6 +210,7 @@ class CMQCluster : public IMQCluster std::string clientID; uint32 ipAddr; std::set setBizFork; + std::atomic_bool isMainChainBlockBest; std::map mapActiveMQForkNode; //only for dpos node std::map mapOuterNode; //for dpos/fork node diff --git a/src/bigbang/mqevent.h b/src/bigbang/mqevent.h index c2b62555..3025287a 100644 --- a/src/bigbang/mqevent.h +++ b/src/bigbang/mqevent.h @@ -31,7 +31,6 @@ typedef TYPE_MQEVENT(EVENT_MQ_SYNCBLOCK, std::string) CEventMQSyncBlock; typedef TYPE_MQEVENT(EVENT_MQ_UPDATEBLOCK, CMqRollbackUpdate) CEventMQChainUpdate; typedef TYPE_MQEVENT(EVENT_MQ_ENROLLUPDATE, CMqSuperNodeUpdate) CEventMQEnrollUpdate; typedef TYPE_MQEVENT(EVENT_MQ_AGREEMENT, CDelegateAgreement) CEventMQAgreement; -//typedef TYPE_MQEVENT(EVENT_MQ_UPDATEBIZFORK, std::vector) CEventMQBizForkUpdate; typedef TYPE_MQEVENT(EVENT_MQ_UPDATEBIZFORK, storage::CForkKnownIPSet) CEventMQBizForkUpdate; class CMQEventListener : virtual public CEventListener diff --git a/src/bigbang/netchn.cpp b/src/bigbang/netchn.cpp index 79d3ca76..843f9a8a 100644 --- a/src/bigbang/netchn.cpp +++ b/src/bigbang/netchn.cpp @@ -368,6 +368,25 @@ void CNetChannel::UnsubscribeFork(const uint256& hashFork) } } +bool CNetChannel::AddBizForkNodes(const std::vector& nodes) +{ + for (auto const& node : nodes) + { + string strIP = storage::CSuperNode::Int2Ip(node); + CNetHost host(strIP, DEFAULT_P2PPORT); + CEventPeerNetAddNode eventAddNode(0); + eventAddNode.data = host; + if (!pPeerNet->DispatchEvent(&eventAddNode)) + { + StdError("NetChannel", "AddBizForkNodes: Adding biz fork peer node[%s] failed", strIP.c_str()); + return false; + } + StdLog("NetChannel", "AddBizForkNodes: Adding biz fork peer node[%s] succeeded", strIP.c_str()); + } + + return true; +} + bool CNetChannel::HandleEvent(network::CEventPeerActive& eventActive) { uint64 nNonce = eventActive.nNonce; @@ -624,10 +643,10 @@ bool CNetChannel::HandleEvent(network::CEventPeerBizForks& eventBizForks) eventAddNode.data = host; if (!pPeerNet->DispatchEvent(&eventAddNode)) { - StdError("NetChannel", "CEventPeerBizForks: Add peer node[%s] failed", strIP.c_str()); + StdError("NetChannel", "CEventPeerBizForks: Adding outer peer node[%s] failed", strIP.c_str()); return false; } - StdLog("NetChannel", "CEventPeerBizForks: Add peer node[%s] succeeded", strIP.c_str()); + StdLog("NetChannel", "CEventPeerBizForks: Adding outer peer node[%s] succeeded", strIP.c_str()); } return true; diff --git a/src/bigbang/netchn.h b/src/bigbang/netchn.h index cdd4f211..004d889f 100644 --- a/src/bigbang/netchn.h +++ b/src/bigbang/netchn.h @@ -238,6 +238,7 @@ class CNetChannel : public network::INetChannel void BroadcastTxInv(const uint256& hashFork) override; void SubscribeFork(const uint256& hashFork, const uint64& nNonce) override; void UnsubscribeFork(const uint256& hashFork) override; + bool AddBizForkNodes(const std::vector& nodes) override; protected: enum diff --git a/src/network/peernet.h b/src/network/peernet.h index 868e1c7c..b567ff01 100644 --- a/src/network/peernet.h +++ b/src/network/peernet.h @@ -25,6 +25,7 @@ class INetChannel : public xengine::IIOModule, virtual public CBbPeerEventListen virtual void BroadcastTxInv(const uint256& hashFork) = 0; virtual void SubscribeFork(const uint256& hashFork, const uint64& nNonce) = 0; virtual void UnsubscribeFork(const uint256& hashFork) = 0; + virtual bool AddBizForkNodes(const std::vector& nodes) = 0; }; class IDelegatedChannel : public xengine::IIOModule, virtual public CBbPeerEventListener From dd0b4e6bd9ae84d903b04b5d8fa5587fd462415e Mon Sep 17 00:00:00 2001 From: OuYun Date: Thu, 16 Apr 2020 16:57:51 +0800 Subject: [PATCH 080/181] transfer routine of adding biz fork nodes from netchannel to service module because of compile error caused by cross including --- src/bigbang/base.h | 1 + src/bigbang/mqcluster.cpp | 10 +--------- src/bigbang/mqcluster.h | 2 -- src/bigbang/netchn.cpp | 19 ------------------- src/bigbang/netchn.h | 1 - src/bigbang/service.cpp | 19 +++++++++++++++++++ src/bigbang/service.h | 1 + src/network/peernet.h | 1 - 8 files changed, 22 insertions(+), 32 deletions(-) diff --git a/src/bigbang/base.h b/src/bigbang/base.h index 5df9f003..afc75ef7 100644 --- a/src/bigbang/base.h +++ b/src/bigbang/base.h @@ -265,6 +265,7 @@ class IService : public xengine::IBase virtual void GetPeers(std::vector& vPeerInfo) = 0; virtual bool AddNode(const xengine::CNetHost& node) = 0; virtual bool RemoveNode(const xengine::CNetHost& node) = 0; + virtual bool AddBizForkNodes(const std::vector& nodes) = 0; /* Blockchain & Tx Pool*/ virtual int GetForkCount() = 0; virtual bool HaveFork(const uint256& hashFork) = 0; diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 0fb921fe..afcaffa4 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -24,7 +24,6 @@ CMQCluster::CMQCluster(int catNodeIn) pCoreProtocol(nullptr), pBlockChain(nullptr), pService(nullptr), - pNetChannel(nullptr), fAuth(false), fAbort(false), addrBroker("tcp://localhost:1883"), @@ -90,12 +89,6 @@ bool CMQCluster::HandleInitialize() return false; } - if (!GetObject("netchannel", pNetChannel)) - { - Error("Failed to request peer net datachannel\n"); - return false; - } - Log("CMQCluster::HandleInitialize() successfully"); return true; } @@ -107,7 +100,6 @@ void CMQCluster::HandleDeinitialize() pDispatcher = nullptr; pService = nullptr; pForkManager = nullptr; - pNetChannel = nullptr; } bool CMQCluster::HandleInvoke() @@ -803,7 +795,7 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) } if (!ips.empty()) { - if (!pNetChannel->AddBizForkNodes(ips)) + if (!pService->AddBizForkNodes(ips)) { Error("CMQCluster::OnReceiveMessage(): failed to post add new node msg"); return; diff --git a/src/bigbang/mqcluster.h b/src/bigbang/mqcluster.h index 8c324af4..8b8eea75 100644 --- a/src/bigbang/mqcluster.h +++ b/src/bigbang/mqcluster.h @@ -9,7 +9,6 @@ #include "mqdb.h" #include "mqevent.h" #include "xengine.h" -#include "netchn.h" namespace bigbang { @@ -165,7 +164,6 @@ class CMQCluster : public IMQCluster IDispatcher* pDispatcher; IService* pService; IForkManager* pForkManager; - INetChannel* pNetChannel; boost::mutex mutex; boost::condition_variable condMQ; diff --git a/src/bigbang/netchn.cpp b/src/bigbang/netchn.cpp index 843f9a8a..12d59f4a 100644 --- a/src/bigbang/netchn.cpp +++ b/src/bigbang/netchn.cpp @@ -368,25 +368,6 @@ void CNetChannel::UnsubscribeFork(const uint256& hashFork) } } -bool CNetChannel::AddBizForkNodes(const std::vector& nodes) -{ - for (auto const& node : nodes) - { - string strIP = storage::CSuperNode::Int2Ip(node); - CNetHost host(strIP, DEFAULT_P2PPORT); - CEventPeerNetAddNode eventAddNode(0); - eventAddNode.data = host; - if (!pPeerNet->DispatchEvent(&eventAddNode)) - { - StdError("NetChannel", "AddBizForkNodes: Adding biz fork peer node[%s] failed", strIP.c_str()); - return false; - } - StdLog("NetChannel", "AddBizForkNodes: Adding biz fork peer node[%s] succeeded", strIP.c_str()); - } - - return true; -} - bool CNetChannel::HandleEvent(network::CEventPeerActive& eventActive) { uint64 nNonce = eventActive.nNonce; diff --git a/src/bigbang/netchn.h b/src/bigbang/netchn.h index 004d889f..cdd4f211 100644 --- a/src/bigbang/netchn.h +++ b/src/bigbang/netchn.h @@ -238,7 +238,6 @@ class CNetChannel : public network::INetChannel void BroadcastTxInv(const uint256& hashFork) override; void SubscribeFork(const uint256& hashFork, const uint64& nNonce) override; void UnsubscribeFork(const uint256& hashFork) override; - bool AddBizForkNodes(const std::vector& nodes) override; protected: enum diff --git a/src/bigbang/service.cpp b/src/bigbang/service.cpp index c98ca494..05edf5d1 100644 --- a/src/bigbang/service.cpp +++ b/src/bigbang/service.cpp @@ -190,6 +190,25 @@ bool CService::RemoveNode(const CNetHost& node) return pNetwork->DispatchEvent(&eventRemoveNode); } +bool CService::AddBizForkNodes(const std::vector& nodes) +{ + for (auto const& node : nodes) + { + string strIP = storage::CSuperNode::Int2Ip(node); + CNetHost host(strIP, DEFAULT_P2PPORT); + CEventPeerNetAddNode eventAddNode(0); + eventAddNode.data = host; + if (!pNetwork->DispatchEvent(&eventAddNode)) + { + StdError("CService", "AddBizForkNodes: Adding biz fork peer node[%s] failed", strIP.c_str()); + return false; + } + StdLog("CService", "AddBizForkNodes: Adding biz fork peer node[%s] succeeded", strIP.c_str()); + } + + return true; +} + int CService::GetForkCount() { return mapForkStatus.size(); diff --git a/src/bigbang/service.h b/src/bigbang/service.h index a9690317..3f654c4b 100644 --- a/src/bigbang/service.h +++ b/src/bigbang/service.h @@ -30,6 +30,7 @@ class CService : public IService void GetPeers(std::vector& vPeerInfo) override; bool AddNode(const xengine::CNetHost& node) override; bool RemoveNode(const xengine::CNetHost& node) override; + bool AddBizForkNodes(const std::vector& nodes) override; /* Blockchain & Tx Pool*/ int GetForkCount() override; bool HaveFork(const uint256& hashFork) override; diff --git a/src/network/peernet.h b/src/network/peernet.h index b567ff01..868e1c7c 100644 --- a/src/network/peernet.h +++ b/src/network/peernet.h @@ -25,7 +25,6 @@ class INetChannel : public xengine::IIOModule, virtual public CBbPeerEventListen virtual void BroadcastTxInv(const uint256& hashFork) = 0; virtual void SubscribeFork(const uint256& hashFork, const uint64& nNonce) = 0; virtual void UnsubscribeFork(const uint256& hashFork) = 0; - virtual bool AddBizForkNodes(const std::vector& nodes) = 0; }; class IDelegatedChannel : public xengine::IIOModule, virtual public CBbPeerEventListener From fe3dacf374212719e48d655fb8c96be21f8f7da7 Mon Sep 17 00:00:00 2001 From: OuYun Date: Thu, 16 Apr 2020 17:18:58 +0800 Subject: [PATCH 081/181] abstract a common function to share with two callers --- src/bigbang/mqcluster.cpp | 39 ++++++++++++++++++++++++--------------- src/bigbang/mqcluster.h | 1 + src/bigbang/netchn.cpp | 2 -- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index afcaffa4..31d6a457 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -207,7 +207,7 @@ bool CMQCluster::HandleInvoke() Error("CMQCluster::HandleInvoke(): list all other outer nodes failed"); return false; } - for (auto const& node : nodes) // todo: should send event to add peers to netchannel module if fork node itself + for (auto const& node : nodes) { mapOuterNode.emplace(make_pair(node.ipAddr, storage::CSuperNode(node.superNodeID, node.ipAddr, @@ -216,6 +216,8 @@ bool CMQCluster::HandleInvoke() Log("CMQCluster::HandleInvoke(): load outer node [%s]", node.ToString().c_str()); } + PostAddBizForkNode(); + if (!ThreadStart(thrMqttClient)) { return false; @@ -788,20 +790,7 @@ void CMQCluster::OnReceiveMessage(const std::string& topic, CBufStream& payload) //launch connecting those outer nodes if main chain has been best block if (std::atomic_load(&isMainChainBlockBest)) { - vector ips; - for (auto const& node : mapOuterNode) - { - ips.push_back(node.first); - } - if (!ips.empty()) - { - if (!pService->AddBizForkNodes(ips)) - { - Error("CMQCluster::OnReceiveMessage(): failed to post add new node msg"); - return; - } - Log("CMQCluster::OnReceiveMessage(): posting add new node msg succeeded"); - } + PostAddBizForkNode(); } return; @@ -1201,4 +1190,24 @@ void CMQCluster::MqttThreadFunc() Log("exiting thread function of MQTT"); } +bool CMQCluster::PostAddBizForkNode() +{ + vector ips; + for (auto const& node : mapOuterNode) + { + ips.push_back(node.first); + } + if (!ips.empty()) + { + if (!pService->AddBizForkNodes(ips)) + { + Error("CMQCluster::PostAddBizForkNode(): failed to post add new node msg"); + return false; + } + Log("CMQCluster::PostAddBizForkNode(): posting add new node msg succeeded"); + } + + return true; +} + } // namespace bigbang diff --git a/src/bigbang/mqcluster.h b/src/bigbang/mqcluster.h index 8b8eea75..f195f857 100644 --- a/src/bigbang/mqcluster.h +++ b/src/bigbang/mqcluster.h @@ -177,6 +177,7 @@ class CMQCluster : public IMQCluster void OnReceiveMessage(const std::string& topic, CBufStream& payload); bool ClientAgent(MQ_CLI_ACTION action); void MqttThreadFunc(); + bool PostAddBizForkNode(); bool fAuth; bool fAbort; diff --git a/src/bigbang/netchn.cpp b/src/bigbang/netchn.cpp index 12d59f4a..4d1183b3 100644 --- a/src/bigbang/netchn.cpp +++ b/src/bigbang/netchn.cpp @@ -599,8 +599,6 @@ bool CNetChannel::HandleEvent(network::CEventPeerBizForks& eventBizForks) return true; } - // todo: here should send event to add peers to netchannel module if fork nodes - vector vIP; for (auto const& it : mapForkIp) { From 4cd82d698196af63b7c9cab5c817c368df84ff59 Mon Sep 17 00:00:00 2001 From: OuYun Date: Thu, 16 Apr 2020 18:25:03 +0800 Subject: [PATCH 082/181] - add two options; - mq cluster gets config value from options above rather than pre-defined; --- script/template/options.json | 16 ++++++++++++++++ src/bigbang/mqcluster.cpp | 3 +++ src/bigbang/mqcluster.h | 2 +- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/script/template/options.json b/script/template/options.json index 6d82ca57..65766b36 100755 --- a/script/template/options.json +++ b/script/template/options.json @@ -16,6 +16,22 @@ "format": "-catofnode=", "desc": "Identify node category: 0 - BBCNODE; 1 - FORKNODE; 2 - DPOSNODE, 0 by default" }, + { + "name": "strMQBrokerURI", + "type": "string", + "opt": "mqbroker", + "default": "tcp://localhost:1883", + "format": "-mqbroker=://:", + "desc": "Specified URI to which dpos node or fork node can connect through MQTT" + }, + { + "name": "strDposNodeID", + "type": "string", + "opt": "dposid", + "default": "DPOSNODE", + "format": "-dposid", + "desc": "Identification of dpos node used by fork nodes in the same cluster" + }, { "name": "fWallet", "type": "bool", diff --git a/src/bigbang/mqcluster.cpp b/src/bigbang/mqcluster.cpp index 31d6a457..f073525d 100644 --- a/src/bigbang/mqcluster.cpp +++ b/src/bigbang/mqcluster.cpp @@ -89,6 +89,9 @@ bool CMQCluster::HandleInitialize() return false; } + addrBroker = dynamic_cast(Config())->strMQBrokerURI; + dposNodeCliID = dynamic_cast(Config())->strDposNodeID; + Log("CMQCluster::HandleInitialize() successfully"); return true; } diff --git a/src/bigbang/mqcluster.h b/src/bigbang/mqcluster.h index f195f857..8358f675 100644 --- a/src/bigbang/mqcluster.h +++ b/src/bigbang/mqcluster.h @@ -182,7 +182,7 @@ class CMQCluster : public IMQCluster bool fAuth; bool fAbort; std::string addrBroker; - const std::string dposNodeCliID = "DPOSNODE"; + std::string dposNodeCliID; const std::string prefixTopic = "Cluster01/"; enum { From c4cffd77bca7304a8f2380c9f4403ed5e2b999bc Mon Sep 17 00:00:00 2001 From: OuYun Date: Thu, 16 Apr 2020 18:29:03 +0800 Subject: [PATCH 083/181] - convert line separators; - add templates for three newly-added options and update some others; --- doc/config-sample/bigbang.conf | 369 +++++++++++++++++---------------- 1 file changed, 192 insertions(+), 177 deletions(-) diff --git a/doc/config-sample/bigbang.conf b/doc/config-sample/bigbang.conf index 8ab242b1..f9408abb 100644 --- a/doc/config-sample/bigbang.conf +++ b/doc/config-sample/bigbang.conf @@ -1,177 +1,192 @@ -## -## bigbang.conf configuration file. Lines beginning with # are comments. -## - - -# Network-related options: - - -# Note that if you use testnet, particularly with the options -# addnode, connect, port, rpcport or rpchost, you will also -# want to read "[Sections]" further down. - -# Run on the test network instead of the real bigbang network. -#testnet=false -#testnet -# or -#testnet=true - -# Listening mode, Accept IPv4 and IPv6 connections from outside (disabled by default) -#listen=false -#listen -# or -#listen=true - -# Accept IPv4 connections from outside (default: false) -#listen4=false - -# Accept IPv6 connections from outside (default: false) -#listen6=false - -# Port on which to listen for connections (default: 9901, testnet: 9903) -#port= - -# Used in the case of node is being behind a NAT, The form of : of address of gateway( can be IPv4 or IPv6, default : 9901, IPv6 format: [ip]:port) -#gateway=: - -# Maximum number of inbound+outbound connections(125 by default). -#maxconnections= - -# Specify connection timeout (in milliseconds) -#timeout= - -# Add a node to connect to and attempt to keep the connection open(
can be IPv4 or IPv6 or domain name, default : 9901, IPv6 format: [ip]:port) -# Use as many addnode= settings as you like to connect to specific peers -#addnode=69.164.218.197 -#addnode=10.0.0.2:8333 - -# Connect only to the specified node(
can be IPv4 or IPv6 or domain name, default : 9901, IPv6 format: [ip]:port) -# Alternatively use as many connect= settings as you like to connect ONLY to specific peers -#connect=69.164.218.197 -#connect=10.0.0.1:8333 - -# Trust node address(
can be IPv4 or IPv6) -#confidentAddress=
- -# DNSeed address list(
can be IPv4 or IPv6 or domain name, default : 9906, IPv6 format: [ip]:port) -#dnseed=
: - - -# JSON-RPC options (for controlling a running bigbang process) - - -# rpclisten=true tells bigbang daemon to accept JSON-RPC commands -#rpclisten=false - -# Bind to given address to listen for JSON-RPC connections. -#rpchost= - -# Listen for JSON-RPC connections on (default: 9902 or testnet: 9904)) -#rpcport=port - -# Accept RPC IPv4 connections (default: 0) -#rpclisten4=false - -# Accept RPC IPv6 connections (default: 0) -#rpclisten6 - -# name for JSON-RPC connections -#rpcuser= - -# for JSON-RPC connections -#rpcpassword= - -# Use OpenSSL (https) for JSON-RPC connections or not (default false) -#rpcssl - -# Verify SSL or not (default yes) -#norpcsslverify - -# SSL CA file name (default ca.crt) -#rpccafile= - -# Server certificate file (default: server.crt) -#rpccertfile= - -# Server private key (default: server.pem) -#rpcpkfile= - -# Acceptable ciphers (default: TLSv1+HIGH:!SSLv2:!aNULL:!eNULL:!AH:!3DES:@STRENGTH) -#rpcciphers= - -# Enable statistical data or not (default false) -#statdata - -# Enable write RPC log (default true) -#rpclog - -# Connection timeout