From a544e1b992c7ee206cdc77231159ce2a5a2becb2 Mon Sep 17 00:00:00 2001 From: Will Sobel Date: Thu, 19 Nov 2026 14:11:17 +0100 Subject: [PATCH 1/2] Fixed some MQTT broker related issues --- README.md | 5 +- src/mtconnect/configuration/parser.cpp | 7 ++- src/mtconnect/pipeline/topic_mapper.hpp | 34 ++++++++++++- .../adapter/mqtt/mqtt_broker_adapter.cpp | 42 +++++++++++++--- test_package/config_parser_test.cpp | 23 +++++++++ test_package/mqtt_broker_adapter_test.cpp | 12 +++++ test_package/topic_mapping_test.cpp | 49 ++++++++++++++++++- 7 files changed, 161 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 5daa04c6..a15cb449 100755 --- a/README.md +++ b/README.md @@ -280,7 +280,7 @@ Adapters { Protocol = mqtt_broker Device = Mill Topics = Mill/# - MqttHost = 0.0.0.0 # bind address, default 0.0.0.0 + MqttHost = 0.0.0.0 # bind address or host name, default 0.0.0.0 MqttPort = 8883 # default 1883, or 8883 with TLS MqttTls = true TlsCertificateChain = server.crt @@ -291,6 +291,9 @@ Adapters { } ``` +* `MqttHost` may be a host name such as `localhost`; it is resolved when the broker starts, preferring an IPv4 address. +* A `#` in a value, as in `Topics = Mill/#`, is part of the value; a comment starts with a `#` at the start of a line or after a space. +* A JSON payload is ingested for the device named by a segment of its topic (name or uuid), eg. `Lathe/data`, and otherwise for the adapter's `Device`. Device keys inside the JSON still take precedence. * Listener, transport, and credential settings (`MqttHost`, `MqttPort`, `MqttTls`, `MqttWs`, `MqttUserName`, `MqttPassword`, `MqttBrokerSecurity`) are only read from the adapter block, so they do not conflict with the MQTT sink settings. TLS certificate settings fall back to the top level values. * `TlsVerifyClientCertificate = true` with `TlsClientCAs` rejects TLS clients without a valid client certificate. `TlsCertificatePassword` and `TlsDHKey` are optional. * `MqttBrokerSecurity` is the path to an mqtt_cpp broker security JSON file for multiple users, salted `sha256` passwords, groups, and per-topic publish/subscribe ACLs. It replaces `MqttUserName`/`MqttPassword`. For example: diff --git a/src/mtconnect/configuration/parser.cpp b/src/mtconnect/configuration/parser.cpp index 38f6d732..bf995245 100644 --- a/src/mtconnect/configuration/parser.cpp +++ b/src/mtconnect/configuration/parser.cpp @@ -135,7 +135,11 @@ namespace mtconnect { using spirit::ascii::char_; m_name %= lexeme[+(char_ - (space | char_("=\\{}") | eol))]; - m_line %= no_skip[+(char_ - (char_("}#") | eol))]; + // A # only starts a comment at the start of a value or after a blank, so a value can + // contain one, eg. the MQTT topic filter Mill/#. Blanks are kept when more of the value + // follows them. + m_word = (char_ - (blank | char_("}#") | eol)) >> *(char_ - (blank | char_('}') | eol)); + m_line %= no_skip[m_word >> *hold[+blank >> m_word]]; m_string %= omit[char_("\"'")[_a = _1]] >> no_skip[*(char_ - char_(_a))] >> lit(_a); m_value %= *blank > (m_string | m_line); m_property = (m_name >> "=" > m_value > (eol | &char_("}#")))[property(_val, _1, _2)]; @@ -163,6 +167,7 @@ namespace mtconnect { protected: qi::rule m_name; + qi::rule m_word; qi::rule m_line; qi::rule> m_string; qi::rule m_value; diff --git a/src/mtconnect/pipeline/topic_mapper.hpp b/src/mtconnect/pipeline/topic_mapper.hpp index 829fa324..70cdff9e 100644 --- a/src/mtconnect/pipeline/topic_mapper.hpp +++ b/src/mtconnect/pipeline/topic_mapper.hpp @@ -140,11 +140,34 @@ namespace mtconnect::pipeline { return std::make_tuple(device, dataItem); } + /// @brief the first device named, by name or uuid, by a segment of the topic + /// @param topic the topic + /// @return the device or nullptr if no segment names one + DevicePtr findTopicDevice(const std::string& topic) + { + std::vector path; + boost::split(path, topic, boost::algorithm::is_any_of("/")); + for (const auto& tok : path) + { + if (tok.empty()) + continue; + if (auto device = m_context->m_contract->findDevice(tok)) + return device; + } + return nullptr; + } + EntityPtr operator()(entity::EntityPtr&& entity) override { PipelineMessagePtr result; auto& body = entity->getValue(); - auto c = body[0]; + + // JSON may be pretty-printed, starting with whitespace, or start with a UTF-8 byte + // order mark, which the JSON parser does not accept and is removed. + static const std::string bom("\xEF\xBB\xBF"); + size_t start = body.compare(0, bom.size(), bom) == 0 ? bom.size() : 0; + auto first = body.find_first_not_of(" \t\r\n", start); + auto c = first == std::string::npos ? '\0' : body[first]; DataItemPtr dataItem; DevicePtr device; @@ -152,7 +175,16 @@ namespace mtconnect::pipeline { if (c == '{' || c == '[') { entity::Properties props {entity->getProperties()}; + if (start > 0) + props["VALUE"] = body.substr(start); + + // A device named in the topic takes precedence over the default device device = m_defaultDevice; + if (auto topic = entity->maybeGet("topic")) + { + if (auto topicDevice = findTopicDevice(*topic)) + device = topicDevice; + } result = std::make_shared("JsonMessage", props); } else diff --git a/src/mtconnect/source/adapter/mqtt/mqtt_broker_adapter.cpp b/src/mtconnect/source/adapter/mqtt/mqtt_broker_adapter.cpp index 03c846b9..a534afc6 100644 --- a/src/mtconnect/source/adapter/mqtt/mqtt_broker_adapter.cpp +++ b/src/mtconnect/source/adapter/mqtt/mqtt_broker_adapter.cpp @@ -367,6 +367,38 @@ namespace mtconnect { std::list> m_connections; }; + /// @brief The address to listen on: MqttHost as an IP address, or else the address the host + /// name resolves to. IPv4 is preferred, on Windows localhost often resolves to ::1 + /// first and clients connecting to 127.0.0.1 would then be refused. + static std::optional resolveBindAddress(const string& host) + { + boost::system::error_code ec; + auto address = asio::ip::make_address(host, ec); + if (!ec) + return address; + + asio::io_context io; + asio::ip::tcp::resolver resolver(io); + auto results = resolver.resolve(host, "", ec); + if (ec || results.empty()) + { + LOG(error) << "MQTT Broker: Cannot resolve bind address '" << host + << "': " << (ec ? ec.message() : "no addresses"s); + return std::nullopt; + } + + std::optional result; + for (const auto& entry : results) + { + auto a = entry.endpoint().address(); + if (a.is_v4()) + return a; + if (!result) + result = a; + } + return result; + } + MqttBrokerAdapter::MqttBrokerAdapter(boost::asio::io_context& io, pipeline::PipelineContextPtr pipelineContext, const ConfigOptions& options, @@ -479,17 +511,13 @@ namespace mtconnect { m_pipeline.start(); - boost::system::error_code ec; - auto address = asio::ip::make_address(m_host, ec); - if (ec) - { - LOG(error) << "MQTT Broker: Invalid bind address '" << m_host << "': " << ec.message(); + auto address = resolveBindAddress(m_host); + if (!address) return false; - } try { - m_broker->start(address, static_cast(m_port)); + m_broker->start(*address, static_cast(m_port)); } catch (std::exception& e) { diff --git a/test_package/config_parser_test.cpp b/test_package/config_parser_test.cpp index 26c091cc..31d17fc3 100644 --- a/test_package/config_parser_test.cpp +++ b/test_package/config_parser_test.cpp @@ -149,3 +149,26 @@ TEST(ConfigParserTest, single_line_block) auto mother = tree.get("parents.mother"); ASSERT_EQ("father", mother); } + +// A # only starts a comment at the start of a value or after a blank, so MQTT +// topic filters such as Mill/# can be written without quotes. +TEST(ConfigParserTest, should_keep_hash_inside_a_value) +{ + string cfg = R"DOC( +Topics = Mill/# +Wild = Mill/+/# # the comment after it is still a comment +Hashed = a#b +block { + Topics = Lathe/# +} +inline { Topics = Robot/# } +)DOC"; + + auto tree = Parser::parse(cfg); + + ASSERT_EQ("Mill/#", tree.get("Topics")); + ASSERT_EQ("Mill/+/#", tree.get("Wild")); + ASSERT_EQ("a#b", tree.get("Hashed")); + ASSERT_EQ("Lathe/#", tree.get("block.Topics")); + ASSERT_EQ("Robot/#", tree.get("inline.Topics")); +} diff --git a/test_package/mqtt_broker_adapter_test.cpp b/test_package/mqtt_broker_adapter_test.cpp index 30c18e1b..59dfc5a1 100644 --- a/test_package/mqtt_broker_adapter_test.cpp +++ b/test_package/mqtt_broker_adapter_test.cpp @@ -168,6 +168,18 @@ TEST_F(MqttBrokerAdapterTest, should_ingest_published_observations) ASSERT_TRUE(waitForValue("p3", "204")); } +// MqttHost may be a host name; localhost must accept clients connecting to 127.0.0.1. +TEST_F(MqttBrokerAdapterTest, should_bind_to_a_host_name) +{ + startBroker({{MqttHost, "localhost"}}); + + auto client = MQTT_NS::make_async_client(ioc(), "127.0.0.1", m_adapter->getPort()); + ASSERT_EQ(MQTT_NS::connect_return_code::accepted, connect(client)); + + publish(client, "LinuxCNC/line", "205"); + ASSERT_TRUE(waitForValue("p3", "205")); +} + TEST_F(MqttBrokerAdapterTest, should_only_ingest_matching_topics) { startBroker({{Topics, "LinuxCNC/program"}}); diff --git a/test_package/topic_mapping_test.cpp b/test_package/topic_mapping_test.cpp index 9546e6da..eeea7811 100644 --- a/test_package/topic_mapping_test.cpp +++ b/test_package/topic_mapping_test.cpp @@ -176,7 +176,7 @@ TEST_F(TopicMappingTest, should_map_json_object_to_json_message) ASSERT_TRUE(res); ASSERT_TRUE(dynamic_pointer_cast(res)); ASSERT_FALSE(dynamic_pointer_cast(res)); - // The default device is attached; topic is not resolved for JSON messages + // The device is the one named in the topic, the default here; no data item is resolved ASSERT_EQ(dev, res->m_device.lock()); ASSERT_FALSE(res->m_dataItem); } @@ -281,3 +281,50 @@ TEST_F(TopicMappingTest, should_handle_message_without_topic) ASSERT_TRUE(dynamic_pointer_cast(res)); ASSERT_FALSE(res->m_dataItem); } + +/// @test pretty-printed JSON, starting with whitespace, is still a JsonMessage +TEST_F(TopicMappingTest, should_map_json_with_leading_whitespace_to_json_message) +{ + auto mapper = makeMapper(); + for (const auto& body : {"\n{\n \"a\": 1\n}\n"s, " [{\"a\":1}]"s, "\r\n\t{\"a\":1}"s}) + { + auto res = map(mapper, body, "topic"); + ASSERT_TRUE(res); + ASSERT_TRUE(dynamic_pointer_cast(res)) << body; + } +} + +/// @test a UTF-8 byte order mark is removed and the body mapped as JSON +TEST_F(TopicMappingTest, should_strip_byte_order_mark_from_json) +{ + auto mapper = makeMapper(); + auto res = map(mapper, "\xEF\xBB\xBF{\"a\":1}", "topic"); + ASSERT_TRUE(res); + ASSERT_TRUE(dynamic_pointer_cast(res)); + ASSERT_EQ("{\"a\":1}", res->getValue()); +} + +/// @test JSON goes to the device named in the topic, not the default device +TEST_F(TopicMappingTest, should_route_json_to_the_device_in_the_topic) +{ + makeDevice("Device", {{"id", "mill"s}, {"name", "mill"s}, {"uuid", "mill"s}}); + auto lathe = makeDevice("Device", {{"id", "lathe"s}, {"name", "lathe"s}, {"uuid", "lathe"s}}); + auto mapper = makeMapper("mill"); + + auto res = map(mapper, R"({"a":"ACTIVE"})", "lathe/data"); + ASSERT_TRUE(res); + ASSERT_TRUE(dynamic_pointer_cast(res)); + ASSERT_EQ(lathe, res->m_device.lock()); +} + +/// @test JSON whose topic names no device goes to the default device +TEST_F(TopicMappingTest, should_route_json_to_the_default_device_without_one_in_the_topic) +{ + auto mill = makeDevice("Device", {{"id", "mill"s}, {"name", "mill"s}, {"uuid", "mill"s}}); + makeDevice("Device", {{"id", "lathe"s}, {"name", "lathe"s}, {"uuid", "lathe"s}}); + auto mapper = makeMapper("mill"); + + auto res = map(mapper, R"({"a":"ACTIVE"})", "plant/data"); + ASSERT_TRUE(res); + ASSERT_EQ(mill, res->m_device.lock()); +} From 8d3f94d8d701d42fa0cb36296d401524300087e0 Mon Sep 17 00:00:00 2001 From: Will Sobel Date: Thu, 19 Nov 2026 15:08:00 +0100 Subject: [PATCH 2/2] Verstion 2.8.0.6 --- agent_version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent_version.txt b/agent_version.txt index 342f75f9..94a38ce1 100644 --- a/agent_version.txt +++ b/agent_version.txt @@ -1 +1 @@ -2.8.0.5 +2.8.0.6