feat(dnsdist): Add global ServerID

pdns/dnsdistdist/dnsdist-actions-definitions.yml

@@ -278,7 +278,11 @@ The function will be invoked in a per-thread Lua state, without access to the gl
     - name: "server_id"
       type: "String"
       default: ""
-      description: "Set the Server Identity field"
+      description: "Set the value for the Server Identity field"
+    - name: "use_server_id"
+      type: "bool"
+      default: "false"
+      description: "Use the value of general.server_id for the Server Identity field. Overrides the value set with server_id"
     - name: "ip_encrypt_key"
       type: "String"
       default: ""

pdns/dnsdistdist/dnsdist-actions-factory.cc

@@ -28,6 +28,7 @@
 #include "dnsdist-actions-factory.hh"
 
 #include "config.h"
+#include "dnsdist-configuration.hh"
 #include "dnsdist.hh"
 #include "dnsdist-async.hh"
 #include "dnsdist-dnsparser.hh"
@@ -1607,7 +1608,7 @@ class RemoteLogAction : public DNSAction, public boost::noncopyable
 public:
   // this action does not stop the processing
   RemoteLogAction(RemoteLogActionConfiguration& config) :
-    d_tagsToExport(std::move(config.tagsToExport)), d_metas(std::move(config.metas)), d_logger(config.logger), d_alterFunc(std::move(config.alterQueryFunc)), d_serverID(config.serverID), d_ipEncryptKey(config.ipEncryptKey), d_ipEncryptMethod(config.ipEncryptMethod)
+    d_tagsToExport(std::move(config.tagsToExport)), d_metas(std::move(config.metas)), d_logger(config.logger), d_alterFunc(std::move(config.alterQueryFunc)), d_serverID(config.serverID), d_ipEncryptKey(config.ipEncryptKey), d_ipEncryptMethod(config.ipEncryptMethod), d_useServerID(config.useServerID)
   {
     if (!d_ipEncryptKey.empty() && d_ipEncryptMethod == "ipcrypt-pfx") {
       d_ipcrypt2 = pdns::ipcrypt2::IPCrypt2(pdns::ipcrypt2::IPCryptMethod::pfx, d_ipEncryptKey);
@@ -1625,7 +1626,10 @@ class RemoteLogAction : public DNSAction, public boost::noncopyable
     }
 
     DNSDistProtoBufMessage message(*dnsquestion);
-    if (!d_serverID.empty()) {
+    if (d_useServerID) {
+      message.setServerIdentity(dnsdist::configuration::getCurrentRuntimeConfiguration().d_server_id);
+    }
+    else if (!d_serverID.empty()) {
       message.setServerIdentity(d_serverID);
     }
 
@@ -1675,6 +1679,7 @@ class RemoteLogAction : public DNSAction, public boost::noncopyable
   std::string d_ipEncryptKey;
   std::string d_ipEncryptMethod;
   std::optional<pdns::ipcrypt2::IPCrypt2> d_ipcrypt2{std::nullopt};
+  bool d_useServerID;
 };
 #endif /* DISABLE_PROTOBUF */
 
@@ -1873,7 +1878,7 @@ class RemoteLogResponseAction : public DNSResponseAction, public boost::noncopya
 public:
   // this action does not stop the processing
   RemoteLogResponseAction(RemoteLogActionConfiguration& config) :
-    d_tagsToExport(std::move(config.tagsToExport)), d_metas(std::move(config.metas)), d_logger(config.logger), d_alterFunc(std::move(config.alterResponseFunc)), d_serverID(config.serverID), d_ipEncryptKey(config.ipEncryptKey), d_ipEncryptMethod(config.ipEncryptMethod), d_exportExtendedErrorsToMeta(std::move(config.exportExtendedErrorsToMeta)), d_includeCNAME(config.includeCNAME), d_delay(config.delay)
+    d_tagsToExport(std::move(config.tagsToExport)), d_metas(std::move(config.metas)), d_logger(config.logger), d_alterFunc(std::move(config.alterResponseFunc)), d_serverID(config.serverID), d_ipEncryptKey(config.ipEncryptKey), d_ipEncryptMethod(config.ipEncryptMethod), d_exportExtendedErrorsToMeta(std::move(config.exportExtendedErrorsToMeta)), d_includeCNAME(config.includeCNAME), d_useServerID(config.useServerID), d_delay(config.delay)
   {
     if (!d_ipEncryptKey.empty() && d_ipEncryptMethod == "ipcrypt-pfx") {
       d_ipcrypt2 = pdns::ipcrypt2::IPCrypt2(pdns::ipcrypt2::IPCryptMethod::pfx, d_ipEncryptKey);
@@ -1890,7 +1895,10 @@ class RemoteLogResponseAction : public DNSResponseAction, public boost::noncopya
     }
 
     DNSDistProtoBufMessage message(*response, d_includeCNAME);
-    if (!d_serverID.empty()) {
+    if (d_useServerID) {
+      message.setServerIdentity(dnsdist::configuration::getCurrentRuntimeConfiguration().d_server_id);
+    }
+    else if (!d_serverID.empty()) {
       message.setServerIdentity(d_serverID);
     }
 
@@ -1953,6 +1961,7 @@ class RemoteLogResponseAction : public DNSResponseAction, public boost::noncopya
   std::optional<pdns::ipcrypt2::IPCrypt2> d_ipcrypt2{std::nullopt};
   std::optional<std::string> d_exportExtendedErrorsToMeta{std::nullopt};
   bool d_includeCNAME;
+  bool d_useServerID{false};
   bool d_delay{false};
 };
 

pdns/dnsdistdist/dnsdist-actions-factory.hh

@@ -117,6 +117,7 @@ struct RemoteLogActionConfiguration
   std::optional<std::string> exportExtendedErrorsToMeta{std::nullopt};
   bool includeCNAME{false};
   bool delay{false};
+  bool useServerID{false};
 };
 std::shared_ptr<DNSAction> getRemoteLogAction(RemoteLogActionConfiguration& config);
 std::shared_ptr<DNSResponseAction> getRemoteLogResponseAction(RemoteLogActionConfiguration& config);

pdns/dnsdistdist/dnsdist-carbon.cc

@@ -43,7 +43,7 @@ static bool doOneCarbonExport(const Carbon::Endpoint& endpoint)
 {
   const auto& server = endpoint.server;
   const std::string& namespace_name = endpoint.namespace_name;
-  const std::string& hostname = endpoint.ourname;
+  const std::string hostname = endpoint.getOurName();
   const std::string& instance_name = endpoint.instance_name;
 
   try {
@@ -330,7 +330,7 @@ static void carbonHandler(const Carbon::Endpoint& endpoint)
         if (consecutiveFailures < std::numeric_limits<decltype(consecutiveFailures)>::max()) {
           consecutiveFailures++;
         }
-        vinfolog("Run for %s - %s failed, next attempt in %d", endpoint.server.toStringWithPort(), endpoint.ourname, backOff);
+        vinfolog("Run for %s - %s failed, next attempt in %d", endpoint.server.toStringWithPort(), endpoint.getOurName(), backOff);
         std::this_thread::sleep_for(std::chrono::seconds(backOff));
       }
     } while (true);
@@ -343,19 +343,11 @@ static void carbonHandler(const Carbon::Endpoint& endpoint)
   }
 }
 
-Carbon::Endpoint Carbon::newEndpoint(const std::string& address, std::string ourName, uint64_t interval, const std::string& namespace_name, const std::string& instance_name)
+Carbon::Endpoint Carbon::newEndpoint(const std::string& address, const std::optional<std::string>& ourName, uint64_t interval, const std::string& namespace_name, const std::string& instance_name)
 {
-  if (ourName.empty()) {
-    try {
-      ourName = getCarbonHostName();
-    }
-    catch (const std::exception& exp) {
-      throw std::runtime_error(std::string("The 'ourname' setting in 'carbonServer()' has not been set and we are unable to determine the system's hostname: ") + exp.what());
-    }
-  }
   return Carbon::Endpoint{ComboAddress(address, 2003),
                           !namespace_name.empty() ? namespace_name : "dnsdist",
-                          std::move(ourName),
+                          ourName,
                           !instance_name.empty() ? instance_name : "main",
                           interval < std::numeric_limits<unsigned int>::max() ? static_cast<unsigned int>(interval) : 30};
 }
@@ -368,6 +360,16 @@ void Carbon::run(const std::vector<Carbon::Endpoint>& endpoints)
   }
 }
 
+const std::string Carbon::Endpoint::getOurName() const
+{
+  std::string ret = ourname.value_or("");
+  if (!ourname) {
+    ret = dnsdist::configuration::getCurrentRuntimeConfiguration().d_server_id;
+  }
+  std::replace(ret.begin(), ret.end(), '.', '_');
+  return ret;
+}
+
 }
 #endif /* DISABLE_CARBON */
 

pdns/dnsdistdist/dnsdist-carbon.hh

@@ -25,6 +25,7 @@
 
 #ifndef DISABLE_CARBON
 #include <string>
+#include <optional>
 #include "iputils.hh"
 
 namespace dnsdist
@@ -36,12 +37,14 @@ public:
   {
     ComboAddress server;
     std::string namespace_name;
-    std::string ourname;
+    std::optional<std::string> ourname; // When unset, we use the Runtime Config server_id
     std::string instance_name;
     unsigned int interval;
+
+    const std::string getOurName() const;
   };
 
-  static Endpoint newEndpoint(const std::string& address, std::string ourName, uint64_t interval, const std::string& namespace_name, const std::string& instance_name);
+  static Endpoint newEndpoint(const std::string& address, const std::optional<std::string>& ourName, uint64_t interval, const std::string& namespace_name, const std::string& instance_name);
   static void run(const std::vector<Endpoint>& endpoints);
 };
 

pdns/dnsdistdist/dnsdist-configuration-yaml.cc

@@ -20,6 +20,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  */
 #include <memory>
+#include <optional>
 #include <stdexcept>
 #include <vector>
 
@@ -890,6 +891,7 @@ static void loadWebServer(const dnsdist::rust::settings::WebserverConfiguration&
     }
 
     config.d_apiRequiresAuthentication = webConfig.api_requires_authentication;
+    config.d_prometheusAddInstanceLabel = webConfig.prometheus_add_instance;
     config.d_dashboardRequiresAuthentication = webConfig.dashboard_requires_authentication;
     config.d_statsRequireAuthentication = webConfig.stats_require_authentication;
     dnsdist::webserver::setMaxConcurrentConnections(webConfig.max_concurrent_connections);
@@ -1092,7 +1094,7 @@ static void handleCarbonConfiguration([[maybe_unused]] const ::rust::Vec<dnsdist
     dnsdist::configuration::updateRuntimeConfiguration([&carbonConfigs](dnsdist::configuration::RuntimeConfiguration& config) {
       for (const auto& carbonConfig : carbonConfigs) {
         auto newEndpoint = dnsdist::Carbon::newEndpoint(std::string(carbonConfig.address),
-                                                        std::string(carbonConfig.name),
+                                                        carbonConfig.name.empty() ? std::nullopt : std::optional<std::string>(carbonConfig.name),
                                                         carbonConfig.interval,
                                                         carbonConfig.name_space.empty() ? "dnsdist" : std::string(carbonConfig.name_space),
                                                         carbonConfig.instance.empty() ? "main" : std::string(carbonConfig.instance));
@@ -1229,6 +1231,12 @@ bool loadConfigurationFromFile(const std::string& fileName, [[maybe_unused]] boo
       });
     }
 
+    if (!globalConfig.general.server_id.empty()) {
+      dnsdist::configuration::updateRuntimeConfiguration([&server_id = globalConfig.general.server_id](dnsdist::configuration::RuntimeConfiguration& config) {
+        config.d_server_id = std::string(server_id);
+      });
+    }
+
     handlePacketCacheConfiguration(globalConfig.packet_caches);
 
     loadCustomPolicies(globalConfig.load_balancing_policies.custom_policies);
@@ -1768,6 +1776,7 @@ std::shared_ptr<DNSActionWrapper> getRemoteLogAction(const RemoteLogActionConfig
   if (dnsdist::configuration::yaml::getLuaFunctionFromConfiguration(alterFunc, config.alter_function_name, config.alter_function_code, config.alter_function_file, "remote log action")) {
     actionConfig.alterQueryFunc = std::move(alterFunc);
   }
+  actionConfig.useServerID = config.use_server_id;
   auto action = dnsdist::actions::getRemoteLogAction(actionConfig);
   return newDNSActionWrapper(std::move(action), config.name);
 #endif
@@ -1805,6 +1814,7 @@ std::shared_ptr<DNSResponseActionWrapper> getRemoteLogResponseAction(const Remot
     actionConfig.alterResponseFunc = std::move(alterFunc);
   }
   actionConfig.delay = config.delay;
+  actionConfig.useServerID = config.use_server_id;
   auto action = dnsdist::actions::getRemoteLogResponseAction(actionConfig);
   return newDNSResponseActionWrapper(std::move(action), config.name);
 #endif

pdns/dnsdistdist/dnsdist-configuration.hh

@@ -134,6 +134,7 @@ struct RuntimeConfiguration
   std::string d_consoleKey;
   std::string d_secPollSuffix{"secpoll.powerdns.com."};
   std::string d_apiConfigDirectory;
+  std::string d_server_id{getHostname().value_or("localhost")};
   uint64_t d_dynBlocksPurgeInterval{60};
   size_t d_maxTCPQueriesPerConn{0};
   size_t d_maxTCPConnectionDuration{0};
@@ -156,6 +157,7 @@ struct RuntimeConfiguration
   bool d_apiRequiresAuthentication{true};
   bool d_dashboardRequiresAuthentication{true};
   bool d_statsRequireAuthentication{true};
+  bool d_prometheusAddInstanceLabel{false};
   bool d_truncateTC{false};
   bool d_fixupCase{false};
   bool d_queryCountEnabled{false};

pdns/dnsdistdist/dnsdist-console-completion.cc

@@ -309,7 +309,7 @@ static std::vector<dnsdist::console::completion::ConsoleKeyword> s_consoleKeywor
   {"setVerbose", true, "bool", "set whether log messages at the verbose level will be logged"},
   {"setVerboseHealthChecks", true, "bool", "set whether health check errors will be logged"},
   {"setVerboseLogDestination", true, "destination file", "Set a destination file to write the 'verbose' log messages to, instead of sending them to syslog and/or the standard output"},
-  {"setWebserverConfig", true, "[{password=string, apiKey=string, customHeaders, statsRequireAuthentication}]", "Updates webserver configuration"},
+  {"setWebserverConfig", true, "[{password=string, apiKey=string, customHeaders, statsRequireAuthentication, prometheusAddInstanceLabel=bool}]", "Updates webserver configuration"},
   {"setWeightedBalancingFactor", true, "factor", "Set the balancing factor for bounded-load weighted policies (whashed, wrandom)"},
   {"setWHashedPerturbation", true, "value", "Set the hash perturbation value to be used in the whashed policy instead of a random one, allowing to have consistent whashed results on different instance"},
   {"show", true, "string", "outputs `string`"},
@@ -381,6 +381,8 @@ static std::vector<dnsdist::console::completion::ConsoleKeyword> s_consoleKeywor
   {"whashed", false, "", "Weighted hashed ('sticky') distribution over available servers, based on the server 'weight' parameter"},
   {"chashed", false, "", "Consistent hashed ('sticky') distribution over available servers, also based on the server 'weight' parameter"},
   {"wrandom", false, "", "Weighted random over available servers, based on the server 'weight' parameter"},
+  {"setServerID", true, "name", "Set the internal Server ID to this value"},
+  {"getServerID", true, "", "Get the internal Server ID as a string"},
 };
 
 #if defined(HAVE_LIBEDIT)

pdns/dnsdistdist/dnsdist-lua-actions.cc

@@ -28,6 +28,7 @@
 #include "dnsdist-protobuf.hh"
 #include "dnsdist-rule-chains.hh"
 #include "dnstap.hh"
+#include "dolog.hh"
 #include "remote_logger.hh"
 #include <memory>
 #include <optional>
@@ -267,7 +268,7 @@ void setupLuaActions(LuaContext& luaCtx)
   // Used for both RemoteLogAction and RemoteLogResponseAction
   static const std::array<std::string, 2> s_validIpEncryptMethods = {"legacy", "ipcrypt-pfx"};
 
-  luaCtx.writeFunction("RemoteLogAction", [](std::shared_ptr<RemoteLoggerInterface> logger, std::optional<dnsdist::actions::ProtobufAlterFunction> alterFunc, std::optional<LuaAssociativeTable<std::string>> vars, std::optional<LuaAssociativeTable<std::string>> metas) {
+  luaCtx.writeFunction("RemoteLogAction", [](std::shared_ptr<RemoteLoggerInterface> logger, std::optional<dnsdist::actions::ProtobufAlterFunction> alterFunc, std::optional<LuaAssociativeTable<boost::variant<std::string, bool>>> vars, std::optional<LuaAssociativeTable<std::string>> metas) {
     if (logger) {
       // avoids potentially-evaluated-expression warning with clang.
       RemoteLoggerInterface& remoteLoggerRef = *logger;
@@ -283,10 +284,25 @@ void setupLuaActions(LuaContext& luaCtx)
     if (alterFunc) {
       config.alterQueryFunc = std::move(*alterFunc);
     }
-    getOptionalValue<std::string>(vars, "serverID", config.serverID);
-    getOptionalValue<std::string>(vars, "ipEncryptKey", config.ipEncryptKey);
-    getOptionalValue<std::string>(vars, "ipEncryptMethod", config.ipEncryptMethod);
-    getOptionalValue<std::string>(vars, "exportTags", tags);
+    if (getOptionalValue<std::string>(vars, "serverID", config.serverID) < 0) {
+      throw std::runtime_error("serverID in RemoteLogAction is not a string");
+    }
+    if (getOptionalValue<std::string>(vars, "ipEncryptKey", config.ipEncryptKey) < 0) {
+      throw std::runtime_error("ipEncryptKey in RemoteLogAction is not a string");
+    }
+    if (getOptionalValue<std::string>(vars, "ipEncryptMethod", config.ipEncryptMethod) < 0) {
+      throw std::runtime_error("ipEncryptMethod in RemoteLogAction is not a string");
+    }
+    if (getOptionalValue<std::string>(vars, "exportTags", tags) < 0) {
+      throw std::runtime_error("exportTags in RemoteLogAction is not a string");
+    }
+    if (getOptionalValue<bool>(vars, "useServerID", config.useServerID) < 0) {
+      throw std::runtime_error("useServerID in RemoteLogAction is not a string");
+    }
+
+    if (config.useServerID && !config.serverID.empty()) {
+      warnlog("useServerID and serverID set in RemoteLogAction configuration. value for serverID will not be used");
+    }
 
     if (metas) {
       for (const auto& [key, value] : *metas) {
@@ -314,7 +330,7 @@ void setupLuaActions(LuaContext& luaCtx)
     return dnsdist::actions::getRemoteLogAction(config);
   });
 
-  luaCtx.writeFunction("RemoteLogResponseAction", [](std::shared_ptr<RemoteLoggerInterface> logger, std::optional<dnsdist::actions::ProtobufAlterResponseFunction> alterFunc, std::optional<bool> includeCNAME, std::optional<LuaAssociativeTable<std::string>> vars, std::optional<LuaAssociativeTable<std::string>> metas, std::optional<bool> delay) {
+  luaCtx.writeFunction("RemoteLogResponseAction", [](std::shared_ptr<RemoteLoggerInterface> logger, std::optional<dnsdist::actions::ProtobufAlterResponseFunction> alterFunc, std::optional<bool> includeCNAME, std::optional<LuaAssociativeTable<boost::variant<std::string, bool>>> vars, std::optional<LuaAssociativeTable<std::string>> metas, std::optional<bool> delay) {
     if (logger) {
       // avoids potentially-evaluated-expression warning with clang.
       RemoteLoggerInterface& remoteLoggerRef = *logger;
@@ -331,11 +347,28 @@ void setupLuaActions(LuaContext& luaCtx)
       config.alterResponseFunc = std::move(*alterFunc);
     }
     config.includeCNAME = includeCNAME ? *includeCNAME : false;
-    getOptionalValue<std::string>(vars, "serverID", config.serverID);
-    getOptionalValue<std::string>(vars, "ipEncryptKey", config.ipEncryptKey);
-    getOptionalValue<std::string>(vars, "ipEncryptMethod", config.ipEncryptMethod);
-    getOptionalValue<std::string>(vars, "exportTags", tags);
-    getOptionalValue<std::string>(vars, "exportExtendedErrorsToMeta", config.exportExtendedErrorsToMeta);
+    if (getOptionalValue<std::string>(vars, "serverID", config.serverID) < 0) {
+      throw std::runtime_error("serverID in RemoteLogResponseAction is not a string");
+    }
+    if (getOptionalValue<std::string>(vars, "ipEncryptKey", config.ipEncryptKey) < 0) {
+      throw std::runtime_error("ipEncryptKey in RemoteLogResponseAction is not a string");
+    }
+    if (getOptionalValue<std::string>(vars, "ipEncryptMethod", config.ipEncryptMethod) < 0) {
+      throw std::runtime_error("ipEncryptMethod in RemoteLogResponseAction is not a string");
+    }
+    if (getOptionalValue<std::string>(vars, "exportTags", tags) < 0) {
+      throw std::runtime_error("exportTags in RemoteLogResponseAction is not a string");
+    }
+    if (getOptionalValue<std::string>(vars, "exportExtendedErrorsToMeta", config.exportExtendedErrorsToMeta) < 0) {
+      throw std::runtime_error("exportExtendedErrorsToMeta in RemoteLogResponseAction is not a string");
+    }
+    if (getOptionalValue<bool>(vars, "useServerID", config.useServerID) < 0) {
+      throw std::runtime_error("useServerID in RemoteLogResponseAction is not a string");
+    }
+
+    if (config.useServerID && !config.serverID.empty()) {
+      warnlog("useServerID and serverID set in RemoteLogResponseAction configuration. value for serverID will not be used");
+    }
 
     if (metas) {
       for (const auto& [key, value] : *metas) {

pdns/dnsdistdist/dnsdist-lua-bindings.cc

@@ -21,6 +21,7 @@
  */
 #include "bpf-filter.hh"
 #include "config.h"
+#include "dnsdist-configuration.hh"
 #include "dnsdist.hh"
 #include "dnsdist-async.hh"
 #include "dnsdist-dynblocks.hh"
@@ -1102,6 +1103,10 @@ void setupLuaBindings(LuaContext& luaCtx, bool client, bool configCheck)
     newThread.detach();
   });
 
+  luaCtx.writeFunction("getServerID", []() -> std::string {
+    return dnsdist::configuration::getCurrentRuntimeConfiguration().d_server_id;
+  });
+
   luaCtx.writeFunction("refreshRuntimeConfiguration", []() {
     dnsdist::configuration::refreshLocalRuntimeConfiguration();
   });