metrics: add experimental internal metrics (#85)

Issue: SILKIT-1379 Subject: Creation, collection, sending, receiving, and storing various metrics as JSON. The metrics can be used to get insights into internal behavior of individual SIL Kit participants or simulations. Currently available metrics: COUNTER: Collects single unsigned integer value. STATISTICS: Collects single double values and accumulates minimum, maximum, exponential incremental mean, and standard deviation. STRING_LIST: Collects a list of strings. If metrics are sent to remote participants, the participant that collects these, should be started first. The registry also supports collection of remote metrics, which is preferable, since it must be started first anyway. Collection and storage of the metrics is configured via the participant configuration file (or the registry configuration file). Metrics are only sent if they have changed.
vectorgrp · Aug 1, 2024 · 5b42508 · 5b42508
1 parent 90755be
commit 5b42508
Show file tree

Hide file tree

Showing 84 changed files with 3,382 additions and 140 deletions.
diff --git a/SilKit/source/CMakeLists.txt b/SilKit/source/CMakeLists.txt
@@ -183,7  183,9 @@ list(APPEND SilKitImplObjectLibraries
     O_SilKit_Services_Orchestration
     O_SilKit_Services_PubSub
     O_SilKit_Services_Rpc
     O_SilKit_Services_Metrics
     O_SilKit_Tracing
     O_SilKit_Util
     O_SilKit_Util_FileHelpers
     O_SilKit_Util_StringHelpers
     O_SilKit_Util_Filesystem

diff --git a/SilKit/source/config/Configuration.hpp b/SilKit/source/config/Configuration.hpp
@@ -168,6  168,7 @@ struct SimulatedNetwork
 inline bool operator==(const Sink& lhs, const Sink& rhs);
 inline bool operator<(const Sink& lhs, const Sink& rhs);
 inline bool operator>(const Sink& lhs, const Sink& rhs);
 
 inline bool operator==(const Logging& lhs, const Logging& rhs);
 inline bool operator==(const TraceSink& lhs, const TraceSink& rhs);
 inline bool operator==(const TraceSource& lhs, const TraceSource& rhs);

diff --git a/SilKit/source/config/ParticipantConfiguration.hpp b/SilKit/source/config/ParticipantConfiguration.hpp
@@ -229,6  229,30 @@ struct Tracing
     std::vector<TraceSource> traceSources;
 };
 
 // ================================================================================
 //  Metrics service
 // ================================================================================
 
 struct MetricsSink
 {
     enum class Type
     {
         Undefined,
         JsonFile,
         Remote,
     };
 
     Type type{Type::Undefined};
     std::string name;
 };
 
 //! \brief Metrics configuration
 struct Metrics
 {
     std::vector<MetricsSink> sinks;
     bool collectFromRemote{false};
 };
 
 // ================================================================================
 //  Extensions
 // ================================================================================
@@ -283,6  307,7 @@ struct TimeSynchronization
 struct Experimental
 {
     TimeSynchronization timeSynchronization;
     Metrics metrics;
 };
 
 // ================================================================================
@@ -336,12  361,17 @@ bool operator==(const RpcServer& lhs, const RpcServer& rhs);
 bool operator==(const RpcClient& lhs, const RpcClient& rhs);
 bool operator==(const HealthCheck& lhs, const HealthCheck& rhs);
 bool operator==(const Tracing& lhs, const Tracing& rhs);
 bool operator==(const MetricsSink& lhs, const MetricsSink& rhs);
 bool operator==(const Metrics& lhs, const Metrics& rhs);
 bool operator==(const Extensions& lhs, const Extensions& rhs);
 bool operator==(const Middleware& lhs, const Middleware& rhs);
 bool operator==(const ParticipantConfiguration& lhs, const ParticipantConfiguration& rhs);
 bool operator==(const TimeSynchronization& lhs, const TimeSynchronization& rhs);
 bool operator==(const Experimental& lhs, const Experimental& rhs);
 
 bool operator<(const MetricsSink& lhs, const MetricsSink& rhs);
 bool operator>(const MetricsSink& lhs, const MetricsSink& rhs);
 
 } // namespace v1
 } // namespace Config
 } // namespace SilKit
diff --git a/SilKit/source/config/ParticipantConfigurationFromXImpl.cpp b/SilKit/source/config/ParticipantConfigurationFromXImpl.cpp
@@ -76,9  76,18 @@ struct TimeSynchronizationCache
     SilKit::Util::Optional<double> animationFactor;
 };
 
 struct MetricsCache
 {
     SilKit::Util::Optional<bool> collectFromRemote;
     std::set<MetricsSink> jsonFileSinks;
     std::set<std::string> fileNames;
     SilKit::Util::Optional<MetricsSink> remoteSink;
 };
 
 struct ExperimentalCache
 {
     TimeSynchronizationCache timeSynchronizationCache;
     MetricsCache metricsCache;
 };
 
 struct ConfigIncludeData
@@ -368,12  377,66 @@ void CacheTimeSynchronization(const YAML::Node& root, TimeSynchronizationCache&
     PopulateCacheField(root, "TimeSynchronization", "AnimationFactor", cache.animationFactor);
 }
 
 void CacheMetrics(const YAML::Node& root, MetricsCache& cache)
 {
     PopulateCacheField(root, "Metrics", "CollectFromRemote", cache.collectFromRemote);
 
     if (root["Sinks"])
     {
         for (const auto& sinkNode : root["Sinks"])
         {
             auto sink = parse_as<MetricsSink>(sinkNode);
             if (sink.type == MetricsSink::Type::JsonFile)
             {
                 if (cache.fileNames.count(sink.name) == 0)
                 {
                     cache.jsonFileSinks.insert(sink);
                     cache.fileNames.insert(sink.name);
                 }
                 else
                 {
                     std::stringstream error_msg;
                     error_msg << "JSON file metrics sink " << sink.name << " already exists!";
                     throw SilKit::ConfigurationError(error_msg.str());
                 }
             }
             else if (sink.type == MetricsSink::Type::Remote)
             {
                 if (!cache.remoteSink.has_value())
                 {
                     // Replace the already included sink with this one
                     // since we have not set it yet
                     cache.remoteSink = sink;
                 }
                 else
                 {
                     std::stringstream error_msg;
                     error_msg << "Remote metrics sink already exists!";
                     throw SilKit::ConfigurationError(error_msg.str());
                 }
             }
             else
             {
                 std::stringstream error_msg;
                 error_msg << "Invalid MetricsSink::Type("
                           << static_cast<std::underlying_type_t<MetricsSink::Type>>(sink.type) << ")";
                 throw SilKit::ConfigurationError(error_msg.str());
             }
         }
     }
 }
 
 void CacheExperimental(const YAML::Node& root, ExperimentalCache& cache)
 {
     if (root["TimeSynchronization"])
     {
         CacheTimeSynchronization(root["TimeSynchronization"], cache.timeSynchronizationCache);
     }
 
     if (root["Metrics"])
     {
         CacheMetrics(root["Metrics"], cache.metricsCache);
     }
 }
 
 void PopulateCaches(const YAML::Node& config, ConfigIncludeData& configIncludeData)
@@ -491,9  554,27 @@ void MergeTimeSynchronizationCache(const TimeSynchronizationCache& cache, TimeSy
     MergeCacheField(cache.animationFactor, timeSynchronization.animationFactor);
 }
 
 void MergeMetricsCache(const MetricsCache& cache, Metrics& metrics)
 {
     MergeCacheField(cache.collectFromRemote, metrics.collectFromRemote);
     MergeCacheSet(cache.jsonFileSinks, metrics.sinks);
 
     if (cache.remoteSink.has_value() && metrics.collectFromRemote)
     {
         throw SilKit::ConfigurationError{
             "Cannot have 'Remote' metrics sink together with 'CollectFromRemote' being true"};
     }
 
     if (cache.remoteSink.has_value())
     {
         metrics.sinks.push_back(cache.remoteSink.value());
     }
 }
 
 void MergeExperimentalCache(const ExperimentalCache& cache, Experimental& experimental)
 {
     MergeTimeSynchronizationCache(cache.timeSynchronizationCache, experimental.timeSynchronization);
     MergeMetricsCache(cache.metricsCache, experimental.metrics);
 }
 
 
@@ -675,6  756,16 @@ bool operator==(const Tracing& lhs, const Tracing& rhs)
     return lhs.traceSinks == rhs.traceSinks && lhs.traceSources == rhs.traceSources;
 }
 
 bool operator==(const MetricsSink& lhs, const MetricsSink& rhs)
 {
     return lhs.type == rhs.type && lhs.name == rhs.name;
 }
 
 bool operator==(const Metrics& lhs, const Metrics& rhs)
 {
     return lhs.sinks == rhs.sinks && lhs.collectFromRemote == rhs.collectFromRemote;
 }
 
 bool operator==(const Extensions& lhs, const Extensions& rhs)
 {
     return lhs.searchPathHints == rhs.searchPathHints;
@@ -696,7  787,7 @@ bool operator==(const ParticipantConfiguration& lhs, const ParticipantConfigurat
            && lhs.flexrayControllers == rhs.flexrayControllers && lhs.dataPublishers == rhs.dataPublishers
            && lhs.dataSubscribers == rhs.dataSubscribers && lhs.rpcClients == rhs.rpcClients
            && lhs.rpcServers == rhs.rpcServers && lhs.logging == rhs.logging && lhs.healthCheck == rhs.healthCheck
-           && lhs.tracing == rhs.tracing && lhs.extensions == rhs.extensions;
            && lhs.tracing == rhs.tracing && lhs.extensions == rhs.extensions && lhs.experimental == rhs.experimental;
 }
 
 bool operator==(const TimeSynchronization& lhs, const TimeSynchronization& rhs)
@@ -706,7  797,17 @@ bool operator==(const TimeSynchronization& lhs, const TimeSynchronization& rhs)
 
 bool operator==(const Experimental& lhs, const Experimental& rhs)
 {
-    return lhs.timeSynchronization == rhs.timeSynchronization;
     return lhs.timeSynchronization == rhs.timeSynchronization && lhs.metrics == rhs.metrics;
 }
 
 bool operator<(const MetricsSink& lhs, const MetricsSink& rhs)
 {
     return std::make_tuple(lhs.type, lhs.name) < std::make_tuple(rhs.type, rhs.name);
 }
 
 bool operator>(const MetricsSink& lhs, const MetricsSink& rhs)
 {
     return !(lhs < rhs);
 }
 
 } // namespace v1

diff --git a/SilKit/source/config/ParticipantConfiguration_Full.json b/SilKit/source/config/ParticipantConfiguration_Full.json
@@ -233,6  233,19 @@
   "Experimental": {
     "TimeSynchronization": {
       "AnimationFactor": 1.5
     },
     "Metrics": {
       "CollectFromRemote": false,
       "Sinks": [
         {
           "Type": "JsonFile",
           "Name": "MyJsonMetrics"
         },
         {
           "Type": "Remote",
           "Name": "MyRemoteMetricsSink"
         }
       ]
     }
   }
 }
diff --git a/SilKit/source/config/ParticipantConfiguration_Full.yaml b/SilKit/source/config/ParticipantConfiguration_Full.yaml
@@ -167,3  167,13 @@ Middleware:
   TcpReceiveBufferSize: 3456
   RegistryAsFallbackProxy: false
   ConnectTimeoutSeconds: 1.234
 Experimental:
   TimeSynchronization:
     AnimationFactor: 1.5
   Metrics:
     CollectFromRemote: false
     Sinks:
       - Type: JsonFile
         Name: MyJsonMetrics
       - Type: Remote
         Name: MyRemoteMetricsSink
diff --git a/SilKit/source/config/Test_ParticipantConfiguration.cpp b/SilKit/source/config/Test_ParticipantConfiguration.cpp
@@ -255,4  255,17 @@ TEST_F(Test_ParticipantConfiguration, full_configuration_file_json_yaml_equal)
     ASSERT_TRUE(participantConfigJson == participantConfigYaml);
 }
 
 TEST_F(Test_ParticipantConfiguration, remote_metric_sink_collect_from_remote_fails)
 {
     constexpr auto configurationString = R"(
 Metrics:
   CollectFromRemote: true
   Sinks:
     - Type: Remote
 )";
 
     ASSERT_THROW(SilKit::Config::ParticipantConfigurationFromStringImpl(configurationString),
                  SilKit::ConfigurationError);
 }
 
 } // anonymous namespace
diff --git a/SilKit/source/config/YamlConversion.cpp b/SilKit/source/config/YamlConversion.cpp
@@ -964,6  964,94 @@ bool Converter::decode(const Node& node, TraceSource::Type& obj)
     return true;
 }
 
 // Metrics
 
 template <>
 Node Converter::encode(const MetricsSink::Type& obj)
 {
     Node node;
     switch (obj)
     {
     case MetricsSink::Type::Undefined:
         node = "Undefined";
         break;
     case MetricsSink::Type::JsonFile:
         node = "JsonFile";
         break;
     case MetricsSink::Type::Remote:
         node = "Remote";
         break;
     default:
         throw ConfigurationError{"Unknown MetricsSink Type"};
     }
     return node;
 }
 
 template <>
 bool Converter::decode(const Node& node, MetricsSink::Type& obj)
 {
     auto&& str = parse_as<std::string>(node);
     if (str == "Undefined" || str == "")
     {
         obj = MetricsSink::Type::Undefined;
     }
     else if (str == "JsonFile")
     {
         obj = MetricsSink::Type::JsonFile;
     }
     else if (str == "Remote")
     {
         obj = MetricsSink::Type::Remote;
     }
     else
     {
         throw ConversionError{node, "Unknown MetricsSink::Type: "   str   "."};
     }
     return true;
 }
 
 template <>
 Node Converter::encode(const MetricsSink& obj)
 {
     Node node;
     node["Type"] = obj.type;
     if (!obj.name.empty())
     {
         node["Name"] = obj.name;
     }
     return node;
 }
 
 template <>
 bool Converter::decode(const Node& node, MetricsSink& obj)
 {
     obj.type = parse_as<decltype(obj.type)>(node["Type"]);
     optional_decode(obj.name, node, "Name");
     return true;
 }
 
 template <>
 Node Converter::encode(const Metrics& obj)
 {
     Node node;
     optional_encode(obj.sinks, node, "Sinks");
     if (obj.collectFromRemote)
     {
         node["CollectFromRemote"] = obj.collectFromRemote;
     }
     return node;
 }
 
 template <>
 bool Converter::decode(const Node& node, Metrics& obj)
 {
     optional_decode(obj.sinks, node, "Sinks");
     optional_decode(obj.collectFromRemote, node, "CollectFromRemote");
     return true;
 }
 
 // Extensions
 
 template <>
 Node Converter::encode(const Extensions& obj)
 {
@@ -1038,12  1126,14 @@ Node Converter::encode(const Experimental& obj)
 
     Node node;
     non_default_encode(obj.timeSynchronization, node, "TimeSynchronization", defaultObj.timeSynchronization);
     non_default_encode(obj.metrics, node, "Metrics", defaultObj.metrics);
     return node;
 }
 template <>
 bool Converter::decode(const Node& node, Experimental& obj)
 {
     optional_decode(obj.timeSynchronization, node, "TimeSynchronization");
     optional_decode(obj.metrics, node, "Metrics");
     return true;
 }