Program Listing for File DataLogger.h#
↰ Return to documentation for file (include/Karana/KUtils/DataLogger.h)
/*
* Copyright (c) 2024-2026 Karana Dynamics Pty Ltd. All rights reserved.
*
* NOTICE TO USER:
*
* This source code and/or documentation (the "Licensed Materials") is
* the confidential and proprietary information of Karana Dynamics Inc.
* Use of these Licensed Materials is governed by the terms and conditions
* of a separate software license agreement between Karana Dynamics and the
* Licensee ("License Agreement"). Unless expressly permitted under that
* agreement, any reproduction, modification, distribution, or disclosure
* of the Licensed Materials, in whole or in part, to any third party
* without the prior written consent of Karana Dynamics is strictly prohibited.
*
* THE LICENSED MATERIALS ARE PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND.
* KARANA DYNAMICS DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING
* BUT NOT LIMITED TO WARRANTIES OF MERCHANTABILITY, NON-INFRINGEMENT, AND
* FITNESS FOR A PARTICULAR PURPOSE.
*
* IN NO EVENT SHALL KARANA DYNAMICS BE LIABLE FOR ANY DAMAGES WHATSOEVER,
* INCLUDING BUT NOT LIMITED TO LOSS OF PROFITS, DATA, OR USE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGES, WHETHER IN CONTRACT, TORT,
* OR OTHERWISE ARISING OUT OF OR IN CONNECTION WITH THE LICENSED MATERIALS.
*
* U.S. Government End Users: The Licensed Materials are a "commercial item"
* as defined at 48 C.F.R. 2.101, and are provided to the U.S. Government
* only as a commercial end item under the terms of this license.
*
* Any use of the Licensed Materials in individual or commercial software must
* include, in the user documentation and internal source code comments,
* this Notice, Disclaimer, and U.S. Government Use Provision.
*/
/**
* @file
* @brief Contains the declarations for the DataLogger class.
*/
// Turn formatting off to avoid changing the order of these includes.
// clang-format off
#include "hdf5.h"
// clang-format on
#include "H5PTpublic.h"
#include "Karana/KCore/LockingBase.h"
#include "Karana/KCore/UsageTrackingMap.h"
#include "Karana/Math/Defs.h"
#include <Eigen/Core>
#include <cstring>
#include <format>
#include <functional>
#include <typeindex>
namespace Karana::KUtils {
namespace kc = Karana::Core;
namespace km = Karana::Math;
/// Primary template (default case: false)
template <typename T>
struct is_eigen_type : std::false_type {}; // NOLINT(readability-identifier-naming)
/// Specialization for Eigen::Matrix
template <int Rows, int Cols>
struct is_eigen_type<Eigen::Matrix<double, Rows, Cols, Eigen::RowMajor>> : std::true_type {};
/// Specialization for Eigen::Vector
template <int Rows> struct is_eigen_type<Eigen::Matrix<double, Rows, 1>> : std::true_type {};
/// Helper variable template to get the value
template <typename T> inline constexpr bool is_eigen_type_v = is_eigen_type<T>::value;
/// Primary template (default case: false)
template <typename T>
struct is_eigen_vector : std::false_type {}; // NOLINT(readability-identifier-naming)
/// Specialization for Eigen::Vector
template <int Rows> struct is_eigen_vector<Eigen::Matrix<double, Rows, 1>> : std::true_type {};
/// Helper variable template to get the value
template <typename T> inline constexpr bool is_eigen_vector_v = is_eigen_vector<T>::value;
/// Default template for extracting matrix dimensions (defaults to -1 x -1)
template <typename T>
struct extract_matrix_dimensions { // NOLINT(readability-identifier-naming)
/// Number of rows in the matrix
static constexpr int rows = -1;
/// Number of cols in the matrix
static constexpr int cols = -1;
};
/// Template to extract matrix dimensions from a vector.
template <int _rows> struct extract_matrix_dimensions<Eigen::Matrix<double, _rows, 1>> {
/// Number of rows in the matrix
static constexpr int rows = _rows;
/// Number of cols in the matrix
static constexpr int cols = 1;
};
/// Template to extract matrix dimensions from a RowMajor matrix.
template <int _rows, int _cols>
struct extract_matrix_dimensions<Eigen::Matrix<double, _rows, _cols, Eigen::RowMajor>> {
/// Number of rows in the matrix
static constexpr int rows = _rows;
/// Number of cols in the matrix
static constexpr int cols = _cols;
};
/// Get the rows value from extract_matrix_dimensions
template <typename T> constexpr int extract_rows_v = extract_matrix_dimensions<T>::rows;
/// Get the cols value from extract_matrix_dimensions
template <typename T> constexpr int extract_cols_v = extract_matrix_dimensions<T>::cols;
/**
* @brief Helper to extract rows from a Type T.
*
* @tparam T The object to extract rows from.
*/
template <typename T> struct extract_rows { // NOLINT(readability-identifier-naming)
/// Number of rows in the matrix
static constexpr int value = -1; // Default case (should never match)
};
/// Specialization for fixed-size Eigen::Matrix<double, rows, 1>
template <int rows> struct extract_rows<Eigen::Matrix<double, rows, 1>> {
/// Number of rows in the matrix
static constexpr int value = rows;
};
/// Helper variable template to get the value
template <typename T> constexpr int extract_vector_rows_v = extract_rows<T>::value;
// Forward declare H5Writer
class H5Writer;
/**
* @class PacketTableConfig
* @brief Define the columns and associated update functions for a PacketTable.
* The PacketTable is the table that stores data in the H5 log file.
*/
class PacketTableConfig : public kc::LockingBase {
friend H5Writer;
public:
/**
* @brief PacketTableConfig constructor. The constructor is not meant to be called directly.
* Please use the create(...) method instead to create an instance.
*
* @param name Name of the table. A / will be interpreted as a new group.
* Groups will be added as needed.
*/
PacketTableConfig(std::string_view name);
/**
* @brief Create a packet table config.
*
* @param name Name of the table. A / will be interpreted as a new group.
* Groups will be added as needed.
* @return A ks_ptr to the newly created instance of PacketTableConfig.
*/
static kc::ks_ptr<PacketTableConfig> create(std::string_view name);
/** @brief Destroy packet table config. */
~PacketTableConfig();
/**
* @brief Add data to the table via function.
*
* @param name The name of the column in the table.
* @param f The function that produces the data (dynamic vector). This vector must always
* be the same size.
* @param vector_size The size of the vector that the function produces. This vector must
* always be the same size.
* @param as_scalars Rather than using one column for the vector, use multiple. This logs
* each vector component as a scalar. The column names will be appended with a _X where "X"
* is an integer that corresponds with the index in the vector.
*/
template <class T>
requires std::same_as<std::invoke_result_t<T>, km::Vec>
void addData(std::string_view name, const T &f, int vector_size, bool as_scalars = false) {
// Ensure this is not healthy.
ensureNotHealthy();
if (as_scalars) {
for (int k = 0; k < vector_size; k++) {
_h5_types.push_back(H5T_NATIVE_DOUBLE);
_names.emplace_back(std::format("{}_{}", name, k));
_offsets.push_back(_offset + k * sizeof(double));
}
} else {
hsize_t dims[1]{hsize_t(vector_size)};
_h5_types.push_back(H5Tarray_create2(H5T_NATIVE_DOUBLE, 1, dims));
_names.emplace_back(name);
_offsets.push_back(_offset);
}
size_t size = vector_size * sizeof(double);
_funcs.push_back([f, offset = _offset, size](char *buf) {
km::Vec val = f();
void *dark = val.data();
std::memcpy(buf + offset, dark, size);
});
// Increment the offset for the next function.
_offset += size;
}
/**
* @brief Add data to the table via function.
* @param name The name of the column in the table.
* @param f The function that produces the data (dynamic matrix). This matrix must always
* be the same size.
* @param matrix_rows The number of rows in the matrix that the function produces. This
* matrix must always be the same size.
* @param matrix_cols The number of columns in the matrix that the function produces. This
* matrix must always be the same size.
* @param as_scalars Rather than using one column for the matrix, use multiple. This logs
* each matrix component as a scalar. The column names will be appended with a _X_Y where
* "X" and "Y" are integers that correspond with the row and index of the element in the
* matrix.
*/
template <class T>
requires std::same_as<std::invoke_result_t<T>, km::Mat>
void addData(std::string_view name,
const T &f,
int matrix_rows,
int matrix_cols,
bool as_scalars = false) {
// Ensure this is not healthy.
ensureNotHealthy();
if (as_scalars) {
for (int j = 0; j < matrix_rows; j++) {
for (int k = 0; k < matrix_cols; k++) {
_h5_types.push_back(H5T_NATIVE_DOUBLE);
_names.emplace_back(std::format("{}_{}_{}", name, j, k));
_offsets.push_back(_offset + (j * matrix_cols + k) * sizeof(double));
}
}
} else {
hsize_t dims[2]{hsize_t(matrix_rows), hsize_t(matrix_cols)};
_h5_types.push_back(H5Tarray_create2(H5T_NATIVE_DOUBLE, 2, dims));
_names.emplace_back(name);
_offsets.push_back(_offset);
}
size_t size = matrix_rows * matrix_cols * sizeof(double);
_funcs.push_back([f, offset = _offset, size](char *buf) {
km::Mat val = f();
void *dark = val.data();
std::memcpy(buf + offset, dark, size);
});
// Increment the offset for the next function.
_offset += size;
}
/**
* @brief Add data to the table via function.
* @param name The name of the column in the table.
* @param f The function that produces the data.
* @param as_scalars Rather than using one column for the vector, use multiple. This logs
* each vector component as a scalar. The column names will be appended with a _X where "X"
* is an integer that corresponds with the index in the vector.
*/
template <class T, class R = typename std::invoke_result<T>::type>
requires(is_eigen_type_v<R> and is_eigen_vector_v<R>)
void addData(std::string_view name, const T &f, bool as_scalars = false) {
// Ensure this is not healthy.
ensureNotHealthy();
constexpr int rows = extract_rows_v<R>;
static_assert(rows != Eigen::Dynamic,
"Vector must have a fixed size (no dynamic rows). For a dynamic vector, "
"use the addData signature that takes in an integer for the number rows "
"as an extra argument.");
if (as_scalars) {
for (int k = 0; k < rows; k++) {
_h5_types.push_back(H5T_NATIVE_DOUBLE);
_names.emplace_back(std::format("{}_{}", name, k));
_offsets.push_back(_offset + k * sizeof(double));
}
} else {
hsize_t dims[1]{hsize_t(rows)};
_h5_types.push_back(H5Tarray_create2(H5T_NATIVE_DOUBLE, 1, dims));
_names.emplace_back(name);
_offsets.push_back(_offset);
}
// Create a copy to store in the new function
size_t size = rows * sizeof(double);
_funcs.push_back([f, offset = _offset, size](char *buf) {
Eigen::Matrix<double, rows, 1> val = f();
void *dark = val.data();
std::memcpy(buf + offset, dark, size);
});
// Increment the offset for the next function.
_offset += size;
}
/**
* @brief Add data to the table via function.
* @param name The name of the column in the table.
* @param f The function that produces the data.
* @param as_scalars Rather than using one column for the matrix, use multiple. This logs
* each matrix component as a scalar. The column names will be appended with a _X_Y where
* "X" and "Y" are integers that correspond with the row and index of the element in the
* matrix.
*/
template <class T, class R = typename std::invoke_result<T>::type>
requires(is_eigen_type_v<R> and not is_eigen_vector_v<R>)
void addData(std::string_view name, const T &f, bool as_scalars = false) {
// Ensure this is not healthy.
ensureNotHealthy();
constexpr int rows = extract_rows_v<R>;
constexpr int cols = extract_rows_v<R>;
static_assert(rows != Eigen::Dynamic and cols != Eigen::Dynamic,
"Matrix must have a fixed size (no dynamic rows or columns). For a "
"dynamic matrix, use the addData signature that takes in integers for "
"the rows and columns of a dynamic matrix as extra arguments.");
if (as_scalars) {
for (int j = 0; j < rows; j++) {
for (int k = 0; k < cols; k++) {
_h5_types.push_back(H5T_NATIVE_DOUBLE);
_names.emplace_back(std::format("{}_{}_{}", name, j, k));
_offsets.push_back(_offset + (j * cols + k) * sizeof(double));
}
}
} else {
hsize_t dims[2]{hsize_t(rows), hsize_t(cols)};
_h5_types.push_back(H5Tarray_create2(H5T_NATIVE_DOUBLE, 2, dims));
_names.emplace_back(name);
_offsets.push_back(_offset);
}
size_t size = rows * cols * sizeof(double);
_funcs.push_back([f, offset = _offset, size](char *buf) {
Eigen::Matrix<double, rows, cols, Eigen::RowMajor> val = f();
void *dark = val.data();
std::memcpy(buf + offset, dark, size);
});
// Increment the offset for the next function.
_offset += size;
}
/**
* @brief Add data to the table via function.
* @param name The name of the column in the table.
* @param f The function that produces the data.
*/
template <class T, class R = typename std::invoke_result<T>::type>
requires(not is_eigen_type_v<R>)
void addData(std::string_view name, const T &f) {
// Ensure this is not healthy.
ensureNotHealthy();
auto it = _h5TypeMap.find(typeid(R));
if (it == _h5TypeMap.end()) {
throw std::runtime_error(std::format("Cannot store H5 data for \"{}\".", name));
}
_h5_types.push_back(it->second);
_names.emplace_back(name);
// Create a copy to store in the new function
size_t offset = _offset;
size_t size = sizeof(R);
_offsets.push_back(offset);
_funcs.push_back([f, offset, size](char *buf) {
R val = f();
void *dark = &val;
std::memcpy(buf + offset, dark, size);
});
// Increment the offset for the next function.
_offset += size;
}
/**
* @brief Fill the buffer by calling all the functions.
*/
void fillBuf();
protected:
/**
* @brief Create the data type and buffer for the data. This can only be done once.
*/
void _makeHealthy();
/**
* @brief This should never be called. If it is, then there is an issue.
*/
void _makeNotHealthy();
private:
/// The name of the group for this PacketTableConfig
std::string _group_name;
// The data type of the packet table.
hid_t _data_type;
// Buffer to hold the table data
char *_buf = nullptr;
/** This vector holds the functions that add memory to the void* data buffer
* used by the packet table. */
std::vector<std::function<void(char *buf)>> _funcs;
/// The names of the columns in the packet table.
std::vector<std::string> _names;
/// The h5 types of the columns in the table.
std::vector<hid_t> _h5_types;
/// The offsets for each type.
std::vector<size_t> _offsets;
/// Memory offset used by the next function.
size_t _offset = 0;
/// Maps C++ types to H5 types.
static const std::unordered_map<std::type_index, hid_t> _h5TypeMap;
};
/**
* @class H5Writer
* @brief Logs data to an HDF5 file. The data to be logged is configured
* via `PacketTableConfig`s.
*/
class H5Writer : public kc::Base {
public:
/**
* @brief H5Writer constructor. The constructor is not meant to be called directly.
* Please use the create(...) method instead to create an instance.
*
* @param filename The name of the h5 file to create.
*/
H5Writer(std::string_view filename);
/**
* @brief Create an H5Writer class.
*
* @param filename The name of the h5 file to create.
* @return A ks_ptr to the newly created instance of H5Writer.
*/
static kc::ks_ptr<H5Writer> create(std::string_view filename);
/**
* @brief Destroy the H5Writer. This closes the H5 file.
*/
~H5Writer();
/**
* @brief Create a packet table to log.
*
* This creates the table and adds it to the active tables. Activate tables
* will have a data entry added whenever log is called. To deactivate a table
* use the deactivateTable method. The name of the tame will be the same as
* the name of the PacketTableConfig.
*
* @param config The config that specifies the data of the packet table.
*/
void createTable(const kc::ks_ptr<PacketTableConfig> &config);
/**
* @brief Log data for all the active tables.
*/
void log();
/**
* @brief Create a log entry for just the given table.
*
* This will log the table regardless of whether it is active or not.
*
* @param name The name of the table to log.
*/
void logTable(const std::string &name);
/**
* @brief Activate the table with the given name.
*
* @param name The name of the table to activate.
*/
void activateTable(const std::string &name);
/**
* @brief Deactivate the table with the given name.
*
* @param name The name of the table to deactivate.
*/
void deactivateTable(const std::string &name);
/**
* @brief Get a vector of the active table names.
*
* @return A vector of the active table names.
*/
std::vector<std::string> getActiveTableNames();
/**
* @brief Get a vector of all the table names.
*
* @return A vector of the table names.
*/
std::vector<std::string> getTableNames();
private:
/// Holds the id of the H5 file.
hid_t _file;
/// Usage tracking map for all the packet tables.
kc::UsageTrackingMap<hid_t, PacketTableConfig> _tables;
/// A map for all the active tables.
std::map<hid_t, kc::ks_ptr<PacketTableConfig>> _active_tables;
/// A map of all the table names to their H5 IDs
std::unordered_map<std::string, hid_t> _table_names;
/// Functions to run when closing.
std::vector<std::function<void()>> _fns_on_close;
std::unordered_set<hid_t> _ids_to_close;
};
} // namespace Karana::KUtils