1#ifndef LIGHTLY_EDGE_SDK_H
2#define LIGHTLY_EDGE_SDK_H
5#include "lightly_edge_cxx.rs.h"
6#include "lightly_edge_error.h"
7#include "lightly_edge_rs_bindings.h"
8#include "lightly_edge_rs_bindings_ext.h"
9#include "tl/expected.hpp"
18const size_t BUFFER_MAX_LENGTH = 3000;
19const uint32_t MAX_CLASSIFICATIONS_DEFAULT = 5;
20const int CPP_SDK_ERROR_CODE = 30;
52 Frame(
size_t width,
size_t height,
void *rgbImageData)
57using lightly_edge_rs::CStatus;
64inline auto default_config() -> LightlyEdgeConfig {
65 LightlyEdgeConfig config{};
66 config.detector_config.object_detector_enable =
false;
67 config.detector_config.classifiers_enable =
false;
68 config.detector_config.max_classifications = MAX_CLASSIFICATIONS_DEFAULT;
69 config.inference_device_type = lightly_edge_rs::InferenceDeviceType::GPUFallbackCPU;
74inline auto cxx_status_to_unexpected(lightly_edge_cxx::CxxStatus &status)
75 -> tl::unexpected<Error> {
76 return tl::make_unexpected(
124 if (this->lightly_edge !=
nullptr) {
125 lightly_edge_rs::destroy(this->lightly_edge);
133 : lightly_edge(other.lightly_edge), lightly_edge_cxx(other.lightly_edge_cxx) {
134 other.lightly_edge =
nullptr;
135 other.lightly_edge_cxx =
nullptr;
142 if (
this != &other) {
143 if (this->lightly_edge !=
nullptr) {
144 lightly_edge_rs::destroy(this->lightly_edge);
146 this->lightly_edge = other.lightly_edge;
147 this->lightly_edge_cxx = other.lightly_edge_cxx;
148 other.lightly_edge =
nullptr;
149 other.lightly_edge_cxx =
nullptr;
183 -> tl::expected<LightlyEdge, Error> {
184 lightly_edge_rs::LightlyEdge *lightly_edge =
nullptr;
186 lightly_edge_rs::lightly_edge_new_from_tar(archive_path, config, &lightly_edge);
187 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
188 return status_to_unexpected(&status);
210 -> tl::expected<std::vector<float>,
Error> {
212 lightly_edge_rs::embedding_dimension(this->lightly_edge);
215 auto status = lightly_edge_rs::embed_frame(
216 this->lightly_edge,
static_cast<uint8_t *
>(frame.rgbImageData_),
217 frame.width_ * frame.height_ * 3, frame.width_, frame.height_,
218 embedding_buffer.data());
219 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
220 return status_to_unexpected(&status);
223 return std::move(embedding_buffer);
232 return lightly_edge_rs::embedding_dimension(this->lightly_edge);
246 [[nodiscard]]
auto embed_texts(
const std::vector<std::string> &texts)
const
247 -> std::vector<std::vector<float>> {
264 -> tl::expected<std::vector<std::vector<float>>,
Error> {
272 std::vector<const char *> c_texts;
273 c_texts.reserve(texts.size());
274 for (
const auto &text : texts) {
275 c_texts.push_back(text.c_str());
281 std::vector<std::vector<float>> embeddings(texts.size(),
285 std::vector<float *> embeddings_ptrs;
286 embeddings_ptrs.reserve(texts.size());
287 for (
auto &embedding : embeddings) {
288 embeddings_ptrs.push_back(embedding.data());
292 auto status = lightly_edge_rs::embed_texts(lightly_edge, c_texts.data(),
293 c_texts.size(), embeddings_ptrs.data());
296 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
297 return status_to_unexpected(&status);
349 const std::vector<float> &embedding)
const noexcept -> tl::expected<void, Error> {
350 auto status = lightly_edge_rs::insert_into_embedding_database(
351 this->lightly_edge, embedding.data(), embedding.size());
352 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
353 return status_to_unexpected(&status);
375 -> tl::expected<
void,
Error> {
376 auto status = lightly_edge_cxx::clear_embedding_database(*this->lightly_edge_cxx);
377 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
378 return cxx_status_to_unexpected(status);
389 return lightly_edge_rs::num_diversity_strategies(this->lightly_edge);
398 return lightly_edge_rs::num_similarity_strategies(this->lightly_edge);
407 return lightly_edge_rs::num_adaptive_diversity_strategies(this->lightly_edge);
416 return lightly_edge_rs::num_detection_strategies(this->lightly_edge);
503 -> tl::expected<void, Error> {
505 lightly_edge_rs::register_diversity_strategy(this->lightly_edge, min_distance);
506 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
507 return status_to_unexpected(&status);
552 float max_distance)
const ->
void {
607 float max_distance)
const noexcept
608 -> tl::expected<void, Error> {
609 auto status = lightly_edge_rs::register_similarity_strategy(
610 this->lightly_edge, query_embedding.data(), query_embedding.size(),
612 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
613 return status_to_unexpected(&status);
650 float target_ratio,
size_t buffer_max_length = BUFFER_MAX_LENGTH)
const ->
void {
690 float target_ratio,
size_t buffer_max_length = BUFFER_MAX_LENGTH)
const noexcept
691 -> tl::expected<void, Error> {
692 auto status = lightly_edge_rs::register_adaptive_diversity_strategy(
693 this->lightly_edge, target_ratio, buffer_max_length);
694 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
695 return status_to_unexpected(&status);
734 float threshold)
const ->
void {
775 float threshold)
const noexcept
776 -> tl::expected<void, Error> {
777 auto status = lightly_edge_rs::register_detection_strategy(
778 this->lightly_edge, class_id, subclass_id, threshold);
779 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
780 return status_to_unexpected(&status);
817 -> std::vector<std::vector<float>> {
847 -> tl::expected<std::vector<std::vector<float>>,
Error> {
849 lightly_edge_rs::embedding_dimension(this->lightly_edge);
850 size_t num_patches = lightly_edge_rs::num_patches(this->lightly_edge);
853 std::vector<std::vector<float>> embeddings(num_patches,
855 std::vector<float *> embeddings_ptrs;
856 embeddings_ptrs.reserve(num_patches);
857 for (
auto &embedding : embeddings) {
858 embeddings_ptrs.push_back(embedding.data());
862 auto status = lightly_edge_rs::embed_patches(
863 this->lightly_edge,
static_cast<uint8_t *
>(frame.rgbImageData_),
864 frame.width_ * frame.height_ * 3, frame.width_, frame.height_,
865 embeddings_ptrs.data());
867 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
868 return status_to_unexpected(&status);
927 const std::vector<PatchCoordsRel> &patches)
const noexcept
928 -> tl::expected<void, Error> {
929 auto status = lightly_edge_rs::set_embedding_patches(
930 this->lightly_edge, patches.data(), patches.size());
931 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
932 return status_to_unexpected(&status);
941 lightly_edge_cxx::reset_embedding_patches(*this->lightly_edge_cxx);
986 const std::vector<ObjectDetection> &detections)
const
1036 const std::vector<ObjectDetection> &detections)
const noexcept
1037 -> tl::expected<SelectInfo, Error> {
1039 CStatus status = {};
1040 auto select_info_buffers = SelectInfoBuffers(this->lightly_edge);
1041 auto out_info = select_info_buffers.to_c_select_info();
1043 status = lightly_edge_rs::should_select(this->lightly_edge, embedding.data(),
1044 embedding.size(), detections.data(),
1045 detections.size(), &out_info);
1046 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
1047 return status_to_unexpected(&status);
1051 auto select_info = select_info_buffers.to_select_info();
1101 const std::vector<ObjectDetection> &detections)
const
1155 const std::vector<std::vector<float>> &embeddings,
1156 const std::vector<ObjectDetection> &detections)
const noexcept
1157 -> tl::expected<SelectInfo, Error> {
1159 CStatus status = {};
1160 auto select_info_buffers = SelectInfoBuffers(this->lightly_edge);
1161 auto out_info = select_info_buffers.to_c_select_info();
1167 size_t embedding_dim = embeddings.empty() ? 0 : embeddings[0].size();
1170 for (
const auto &embedding : embeddings) {
1171 if (embedding.size() != embedding_dim) {
1173 CPP_SDK_ERROR_CODE,
"All patch embeddings must have the same dimension"));
1178 std::vector<const float *> embeddings_ptrs;
1179 embeddings_ptrs.reserve(embeddings.size());
1180 for (
const auto &embedding : embeddings) {
1181 embeddings_ptrs.push_back(embedding.data());
1184 status = lightly_edge_rs::should_select_with_patches(
1185 this->lightly_edge, embeddings_ptrs.data(), embeddings_ptrs.size(),
1186 embedding_dim, detections.data(), detections.size(), &out_info);
1187 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
1188 return status_to_unexpected(&status);
1192 auto select_info = select_info_buffers.to_select_info();
1203 -> tl::expected<std::vector<ObjectDetection>,
Error> {
1204 std::vector<ObjectDetection> detections;
1207 const auto capacity = 1000;
1208 detections.resize(capacity);
1209 size_t detections_count = detections.size();
1210 auto status = lightly_edge_rs::detect(
1211 this->lightly_edge,
static_cast<uint8_t *
>(frame.rgbImageData_),
1212 frame.width_ * frame.height_ * 3, frame.width_, frame.height_,
1213 detections.data(), &detections_count);
1214 detections.resize(detections_count);
1215 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
1216 return status_to_unexpected(&status);
1226 [[nodiscard]]
auto detect(
const Frame &frame)
const -> std::vector<ObjectDetection> {
1243 -> std::vector<std::
string> {
1246 const auto capacity = 1000;
1247 size_t classes_count = capacity;
1248 std::vector<const char *> cstr_class_labels(classes_count);
1249 auto status = lightly_edge_rs::detection_class_labels(
1250 this->lightly_edge, cstr_class_labels.data(), &classes_count);
1254 assert(status.code == lightly_edge_rs::SUCCESS_CODE);
1257 std::vector<std::string> class_labels(classes_count);
1258 for (
size_t i = 0; i < classes_count; ++i) {
1259 class_labels[i] = cstr_class_labels[i];
1261 return class_labels;
1281 -> std::vector<std::string> {
1284 const auto capacity = 1000;
1285 size_t subclasses_count = capacity;
1286 std::vector<const char *> cstr_subclass_labels(subclasses_count);
1287 auto status = lightly_edge_rs::detection_subclass_labels(
1288 this->lightly_edge, class_id, cstr_subclass_labels.data(), &subclasses_count);
1292 assert(status.code == lightly_edge_rs::SUCCESS_CODE);
1295 std::vector<std::string> subclass_labels(subclasses_count);
1296 for (
size_t i = 0; i < subclasses_count; ++i) {
1297 subclass_labels[i] = cstr_subclass_labels[i];
1299 return subclass_labels;
1304 explicit LightlyEdge(lightly_edge_rs::LightlyEdge *lightly_edge_)
1305 : lightly_edge(lightly_edge_),
1307 reinterpret_cast<lightly_edge_cxx::
LightlyEdge *>(lightly_edge_)) {}
1308 lightly_edge_cxx::LightlyEdge *lightly_edge_cxx{};
1309 lightly_edge_rs::LightlyEdge *lightly_edge{};
1314 class SelectInfoBuffers {
1316 explicit SelectInfoBuffers(lightly_edge_rs::LightlyEdge *lightly_edge)
1319 out_adaptive_diversity_size(
1321 out_detection_selection_size(
1324 resize_buffers_to_match_sizes();
1331 [[nodiscard]]
auto to_c_select_info()
1332 -> lightly_edge_rs::SelectionStrategiesSelectInfo {
1333 lightly_edge_rs::SelectionStrategiesSelectInfo out_info{};
1334 out_info.out_diversity_buffer = diversity.data();
1335 out_info.diversity_buffer_size = diversity.size();
1336 out_info.out_diversity_size = &out_diversity_size;
1338 out_info.out_similarity_buffer = similarity.data();
1339 out_info.similarity_buffer_size = similarity.size();
1340 out_info.out_similarity_size = &out_similarity_size;
1342 out_info.out_adaptive_diversity_buffer = adaptive_diversity.data();
1343 out_info.adaptive_diversity_buffer_size = adaptive_diversity.size();
1344 out_info.out_adaptive_diversity_size = &out_adaptive_diversity_size;
1346 out_info.out_detection_buffer = detection.data();
1347 out_info.detection_buffer_size = detection.size();
1348 out_info.out_detection_size = &out_detection_selection_size;
1360 resize_buffers_to_match_sizes();
1363 select_info.
diversity = std::move(diversity);
1364 select_info.
similarity = std::move(similarity);
1366 select_info.
detection = std::move(detection);
1371 size_t out_diversity_size;
1372 size_t out_similarity_size;
1373 size_t out_adaptive_diversity_size;
1374 size_t out_detection_selection_size;
1375 std::vector<lightly_edge_rs::DiversitySelectInfo> diversity;
1376 std::vector<lightly_edge_rs::SimilaritySelectInfo> similarity;
1377 std::vector<lightly_edge_rs::AdaptiveDiversitySelectInfo> adaptive_diversity;
1378 std::vector<lightly_edge_rs::DetectionSelectInfo> detection;
1380 void resize_buffers_to_match_sizes() {
1381 diversity.resize(out_diversity_size);
1382 similarity.resize(out_similarity_size);
1383 adaptive_diversity.resize(out_adaptive_diversity_size);
1384 detection.resize(out_detection_selection_size);
LightlyEdge.
Definition lightly_edge_sdk.h:118
auto embed_texts(const std::vector< std::string > &texts) const -> std::vector< std::vector< float > >
Embed a list of text strings.
Definition lightly_edge_sdk.h:246
auto register_similarity_strategy_noexcept(const std::vector< float > &query_embedding, float max_distance) const noexcept -> tl::expected< void, Error >
Register a similarity strategy, noexcept version.
Definition lightly_edge_sdk.h:606
auto embed_frame_noexcept(const Frame &frame) const noexcept -> tl::expected< std::vector< float >, Error >
Embed an RGB image.
Definition lightly_edge_sdk.h:209
auto num_diversity_strategies() const noexcept -> size_t
Get the number of registered diversity strategies.
Definition lightly_edge_sdk.h:388
LightlyEdge(LightlyEdge &&other) noexcept
Move constructor.
Definition lightly_edge_sdk.h:132
auto clear_embedding_database() const -> void
Clear the embedding database.
Definition lightly_edge_sdk.h:365
auto register_diversity_strategy(float min_distance) const -> void
Register a diversity strategy.
Definition lightly_edge_sdk.h:455
LightlyEdge(const LightlyEdge &other)=delete
Copy constructor (disallowed).
auto register_adaptive_diversity_strategy_noexcept(float target_ratio, size_t buffer_max_length=BUFFER_MAX_LENGTH) const noexcept -> tl::expected< void, Error >
Register an adaptive diversity strategy, no except version.
Definition lightly_edge_sdk.h:689
void reset_embedding_patches()
Reset the embedding patches to full image patch.
Definition lightly_edge_sdk.h:940
auto num_similarity_strategies() const noexcept -> size_t
Get the number of registered similarity strategies.
Definition lightly_edge_sdk.h:397
auto should_select_with_patches(const std::vector< std::vector< float > > &embeddings, const std::vector< ObjectDetection > &detections) const -> SelectInfo
Check if a frame should be selected taking patches into account.
Definition lightly_edge_sdk.h:1100
auto set_embedding_patches(const std::vector< PatchCoordsRel > &patches) const -> void
Set the embedding patches.
Definition lightly_edge_sdk.h:901
auto register_detection_strategy_noexcept(uint32_t class_id, int32_t subclass_id, float threshold) const noexcept -> tl::expected< void, Error >
Register a detection strategy, noexcept version.
Definition lightly_edge_sdk.h:774
auto register_detection_strategy(uint32_t class_id, int32_t subclass_id, float threshold) const -> void
Register a detection strategy.
Definition lightly_edge_sdk.h:733
auto detect(const Frame &frame) const -> std::vector< ObjectDetection >
Run object detection on a frame.
Definition lightly_edge_sdk.h:1226
auto register_similarity_strategy(const std::vector< float > &query_embedding, float max_distance) const -> void
Register a similarity strategy.
Definition lightly_edge_sdk.h:551
auto insert_into_embedding_database_noexcept(const std::vector< float > &embedding) const noexcept -> tl::expected< void, Error >
Insert an embedding into the database.
Definition lightly_edge_sdk.h:348
auto register_diversity_strategy_noexcept(float min_distance) const noexcept -> tl::expected< void, Error >
Register a diversity strategy, noexcept version.
Definition lightly_edge_sdk.h:502
auto operator=(const LightlyEdge &other) -> LightlyEdge &=delete
Copy assignment operator (disallowed).
static auto new_from_tar_noexcept(const char *archive_path, LightlyEdgeConfig config) noexcept -> tl::expected< LightlyEdge, Error >
Initialize LightlyEdge with an ort embedding inference backend.
Definition lightly_edge_sdk.h:181
void insert_into_embedding_database(const std::vector< float > &embedding) const
Insert an embedding into the database.
Definition lightly_edge_sdk.h:324
~LightlyEdge()
Deallocate the memory associated with a LightlyEdge.
Definition lightly_edge_sdk.h:123
auto detect_noexcept(const Frame &frame) const noexcept -> tl::expected< std::vector< ObjectDetection >, Error >
Run object detection on a frame.
Definition lightly_edge_sdk.h:1202
auto embed_texts_noexcept(const std::vector< std::string > &texts) const noexcept -> tl::expected< std::vector< std::vector< float > >, Error >
Embed a list of text strings.
Definition lightly_edge_sdk.h:263
auto should_select(const std::vector< float > &embedding, const std::vector< ObjectDetection > &detections) const -> SelectInfo
Check if a frame should be selected.
Definition lightly_edge_sdk.h:985
auto detection_class_labels() const noexcept -> std::vector< std::string >
Get labels of object detection classes.
Definition lightly_edge_sdk.h:1242
auto clear_embedding_database_noexcept() const noexcept -> tl::expected< void, Error >
Clear the embedding database.
Definition lightly_edge_sdk.h:374
auto should_select_with_patches_noexcept(const std::vector< std::vector< float > > &embeddings, const std::vector< ObjectDetection > &detections) const noexcept -> tl::expected< SelectInfo, Error >
Check if a frame should be selected taking patches into account.
Definition lightly_edge_sdk.h:1154
void clear_strategies()
Clear all registered selection strategies.
Definition lightly_edge_sdk.h:788
auto detection_subclass_labels(uint32_t class_id) const noexcept -> std::vector< std::string >
Get the subclass labels for a specified detection class.
Definition lightly_edge_sdk.h:1280
auto embed_patches_noexcept(const Frame &frame) const noexcept -> tl::expected< std::vector< std::vector< float > >, Error >
Embed patches of an image, noexcept version.
Definition lightly_edge_sdk.h:846
auto operator=(LightlyEdge &&other) noexcept -> LightlyEdge &
Move assignment operator.
Definition lightly_edge_sdk.h:141
auto num_adaptive_diversity_strategies() const noexcept -> size_t
Get the number of registered adaptive diversity strategies.
Definition lightly_edge_sdk.h:406
auto embed_patches(const Frame &frame) const -> std::vector< std::vector< float > >
Embed patches of an image.
Definition lightly_edge_sdk.h:816
auto num_detection_strategies() const noexcept -> size_t
Get the number of registered detection strategies.
Definition lightly_edge_sdk.h:415
auto register_adaptive_diversity_strategy(float target_ratio, size_t buffer_max_length=BUFFER_MAX_LENGTH) const -> void
Register an adaptive diversity strategy.
Definition lightly_edge_sdk.h:649
auto set_embedding_patches_noexcept(const std::vector< PatchCoordsRel > &patches) const noexcept -> tl::expected< void, Error >
Set the embedding patches, noexcept version.
Definition lightly_edge_sdk.h:926
static auto new_from_tar(const char *archive_path, LightlyEdgeConfig config) -> LightlyEdge
Initialize LightlyEdge with an ort embedding inference backend.
Definition lightly_edge_sdk.h:170
auto should_select_noexcept(const std::vector< float > &embedding, const std::vector< ObjectDetection > &detections) const noexcept -> tl::expected< SelectInfo, Error >
Check if a frame should be selected.
Definition lightly_edge_sdk.h:1035
auto embed_frame(const Frame &frame) const -> std::vector< float >
Embed an RGB image.
Definition lightly_edge_sdk.h:199
auto embedding_dimension() const noexcept -> size_t
Get the embedding dimension.
Definition lightly_edge_sdk.h:231
Namespace with core LightlyEdge SDK functionality.
Definition lightly_edge_error.h:15
auto value_or_throw(tl::expected< T, lightly_edge_sdk::Error > &&result) -> T
Convenience function to unwrap a result of lightly_edge_sdk::LightlyEdge functions.
Definition lightly_edge_error.h:79
Holds information whether a frame should be selected or not.
Definition lightly_edge_rs_bindings.h:71
Definition lightly_edge_rs_bindings.h:76
Holds information whether a frame should be selected or not.
Definition lightly_edge_rs_bindings.h:59
Definition lightly_edge_rs_bindings.h:43
Definition lightly_edge_rs_bindings.h:48
A patch specified by relative coordinates.
Definition lightly_edge_rs_bindings.h:99
Holds information whether a frame should be selected or not.
Definition lightly_edge_rs_bindings.h:65
Error struct to hold error code and message.
Definition lightly_edge_error.h:20
Frame data for LightlyEdge.
Definition lightly_edge_sdk.h:25
size_t width_
Width of the image.
Definition lightly_edge_sdk.h:29
void * rgbImageData_
Pointer to the RGB image data.
Definition lightly_edge_sdk.h:42
size_t height_
Height of the image.
Definition lightly_edge_sdk.h:34
Frame(size_t width, size_t height, void *rgbImageData)
Construct a new Frame object.
Definition lightly_edge_sdk.h:52
Selection information about a processed frame.
Definition lightly_edge_sdk.h:83
std::vector< DiversitySelectInfo > diversity
The diversity selection info for each diversity strategy in the order of registration.
Definition lightly_edge_sdk.h:88
std::vector< SimilaritySelectInfo > similarity
The similarity selection info for each similarity strategy in the order of registration.
Definition lightly_edge_sdk.h:94
std::vector< AdaptiveDiversitySelectInfo > adaptive_diversity
The adaptive diversity selection info for each adaptive diversity strategy in the order of registrati...
Definition lightly_edge_sdk.h:100
std::vector< DetectionSelectInfo > detection
The detection selection info for each detection strategy in the order of registration.
Definition lightly_edge_sdk.h:106