1#ifndef LIGHTLY_EDGE_SDK_H
2#define LIGHTLY_EDGE_SDK_H
5#include "lightly_edge_cxx.rs.h"
6#include "lightly_edge_error.h"
7#include "lightly_edge_rs_bindings.h"
8#include "lightly_edge_rs_bindings_ext.h"
9#include "tl/expected.hpp"
18const size_t BUFFER_MAX_LENGTH = 3000;
19const uint32_t MAX_CLASSIFICATIONS_DEFAULT = 5;
20const int CPP_SDK_ERROR_CODE = 30;
52 Frame(
size_t width,
size_t height,
void *rgbImageData)
57using lightly_edge_rs::CStatus;
60using lightly_edge_rs::InferenceDeviceType;
65inline auto default_config() -> LightlyEdgeConfig {
66 LightlyEdgeConfig config{};
67 config.detector_config.object_detector_enable =
false;
68 config.detector_config.classifiers_enable =
false;
69 config.detector_config.max_classifications = MAX_CLASSIFICATIONS_DEFAULT;
70 config.inference_device_type = lightly_edge_rs::InferenceDeviceType::Auto;
75inline auto cxx_status_to_unexpected(lightly_edge_cxx::CxxStatus &status)
76 -> tl::unexpected<Error> {
77 return tl::make_unexpected(
125 if (this->lightly_edge !=
nullptr) {
126 lightly_edge_rs::destroy(this->lightly_edge);
134 : lightly_edge(other.lightly_edge), lightly_edge_cxx(other.lightly_edge_cxx) {
135 other.lightly_edge =
nullptr;
136 other.lightly_edge_cxx =
nullptr;
143 if (
this != &other) {
144 if (this->lightly_edge !=
nullptr) {
145 lightly_edge_rs::destroy(this->lightly_edge);
147 this->lightly_edge = other.lightly_edge;
148 this->lightly_edge_cxx = other.lightly_edge_cxx;
149 other.lightly_edge =
nullptr;
150 other.lightly_edge_cxx =
nullptr;
184 -> tl::expected<LightlyEdge, Error> {
185 lightly_edge_rs::LightlyEdge *lightly_edge =
nullptr;
187 lightly_edge_rs::lightly_edge_new_from_tar(archive_path, config, &lightly_edge);
188 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
189 return status_to_unexpected(&status);
212 [[nodiscard]]
static auto
215 -> tl::expected<LightlyEdge, Error> {
216 lightly_edge_rs::LightlyEdge *lightly_edge =
nullptr;
217 CStatus status = lightly_edge_rs::lightly_edge_new_from_directory(
218 model_directory, config, &lightly_edge);
219 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
220 return status_to_unexpected(&status);
242 -> tl::expected<std::vector<float>,
Error> {
244 lightly_edge_rs::embedding_dimension(this->lightly_edge);
247 auto status = lightly_edge_rs::embed_frame(
248 this->lightly_edge,
static_cast<uint8_t *
>(frame.rgbImageData_),
249 frame.width_ * frame.height_ * 3, frame.width_, frame.height_,
250 embedding_buffer.data());
251 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
252 return status_to_unexpected(&status);
255 return std::move(embedding_buffer);
264 return lightly_edge_rs::embedding_dimension(this->lightly_edge);
278 [[nodiscard]]
auto embed_texts(
const std::vector<std::string> &texts)
const
279 -> std::vector<std::vector<float>> {
296 -> tl::expected<std::vector<std::vector<float>>,
Error> {
304 std::vector<const char *> c_texts;
305 c_texts.reserve(texts.size());
306 for (
const auto &text : texts) {
307 c_texts.push_back(text.c_str());
313 std::vector<std::vector<float>> embeddings(texts.size(),
317 std::vector<float *> embeddings_ptrs;
318 embeddings_ptrs.reserve(texts.size());
319 for (
auto &embedding : embeddings) {
320 embeddings_ptrs.push_back(embedding.data());
324 auto status = lightly_edge_rs::embed_texts(lightly_edge, c_texts.data(),
325 c_texts.size(), embeddings_ptrs.data());
328 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
329 return status_to_unexpected(&status);
381 const std::vector<float> &embedding)
const noexcept -> tl::expected<void, Error> {
382 auto status = lightly_edge_rs::insert_into_embedding_database(
383 this->lightly_edge, embedding.data(), embedding.size());
384 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
385 return status_to_unexpected(&status);
407 -> tl::expected<
void,
Error> {
408 auto status = lightly_edge_cxx::clear_embedding_database(*this->lightly_edge_cxx);
409 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
410 return cxx_status_to_unexpected(status);
421 return lightly_edge_rs::num_diversity_strategies(this->lightly_edge);
430 return lightly_edge_rs::num_similarity_strategies(this->lightly_edge);
439 return lightly_edge_rs::num_adaptive_diversity_strategies(this->lightly_edge);
448 return lightly_edge_rs::num_detection_strategies(this->lightly_edge);
535 -> tl::expected<void, Error> {
537 lightly_edge_rs::register_diversity_strategy(this->lightly_edge, min_distance);
538 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
539 return status_to_unexpected(&status);
584 float max_distance)
const ->
void {
639 float max_distance)
const noexcept
640 -> tl::expected<void, Error> {
641 auto status = lightly_edge_rs::register_similarity_strategy(
642 this->lightly_edge, query_embedding.data(), query_embedding.size(),
644 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
645 return status_to_unexpected(&status);
682 float target_ratio,
size_t buffer_max_length = BUFFER_MAX_LENGTH)
const ->
void {
722 float target_ratio,
size_t buffer_max_length = BUFFER_MAX_LENGTH)
const noexcept
723 -> tl::expected<void, Error> {
724 auto status = lightly_edge_rs::register_adaptive_diversity_strategy(
725 this->lightly_edge, target_ratio, buffer_max_length);
726 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
727 return status_to_unexpected(&status);
766 float threshold)
const ->
void {
807 float threshold)
const noexcept
808 -> tl::expected<void, Error> {
809 auto status = lightly_edge_rs::register_detection_strategy(
810 this->lightly_edge, class_id, subclass_id, threshold);
811 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
812 return status_to_unexpected(&status);
849 -> std::vector<std::vector<float>> {
879 -> tl::expected<std::vector<std::vector<float>>,
Error> {
881 lightly_edge_rs::embedding_dimension(this->lightly_edge);
882 size_t num_patches = lightly_edge_rs::num_patches(this->lightly_edge);
885 std::vector<std::vector<float>> embeddings(num_patches,
887 std::vector<float *> embeddings_ptrs;
888 embeddings_ptrs.reserve(num_patches);
889 for (
auto &embedding : embeddings) {
890 embeddings_ptrs.push_back(embedding.data());
894 auto status = lightly_edge_rs::embed_patches(
895 this->lightly_edge,
static_cast<uint8_t *
>(frame.rgbImageData_),
896 frame.width_ * frame.height_ * 3, frame.width_, frame.height_,
897 embeddings_ptrs.data());
899 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
900 return status_to_unexpected(&status);
959 const std::vector<PatchCoordsRel> &patches)
const noexcept
960 -> tl::expected<void, Error> {
961 auto status = lightly_edge_rs::set_embedding_patches(
962 this->lightly_edge, patches.data(), patches.size());
963 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
964 return status_to_unexpected(&status);
973 lightly_edge_cxx::reset_embedding_patches(*this->lightly_edge_cxx);
1018 const std::vector<ObjectDetection> &detections)
const
1068 const std::vector<ObjectDetection> &detections)
const noexcept
1069 -> tl::expected<SelectInfo, Error> {
1071 CStatus status = {};
1072 auto select_info_buffers = SelectInfoBuffers(this->lightly_edge);
1073 auto out_info = select_info_buffers.to_c_select_info();
1075 status = lightly_edge_rs::should_select(this->lightly_edge, embedding.data(),
1076 embedding.size(), detections.data(),
1077 detections.size(), &out_info);
1078 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
1079 return status_to_unexpected(&status);
1083 auto select_info = select_info_buffers.to_select_info();
1133 const std::vector<ObjectDetection> &detections)
const
1187 const std::vector<std::vector<float>> &embeddings,
1188 const std::vector<ObjectDetection> &detections)
const noexcept
1189 -> tl::expected<SelectInfo, Error> {
1191 CStatus status = {};
1192 auto select_info_buffers = SelectInfoBuffers(this->lightly_edge);
1193 auto out_info = select_info_buffers.to_c_select_info();
1199 size_t embedding_dim = embeddings.empty() ? 0 : embeddings[0].size();
1202 for (
const auto &embedding : embeddings) {
1203 if (embedding.size() != embedding_dim) {
1205 CPP_SDK_ERROR_CODE,
"All patch embeddings must have the same dimension"));
1210 std::vector<const float *> embeddings_ptrs;
1211 embeddings_ptrs.reserve(embeddings.size());
1212 for (
const auto &embedding : embeddings) {
1213 embeddings_ptrs.push_back(embedding.data());
1216 status = lightly_edge_rs::should_select_with_patches(
1217 this->lightly_edge, embeddings_ptrs.data(), embeddings_ptrs.size(),
1218 embedding_dim, detections.data(), detections.size(), &out_info);
1219 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
1220 return status_to_unexpected(&status);
1224 auto select_info = select_info_buffers.to_select_info();
1235 -> tl::expected<std::vector<ObjectDetection>,
Error> {
1236 std::vector<ObjectDetection> detections;
1239 const auto capacity = 1000;
1240 detections.resize(capacity);
1241 size_t detections_count = detections.size();
1242 auto status = lightly_edge_rs::detect(
1243 this->lightly_edge,
static_cast<uint8_t *
>(frame.rgbImageData_),
1244 frame.width_ * frame.height_ * 3, frame.width_, frame.height_,
1245 detections.data(), &detections_count);
1246 detections.resize(detections_count);
1247 if (status.code != lightly_edge_rs::SUCCESS_CODE) {
1248 return status_to_unexpected(&status);
1258 [[nodiscard]]
auto detect(
const Frame &frame)
const -> std::vector<ObjectDetection> {
1275 -> std::vector<std::
string> {
1278 const auto capacity = 1000;
1279 size_t classes_count = capacity;
1280 std::vector<const char *> cstr_class_labels(classes_count);
1281 auto status = lightly_edge_rs::detection_class_labels(
1282 this->lightly_edge, cstr_class_labels.data(), &classes_count);
1286 assert(status.code == lightly_edge_rs::SUCCESS_CODE);
1289 std::vector<std::string> class_labels(classes_count);
1290 for (
size_t i = 0; i < classes_count; ++i) {
1291 class_labels[i] = cstr_class_labels[i];
1293 return class_labels;
1313 -> std::vector<std::string> {
1316 const auto capacity = 1000;
1317 size_t subclasses_count = capacity;
1318 std::vector<const char *> cstr_subclass_labels(subclasses_count);
1319 auto status = lightly_edge_rs::detection_subclass_labels(
1320 this->lightly_edge, class_id, cstr_subclass_labels.data(), &subclasses_count);
1324 assert(status.code == lightly_edge_rs::SUCCESS_CODE);
1327 std::vector<std::string> subclass_labels(subclasses_count);
1328 for (
size_t i = 0; i < subclasses_count; ++i) {
1329 subclass_labels[i] = cstr_subclass_labels[i];
1331 return subclass_labels;
1336 explicit LightlyEdge(lightly_edge_rs::LightlyEdge *lightly_edge_)
1337 : lightly_edge(lightly_edge_),
1339 reinterpret_cast<lightly_edge_cxx::
LightlyEdge *>(lightly_edge_)) {}
1340 lightly_edge_cxx::LightlyEdge *lightly_edge_cxx{};
1341 lightly_edge_rs::LightlyEdge *lightly_edge{};
1346 class SelectInfoBuffers {
1348 explicit SelectInfoBuffers(lightly_edge_rs::LightlyEdge *lightly_edge)
1351 out_adaptive_diversity_size(
1353 out_detection_selection_size(
1356 resize_buffers_to_match_sizes();
1363 [[nodiscard]]
auto to_c_select_info()
1364 -> lightly_edge_rs::SelectionStrategiesSelectInfo {
1365 lightly_edge_rs::SelectionStrategiesSelectInfo out_info{};
1366 out_info.out_diversity_buffer = diversity.data();
1367 out_info.diversity_buffer_size = diversity.size();
1368 out_info.out_diversity_size = &out_diversity_size;
1370 out_info.out_similarity_buffer = similarity.data();
1371 out_info.similarity_buffer_size = similarity.size();
1372 out_info.out_similarity_size = &out_similarity_size;
1374 out_info.out_adaptive_diversity_buffer = adaptive_diversity.data();
1375 out_info.adaptive_diversity_buffer_size = adaptive_diversity.size();
1376 out_info.out_adaptive_diversity_size = &out_adaptive_diversity_size;
1378 out_info.out_detection_buffer = detection.data();
1379 out_info.detection_buffer_size = detection.size();
1380 out_info.out_detection_size = &out_detection_selection_size;
1392 resize_buffers_to_match_sizes();
1395 select_info.
diversity = std::move(diversity);
1396 select_info.
similarity = std::move(similarity);
1398 select_info.
detection = std::move(detection);
1403 size_t out_diversity_size;
1404 size_t out_similarity_size;
1405 size_t out_adaptive_diversity_size;
1406 size_t out_detection_selection_size;
1407 std::vector<lightly_edge_rs::DiversitySelectInfo> diversity;
1408 std::vector<lightly_edge_rs::SimilaritySelectInfo> similarity;
1409 std::vector<lightly_edge_rs::AdaptiveDiversitySelectInfo> adaptive_diversity;
1410 std::vector<lightly_edge_rs::DetectionSelectInfo> detection;
1412 void resize_buffers_to_match_sizes() {
1413 diversity.resize(out_diversity_size);
1414 similarity.resize(out_similarity_size);
1415 adaptive_diversity.resize(out_adaptive_diversity_size);
1416 detection.resize(out_detection_selection_size);
LightlyEdge.
Definition lightly_edge_sdk.h:119
static auto new_from_directory(const char *model_directory, LightlyEdgeConfig config)
Initialize LightlyEdge with an ort inference backend.
Definition lightly_edge_sdk.h:200
auto embed_texts(const std::vector< std::string > &texts) const -> std::vector< std::vector< float > >
Embed a list of text strings.
Definition lightly_edge_sdk.h:278
auto register_similarity_strategy_noexcept(const std::vector< float > &query_embedding, float max_distance) const noexcept -> tl::expected< void, Error >
Register a similarity strategy, noexcept version.
Definition lightly_edge_sdk.h:638
auto embed_frame_noexcept(const Frame &frame) const noexcept -> tl::expected< std::vector< float >, Error >
Embed an RGB image.
Definition lightly_edge_sdk.h:241
auto num_diversity_strategies() const noexcept -> size_t
Get the number of registered diversity strategies.
Definition lightly_edge_sdk.h:420
LightlyEdge(LightlyEdge &&other) noexcept
Move constructor.
Definition lightly_edge_sdk.h:133
auto clear_embedding_database() const -> void
Clear the embedding database.
Definition lightly_edge_sdk.h:397
auto register_diversity_strategy(float min_distance) const -> void
Register a diversity strategy.
Definition lightly_edge_sdk.h:487
LightlyEdge(const LightlyEdge &other)=delete
Copy constructor (disallowed).
auto register_adaptive_diversity_strategy_noexcept(float target_ratio, size_t buffer_max_length=BUFFER_MAX_LENGTH) const noexcept -> tl::expected< void, Error >
Register an adaptive diversity strategy, no except version.
Definition lightly_edge_sdk.h:721
void reset_embedding_patches()
Reset the embedding patches to full image patch.
Definition lightly_edge_sdk.h:972
auto num_similarity_strategies() const noexcept -> size_t
Get the number of registered similarity strategies.
Definition lightly_edge_sdk.h:429
auto should_select_with_patches(const std::vector< std::vector< float > > &embeddings, const std::vector< ObjectDetection > &detections) const -> SelectInfo
Check if a frame should be selected taking patches into account.
Definition lightly_edge_sdk.h:1132
auto set_embedding_patches(const std::vector< PatchCoordsRel > &patches) const -> void
Set the embedding patches.
Definition lightly_edge_sdk.h:933
auto register_detection_strategy_noexcept(uint32_t class_id, int32_t subclass_id, float threshold) const noexcept -> tl::expected< void, Error >
Register a detection strategy, noexcept version.
Definition lightly_edge_sdk.h:806
auto register_detection_strategy(uint32_t class_id, int32_t subclass_id, float threshold) const -> void
Register a detection strategy.
Definition lightly_edge_sdk.h:765
auto detect(const Frame &frame) const -> std::vector< ObjectDetection >
Run object detection on a frame.
Definition lightly_edge_sdk.h:1258
auto register_similarity_strategy(const std::vector< float > &query_embedding, float max_distance) const -> void
Register a similarity strategy.
Definition lightly_edge_sdk.h:583
auto insert_into_embedding_database_noexcept(const std::vector< float > &embedding) const noexcept -> tl::expected< void, Error >
Insert an embedding into the database.
Definition lightly_edge_sdk.h:380
auto register_diversity_strategy_noexcept(float min_distance) const noexcept -> tl::expected< void, Error >
Register a diversity strategy, noexcept version.
Definition lightly_edge_sdk.h:534
auto operator=(const LightlyEdge &other) -> LightlyEdge &=delete
Copy assignment operator (disallowed).
static auto new_from_tar_noexcept(const char *archive_path, LightlyEdgeConfig config) noexcept -> tl::expected< LightlyEdge, Error >
Initialize LightlyEdge with an ort inference backend.
Definition lightly_edge_sdk.h:182
void insert_into_embedding_database(const std::vector< float > &embedding) const
Insert an embedding into the database.
Definition lightly_edge_sdk.h:356
~LightlyEdge()
Deallocate the memory associated with a LightlyEdge.
Definition lightly_edge_sdk.h:124
auto detect_noexcept(const Frame &frame) const noexcept -> tl::expected< std::vector< ObjectDetection >, Error >
Run object detection on a frame.
Definition lightly_edge_sdk.h:1234
auto embed_texts_noexcept(const std::vector< std::string > &texts) const noexcept -> tl::expected< std::vector< std::vector< float > >, Error >
Embed a list of text strings.
Definition lightly_edge_sdk.h:295
auto should_select(const std::vector< float > &embedding, const std::vector< ObjectDetection > &detections) const -> SelectInfo
Check if a frame should be selected.
Definition lightly_edge_sdk.h:1017
auto detection_class_labels() const noexcept -> std::vector< std::string >
Get labels of object detection classes.
Definition lightly_edge_sdk.h:1274
auto clear_embedding_database_noexcept() const noexcept -> tl::expected< void, Error >
Clear the embedding database.
Definition lightly_edge_sdk.h:406
auto should_select_with_patches_noexcept(const std::vector< std::vector< float > > &embeddings, const std::vector< ObjectDetection > &detections) const noexcept -> tl::expected< SelectInfo, Error >
Check if a frame should be selected taking patches into account.
Definition lightly_edge_sdk.h:1186
void clear_strategies()
Clear all registered selection strategies.
Definition lightly_edge_sdk.h:820
auto detection_subclass_labels(uint32_t class_id) const noexcept -> std::vector< std::string >
Get the subclass labels for a specified detection class.
Definition lightly_edge_sdk.h:1312
auto embed_patches_noexcept(const Frame &frame) const noexcept -> tl::expected< std::vector< std::vector< float > >, Error >
Embed patches of an image, noexcept version.
Definition lightly_edge_sdk.h:878
auto operator=(LightlyEdge &&other) noexcept -> LightlyEdge &
Move assignment operator.
Definition lightly_edge_sdk.h:142
auto num_adaptive_diversity_strategies() const noexcept -> size_t
Get the number of registered adaptive diversity strategies.
Definition lightly_edge_sdk.h:438
auto embed_patches(const Frame &frame) const -> std::vector< std::vector< float > >
Embed patches of an image.
Definition lightly_edge_sdk.h:848
auto num_detection_strategies() const noexcept -> size_t
Get the number of registered detection strategies.
Definition lightly_edge_sdk.h:447
auto register_adaptive_diversity_strategy(float target_ratio, size_t buffer_max_length=BUFFER_MAX_LENGTH) const -> void
Register an adaptive diversity strategy.
Definition lightly_edge_sdk.h:681
auto set_embedding_patches_noexcept(const std::vector< PatchCoordsRel > &patches) const noexcept -> tl::expected< void, Error >
Set the embedding patches, noexcept version.
Definition lightly_edge_sdk.h:958
static auto new_from_tar(const char *archive_path, LightlyEdgeConfig config) -> LightlyEdge
Initialize LightlyEdge with an ort embedding inference backend.
Definition lightly_edge_sdk.h:171
auto should_select_noexcept(const std::vector< float > &embedding, const std::vector< ObjectDetection > &detections) const noexcept -> tl::expected< SelectInfo, Error >
Check if a frame should be selected.
Definition lightly_edge_sdk.h:1067
static auto new_from_directory_noexcept(const char *model_directory, LightlyEdgeConfig config) noexcept -> tl::expected< LightlyEdge, Error >
Initialize LightlyEdge with an ort inference backend.
Definition lightly_edge_sdk.h:213
auto embed_frame(const Frame &frame) const -> std::vector< float >
Embed an RGB image.
Definition lightly_edge_sdk.h:231
auto embedding_dimension() const noexcept -> size_t
Get the embedding dimension.
Definition lightly_edge_sdk.h:263
Namespace with core LightlyEdge SDK functionality.
Definition lightly_edge_error.h:15
auto value_or_throw(tl::expected< T, lightly_edge_sdk::Error > &&result) -> T
Convenience function to unwrap a result of lightly_edge_sdk::LightlyEdge functions.
Definition lightly_edge_error.h:79
Holds information whether a frame should be selected or not.
Definition lightly_edge_rs_bindings.h:72
Definition lightly_edge_rs_bindings.h:77
Holds information whether a frame should be selected or not.
Definition lightly_edge_rs_bindings.h:60
Definition lightly_edge_rs_bindings.h:44
Definition lightly_edge_rs_bindings.h:49
A patch specified by relative coordinates.
Definition lightly_edge_rs_bindings.h:100
Holds information whether a frame should be selected or not.
Definition lightly_edge_rs_bindings.h:66
Error struct to hold error code and message.
Definition lightly_edge_error.h:20
Frame data for LightlyEdge.
Definition lightly_edge_sdk.h:25
size_t width_
Width of the image.
Definition lightly_edge_sdk.h:29
void * rgbImageData_
Pointer to the RGB image data.
Definition lightly_edge_sdk.h:42
size_t height_
Height of the image.
Definition lightly_edge_sdk.h:34
Frame(size_t width, size_t height, void *rgbImageData)
Construct a new Frame object.
Definition lightly_edge_sdk.h:52
Selection information about a processed frame.
Definition lightly_edge_sdk.h:84
std::vector< DiversitySelectInfo > diversity
The diversity selection info for each diversity strategy in the order of registration.
Definition lightly_edge_sdk.h:89
std::vector< SimilaritySelectInfo > similarity
The similarity selection info for each similarity strategy in the order of registration.
Definition lightly_edge_sdk.h:95
std::vector< AdaptiveDiversitySelectInfo > adaptive_diversity
The adaptive diversity selection info for each adaptive diversity strategy in the order of registrati...
Definition lightly_edge_sdk.h:101
std::vector< DetectionSelectInfo > detection
The detection selection info for each detection strategy in the order of registration.
Definition lightly_edge_sdk.h:107