LightlyEdge C++ SDK
Loading...
Searching...
No Matches
lightly_edge_sdk::LightlyEdge Class Reference

LightlyEdge. More...

#include <lightly_edge_sdk.h>

Public Member Functions

 ~LightlyEdge ()
 Deallocate the memory associated with a LightlyEdge.
 
 LightlyEdge (LightlyEdge &&other) noexcept
 Move constructor.
 
auto operator= (LightlyEdge &&other) noexcept -> LightlyEdge &
 Move assignment operator.
 
 LightlyEdge (const LightlyEdge &other)=delete
 Copy constructor (disallowed).
 
auto operator= (const LightlyEdge &other) -> LightlyEdge &=delete
 Copy assignment operator (disallowed).
 
auto embed_frame (const Frame &frame) const -> std::vector< float >
 Embed an RGB image.
 
auto embed_frame_noexcept (const Frame &frame) const noexcept -> tl::expected< std::vector< float >, Error >
 Embed an RGB image.
 
auto embedding_dimension () const noexcept -> size_t
 Get the embedding dimension.
 
auto embed_texts (const std::vector< std::string > &texts) const -> std::vector< std::vector< float > >
 Embed a list of text strings.
 
auto embed_texts_noexcept (const std::vector< std::string > &texts) const noexcept -> tl::expected< std::vector< std::vector< float > >, Error >
 Embed a list of text strings.
 
void insert_into_embedding_database (const std::vector< float > &embedding) const
 Insert an embedding into the database.
 
auto insert_into_embedding_database_noexcept (const std::vector< float > &embedding) const noexcept -> tl::expected< void, Error >
 Insert an embedding into the database.
 
auto clear_embedding_database () const -> void
 Clear the embedding database.
 
auto clear_embedding_database_noexcept () const noexcept -> tl::expected< void, Error >
 Clear the embedding database.
 
auto num_diversity_strategies () const noexcept -> size_t
 Get the number of registered diversity strategies.
 
auto num_similarity_strategies () const noexcept -> size_t
 Get the number of registered similarity strategies.
 
auto num_adaptive_diversity_strategies () const noexcept -> size_t
 Get the number of registered adaptive diversity strategies.
 
auto num_detection_strategies () const noexcept -> size_t
 Get the number of registered detection strategies.
 
auto register_diversity_strategy (float min_distance) const -> void
 Register a diversity strategy.
 
auto register_diversity_strategy_noexcept (float min_distance) const noexcept -> tl::expected< void, Error >
 Register a diversity strategy, noexcept version.
 
auto register_similarity_strategy (const std::vector< float > &query_embedding, float max_distance) const -> void
 Register a similarity strategy.
 
auto register_similarity_strategy_noexcept (const std::vector< float > &query_embedding, float max_distance) const noexcept -> tl::expected< void, Error >
 Register a similarity strategy, noexcept version.
 
auto register_adaptive_diversity_strategy (float target_ratio, size_t buffer_max_length=BUFFER_MAX_LENGTH) const -> void
 Register an adaptive diversity strategy.
 
auto register_adaptive_diversity_strategy_noexcept (float target_ratio, size_t buffer_max_length=BUFFER_MAX_LENGTH) const noexcept -> tl::expected< void, Error >
 Register an adaptive diversity strategy, no except version.
 
auto register_detection_strategy (uint32_t class_id, int32_t subclass_id, float threshold) const -> void
 Register a detection strategy.
 
auto register_detection_strategy_noexcept (uint32_t class_id, int32_t subclass_id, float threshold) const noexcept -> tl::expected< void, Error >
 Register a detection strategy, noexcept version.
 
void clear_strategies ()
 Clear all registered selection strategies.
 
auto embed_patches (const Frame &frame) const -> std::vector< std::vector< float > >
 Embed patches of an image.
 
auto embed_patches_noexcept (const Frame &frame) const noexcept -> tl::expected< std::vector< std::vector< float > >, Error >
 Embed patches of an image, noexcept version.
 
auto set_embedding_patches (const std::vector< PatchCoordsRel > &patches) const -> void
 Set the embedding patches.
 
auto set_embedding_patches_noexcept (const std::vector< PatchCoordsRel > &patches) const noexcept -> tl::expected< void, Error >
 Set the embedding patches, noexcept version.
 
void reset_embedding_patches ()
 Reset the embedding patches to full image patch.
 
auto should_select (const std::vector< float > &embedding, const std::vector< ObjectDetection > &detections) const -> SelectInfo
 Check if a frame should be selected.
 
auto should_select_noexcept (const std::vector< float > &embedding, const std::vector< ObjectDetection > &detections) const noexcept -> tl::expected< SelectInfo, Error >
 Check if a frame should be selected.
 
auto should_select_with_patches (const std::vector< std::vector< float > > &embeddings, const std::vector< ObjectDetection > &detections) const -> SelectInfo
 Check if a frame should be selected taking patches into account.
 
auto should_select_with_patches_noexcept (const std::vector< std::vector< float > > &embeddings, const std::vector< ObjectDetection > &detections) const noexcept -> tl::expected< SelectInfo, Error >
 Check if a frame should be selected taking patches into account.
 
auto detect_noexcept (const Frame &frame) const noexcept -> tl::expected< std::vector< ObjectDetection >, Error >
 Run object detection on a frame.
 
auto detect (const Frame &frame) const -> std::vector< ObjectDetection >
 Run object detection on a frame.
 
auto detection_class_labels () const noexcept -> std::vector< std::string >
 Get labels of object detection classes.
 
auto detection_subclass_labels (uint32_t class_id) const noexcept -> std::vector< std::string >
 Get the subclass labels for a specified detection class.
 

Static Public Member Functions

static auto new_from_tar (const char *archive_path, LightlyEdgeConfig config) -> LightlyEdge
 Initialize LightlyEdge with an ort embedding inference backend.
 
static auto new_from_tar_noexcept (const char *archive_path, LightlyEdgeConfig config) noexcept -> tl::expected< LightlyEdge, Error >
 Initialize LightlyEdge with an ort embedding inference backend.
 

Detailed Description

LightlyEdge.

This class provides access to the main functionalities of the LightlyEdge SDK. It provides methods to embed images and texts and make decisions about which frames to select.

Member Function Documentation

◆ clear_embedding_database()

auto lightly_edge_sdk::LightlyEdge::clear_embedding_database ( ) const -> void
inline

Clear the embedding database.

Returns
Void or throw an exception.
See also
clear_embedding_database_noexcept

◆ clear_embedding_database_noexcept()

auto lightly_edge_sdk::LightlyEdge::clear_embedding_database_noexcept ( ) const -> tl::expected<void, Error>
inlinenoexcept

Clear the embedding database.

Returns
Void or an error.

◆ detect()

auto lightly_edge_sdk::LightlyEdge::detect ( const Frame & frame) const -> std::vector<ObjectDetection>
inline

Run object detection on a frame.

Run object detection on a frame.

◆ detect_noexcept()

auto lightly_edge_sdk::LightlyEdge::detect_noexcept ( const Frame & frame) const -> tl::expected<std::vector<ObjectDetection>, Error>
inlinenoexcept

Run object detection on a frame.

Run object detection on a frame.

◆ detection_class_labels()

auto lightly_edge_sdk::LightlyEdge::detection_class_labels ( ) const -> std::vector<std::string>
inlinenoexcept

Get labels of object detection classes.

Class IDs contained in ObjectDetection.class_id also have names (or labels). This methods returns the list of class labels, starting from ID 0.

Returns
Labels of object detection classes. Label in position i (0-indexed) of the vector corresponds to class ID equal to i.
See also
detect
detect_noexcept

◆ detection_subclass_labels()

auto lightly_edge_sdk::LightlyEdge::detection_subclass_labels ( uint32_t class_id) const -> std::vector<std::string>
inlinenoexcept

Get the subclass labels for a specified detection class.

Subclass IDs returned as ObjectDetection.subclass_id are generated by a classification model specific for each detection class. This methods returns the list mapping the IDs to subclass labels.

Parameters
class_idClass ID of the detection class.
Returns
Class labels for the detection class. Label in position i (0-indexed) of the vector corresponds to class ID equal to i. The list is empty if the class has no subclasses.
See also
detect
detect_noexcept

◆ embed_frame()

auto lightly_edge_sdk::LightlyEdge::embed_frame ( const Frame & frame) const -> std::vector<float>
inline

Embed an RGB image.

Parameters
frameFrame to embed.
Returns
An embedding vector or throw an exception

◆ embed_frame_noexcept()

auto lightly_edge_sdk::LightlyEdge::embed_frame_noexcept ( const Frame & frame) const -> tl::expected<std::vector<float>, Error>
inlinenoexcept

Embed an RGB image.

Parameters
frameFrame to embed.
Returns
An embedding vector or an error.

◆ embed_patches()

auto lightly_edge_sdk::LightlyEdge::embed_patches ( const Frame & frame) const -> std::vector<std::vector<float>>
inline

Embed patches of an image.

This method extracts patches from an image and returns embeddings.

Parameters
frameFrame to embed.
num_patchesNumber of patches to get

Example

// Set the patches coordinates.
lightly_edge.set_embedding_patches({
{0.0, 0.0, 0.2, 0.2},
{0.5, 0.5, 0.5, 0.5},
});
// Embed patches and get embeddings.
embeddings = lightly_edge.embed_patches(
imageRgb
);
Returns
A vector of vectors or throw an exception. Each inner vector is the embedding of one of the patches.

◆ embed_patches_noexcept()

auto lightly_edge_sdk::LightlyEdge::embed_patches_noexcept ( const Frame & frame) const -> tl::expected<std::vector<std::vector<float>>, Error>
inlinenoexcept

Embed patches of an image, noexcept version.

This method extracts patches from an image and returns embeddings.

Parameters
frameFrame to embed.
num_patchesNumber of patches to get

Example

// Set the patches coordinates.
lightly_edge.set_embedding_patches({
{0.0, 0.0, 0.2, 0.2},
{0.5, 0.5, 0.5, 0.5},
});
// Embed patches and get embeddings.
embeddings = lightly_edge.embed_patches(
imageRgb
);
Returns
A vector of vectors or an error. Each inner vector is the embedding of one of the patches.

◆ embed_texts()

auto lightly_edge_sdk::LightlyEdge::embed_texts ( const std::vector< std::string > & texts) const -> std::vector<std::vector<float>>
inline

Embed a list of text strings.

This method processes a list of text strings, generating an embedding for each text. The embeddings are returned as a vector of vectors, where each inner vector represents the embedding of a single text.

Parameters
textsA vector of strings representing the texts to be embedded.
Returns
A vector of vectors or throw an exception. Each inner vector is the embedding of one of the input texts.

◆ embed_texts_noexcept()

auto lightly_edge_sdk::LightlyEdge::embed_texts_noexcept ( const std::vector< std::string > & texts) const -> tl::expected<std::vector<std::vector<float>>, Error>
inlinenoexcept

Embed a list of text strings.

This method processes a list of text strings, generating an embedding for each text. The embeddings are returned as a vector of vectors, where each inner vector represents the embedding of a single text.

Parameters
textsA vector of strings representing the texts to be embedded.
Returns
A vector of vectors or an error. Each inner vector is the embedding of one of the input texts.

◆ embedding_dimension()

auto lightly_edge_sdk::LightlyEdge::embedding_dimension ( ) const -> size_t
inlinenoexcept

Get the embedding dimension.

Returns
The embedding dimension

◆ insert_into_embedding_database()

void lightly_edge_sdk::LightlyEdge::insert_into_embedding_database ( const std::vector< float > & embedding) const
inline

Insert an embedding into the database.

Embedding database is used by the diversity strategy. If an embedding is inserted into the database the strategy will not anymore select frames with embeddings similar to it. You should call insert_into_embedding_database for frames marked by should_select with diversity strategy.

The function can fail if:

  • The database is full (maximum number of embeddings reached).
  • The embedding vector has a different dimension than other embeddings in the database.
Parameters
embeddingEmbedding of a frame. Can be obtained using embed_frame.
Returns
Void or throw an exception.
See also
embed_frame
register_diversity_strategy

◆ insert_into_embedding_database_noexcept()

auto lightly_edge_sdk::LightlyEdge::insert_into_embedding_database_noexcept ( const std::vector< float > & embedding) const -> tl::expected<void, Error>
inlinenoexcept

Insert an embedding into the database.

Embedding database is used by the diversity strategy. If an embedding is inserted into the database the strategy will not anymore select frames with embeddings similar to it. You should call insert_into_embedding_database_noexcept for frames marked by should_select_noexcept with diversity strategy.

The function can fail if:

  • The database is full (maximum number of embeddings reached).
  • The embedding vector has a different dimension than other embeddings in the database.
Parameters
embeddingEmbedding of a frame. Can be obtained using embed_frame.
Returns
Void or an error.
See also
embed_frame_noexcept
register_diversity_strategy_noexcept

◆ new_from_tar()

static auto lightly_edge_sdk::LightlyEdge::new_from_tar ( const char * archive_path,
LightlyEdgeConfig config ) -> LightlyEdge
inlinestatic

Initialize LightlyEdge with an ort embedding inference backend.

Parameters
archive_pathPath to an embedding model tar file.
Returns
A new LightlyEdge or throw an error.

◆ new_from_tar_noexcept()

static auto lightly_edge_sdk::LightlyEdge::new_from_tar_noexcept ( const char * archive_path,
LightlyEdgeConfig config ) -> tl::expected<LightlyEdge, Error>
inlinestaticnoexcept

Initialize LightlyEdge with an ort embedding inference backend.

Parameters
archive_pathPath to an embedding model tar file.
Returns
A new LightlyEdge or an error.

◆ num_adaptive_diversity_strategies()

auto lightly_edge_sdk::LightlyEdge::num_adaptive_diversity_strategies ( ) const -> size_t
inlinenoexcept

Get the number of registered adaptive diversity strategies.

Returns
The number of adaptive diversity strategies.

◆ num_detection_strategies()

auto lightly_edge_sdk::LightlyEdge::num_detection_strategies ( ) const -> size_t
inlinenoexcept

Get the number of registered detection strategies.

Returns
The number of detection strategies.

◆ num_diversity_strategies()

auto lightly_edge_sdk::LightlyEdge::num_diversity_strategies ( ) const -> size_t
inlinenoexcept

Get the number of registered diversity strategies.

Returns
The number of diversity strategies.

◆ num_similarity_strategies()

auto lightly_edge_sdk::LightlyEdge::num_similarity_strategies ( ) const -> size_t
inlinenoexcept

Get the number of registered similarity strategies.

Returns
The number of similarity strategies.

◆ register_adaptive_diversity_strategy()

auto lightly_edge_sdk::LightlyEdge::register_adaptive_diversity_strategy ( float target_ratio,
size_t buffer_max_length = BUFFER_MAX_LENGTH ) const -> void
inline

Register an adaptive diversity strategy.

It selects frames based on the distance to the database and the target ratio of frames that should be selected.

Note
You can register multiple adaptive diversity strategies.

Example

// Register an adaptive diversity strategy.
lightly_edge.register_adaptive_diversity_strategy(0.3, 500);
lightly_edge.register_adaptive_diversity_strategy(0.4, 3000);
// Embed a frame.
std::vector<float> image_embedding = lightly_edge.embed_frame(frame);
// Check if the frame is selected by any strategy.
SelectInfo select_info = lightly_edge.should_select(image_embedding, {});
bool first_selected = select_info.adaptive_diversity[0].should_select;
bool second_selected = select_info.adaptive_diversity[1].should_select;
Selection information about a processed frame.
Definition lightly_edge_sdk.h:83
std::vector< AdaptiveDiversitySelectInfo > adaptive_diversity
The adaptive diversity selection info for each adaptive diversity strategy in the order of registrati...
Definition lightly_edge_sdk.h:100
Parameters
target_ratioThe ratio of frames that should be selected. Must be between 0 and 1.
buffer_max_length(Optional, Default 3000) The maximum number of embeddings that can be stored in the buffer.
Returns
Void or throw an exception.
See also
should_select

◆ register_adaptive_diversity_strategy_noexcept()

auto lightly_edge_sdk::LightlyEdge::register_adaptive_diversity_strategy_noexcept ( float target_ratio,
size_t buffer_max_length = BUFFER_MAX_LENGTH ) const -> tl::expected<void, Error>
inlinenoexcept

Register an adaptive diversity strategy, no except version.

It selects frames based on the distance to the database and the target ratio of frames that should be selected.

Note
You can register multiple adaptive diversity strategies.

Example

// Register an adaptive diversity strategy.
lightly_edge.register_adaptive_diversity_strategy_noexcept(0.3, 500).value();
lightly_edge.register_adaptive_diversity_strategy_noexcept(0.4, 3000).value();
// Embed a frame.
std::vector<float> image_embedding =
lightly_edge.embed_frame_noexcept(frame).value();
// Check if the frame is selected by any strategy.
SelectInfo select_info =
lightly_edge.should_select_noexcept(image_embedding, {}).value();
bool first_selected = select_info.adaptive_diversity[0].should_select;
bool second_selcted = select_info.adaptive_diversity[1].should_select;
Parameters
target_ratioThe ratio of frames that should be selected. Must be between 0 and 1.
buffer_max_length(Optional, Default 3000) The maximum number of embeddings that can be stored in the buffer.
Returns
Void or an error.
See also
should_select_noexcept

◆ register_detection_strategy()

auto lightly_edge_sdk::LightlyEdge::register_detection_strategy ( uint32_t class_id,
int32_t subclass_id,
float threshold ) const -> void
inline

Register a detection strategy.

A detection strategy selects frames that contain object detections that match the specified query. Only frames with a confidence higher than the threshold will be selected.

Note
You can register multiple detection strategies.

Example

lightly_edge.register_detection_strategy(1, -1, 0.5);
lightly_edge.register_detection_strategy(2, 1, 0.5);
// Embed a frame and run detections.
std::vector<float> image_embedding = lightly_edge.embed_frame(frame);
std::vector<ObjectDetection> detections = lightly_edge.detect(frame);
// Check if the frame is selected by any strategy.
SelectInfo select_info = lightly_edge.should_select(image_embedding,
detections).value();
bool is_selected = select_info.detection[0].should_select;
std::vector< DetectionSelectInfo > detection
The detection selection info for each detection strategy in the order of registration.
Definition lightly_edge_sdk.h:106
Parameters
class_idClass identifier of the search object.
subclass_idSubclass identifier, use -1 to ignore the subclass
thresholdFrame matches the query if any detection has higher confidence than threshold
Returns
Void or throw an exception.
See also
should_select

◆ register_detection_strategy_noexcept()

auto lightly_edge_sdk::LightlyEdge::register_detection_strategy_noexcept ( uint32_t class_id,
int32_t subclass_id,
float threshold ) const -> tl::expected<void, Error>
inlinenoexcept

Register a detection strategy, noexcept version.

A detection strategy selects frames that contain object detections that match the specified query. Only frames with a confidence higher than the threshold will be selected.

Note
You can register multiple detection strategies.

Example

lightly_edge.register_detection_strategy_noexcept(1, -1, 0.5);
lightly_edge.register_detection_strategy_noexcept(2, 1, 0.5);
// Embed a frame and run detections.
std::vector<float> image_embedding = lightly_edge.embed_frame_noexcept(frame);
std::vector<ObjectDetection> detections = lightly_edge.detect_noexcept(frame);
// Check if the frame is selected by any strategy.
SelectInfo select_info = lightly_edge.should_select_noexcept(image_embedding,
detections).value();
bool is_selected = select_info.detection[0].should_select;
Parameters
class_idClass identifier of the search object.
subclass_idSubclass identifier, use -1 to ignore the subclass
thresholdFrame matches the query if any detection has higher confidence than threshold
Returns
Void or an error.
See also
should_select_noexcept

◆ register_diversity_strategy()

auto lightly_edge_sdk::LightlyEdge::register_diversity_strategy ( float min_distance) const -> void
inline

Register a diversity strategy.

A diversity strategy selects frames that are different from previously selected frames. The minimum distance parameter must be between 0 and 1 and controls how different the new frame must be from the previously selected frames: Only frames with a distance greater than the minimum distance will be selected. To remember selected frames across multiple calls, the user must fill the database with embeddings by calling insert_into_embedding_database.

Example

// Register a diversity strategy.
lightly_edge.register_diversity_strategy(0.1);
// Embed a frame.
std::vector<float> image_embedding = lightly_edge.embed_frame(frame);
// Check if the frame is selected by any strategy.
SelectInfo select_info = lightly_edge.should_select(image_embedding, {}).value();
// Add to the embedding database if the image is selected.
DiversitySelectInfo diversity_select_info = select_info.diversity[0];
if (diversity_select_info.should_select) {
lightly_edge.insert_into_embedding_database(image_embedding);
}
Holds information whether a frame should be selected or not.
Definition lightly_edge_rs_bindings.h:59
std::vector< DiversitySelectInfo > diversity
The diversity selection info for each diversity strategy in the order of registration.
Definition lightly_edge_sdk.h:88
Parameters
min_distanceOnly frame embeddings with a distance greater than min_distance to all embedding in the database will be selected. Must be between 0 and 1.
A higher value will result in fewer frames being selected.
Returns
Void or throws an exception.
See also
embed_frame
insert_into_embedding_database
should_select

◆ register_diversity_strategy_noexcept()

auto lightly_edge_sdk::LightlyEdge::register_diversity_strategy_noexcept ( float min_distance) const -> tl::expected<void, Error>
inlinenoexcept

Register a diversity strategy, noexcept version.

A diversity strategy selects frames that are different from previously selected frames. The minimum distance parameter must be between 0 and 1 and controls how different the new frame must be from the previously selected frames: Only frames with a distance greater than the minimum distance will be selected. To remember selected frames across multiple calls, the user must fill the database with embeddings by calling insert_into_embedding_database_noexcept.

Example

// Warning: This is an example code. Don't use `.value()` in production code.
// Instead, test if the result has value and handle the error case.
// Register a diversity strategy.
lightly_edge.register_diversity_strategy_noexcept(0.1).value();
// Embed a frame.
std::vector<float> image_embedding =
lightly_edge.embed_frame_noexcept(frame).value();
// Check if the frame is selected by any strategy.
SelectInfo select_info =
lightly_edge.should_select_noexcept(image_embedding, {}).value();
// Add to the embedding database if the image is selected.
DiversitySelectInfo diversity_select_info = select_info.diversity[0];
if (diversity_select_info.should_select) {
lightly_edge.insert_into_embedding_database_noexcept(image_embedding).value();
}
Parameters
min_distanceOnly frame embeddings with a distance greater than min_distance to all embedding in the database will be selected. Must be between 0 and 1.
A higher value will result in fewer frames being selected.
Returns
Void or an error.
See also
embed_frame_noexcept
insert_into_embedding_database_noexcept
should_select_noexcept

◆ register_similarity_strategy()

auto lightly_edge_sdk::LightlyEdge::register_similarity_strategy ( const std::vector< float > & query_embedding,
float max_distance ) const -> void
inline

Register a similarity strategy.

A similarity strategy selects frames that are similar to a query embedding. The max_distance parameter must be between 0 and 1 and controls how similar the embeddings must be for a frame to be selected: Only frames with a distance less than the max_distance will be selected.

Note
You can register multiple similarity strategies.

Example

// Embed query texts.
std::vector<std::vector<float>> query_embeddings =
lightly_edge.embed_texts({"cat", "dog"});
// Register a similarity strategy for each query.
lightly_edge.register_similarity_strategy(query_embeddings[0], 0.3); // cat
lightly_edge.register_similarity_strategy(query_embeddings[1], 0.4); // dog
// Embed a frame.
std::vector<float> image_embedding = lightly_edge.embed_frame(frame);
// Check if the frame is selected by any strategy.
SelectInfo select_info = lightly_edge.should_select(image_embedding, {}).value();
bool is_cat = select_info.similarity[0].should_select;
bool is_dog = select_info.similarity[1].should_select;
std::vector< SimilaritySelectInfo > similarity
The similarity selection info for each similarity strategy in the order of registration.
Definition lightly_edge_sdk.h:94
Parameters
query_embeddingQuery embedding vector. Can be obtained using embed_texts.
max_distanceMaximum distance to the query embedding for a frame to be selected. Must be between 0 and 1.
A higher value will result in more frames being selected.
Returns
Void or throw an exception.
See also
embed_texts
should_select

◆ register_similarity_strategy_noexcept()

auto lightly_edge_sdk::LightlyEdge::register_similarity_strategy_noexcept ( const std::vector< float > & query_embedding,
float max_distance ) const -> tl::expected<void, Error>
inlinenoexcept

Register a similarity strategy, noexcept version.

A similarity strategy selects frames that are similar to a query embedding. The max_distance parameter must be between 0 and 1 and controls how similar the embeddings must be for a frame to be selected: Only frames with a distance less than the max_distance will be selected.

Note
You can register multiple similarity strategies.

Example

// Warning: This is an example code. Don't use `.value()` in production code.
// Instead, test if the result has value and handle the error case.
// Embed query texts.
std::vector<std::vector<float>> query_embeddings =
lightly_edge.embed_texts_noexcept({"cat", "dog"}).value();
// Register a similarity strategy for each query.
lightly_edge.register_similarity_strategy_noexcept(
query_embeddings[0], 0.3
).value(); // cat
lightly_edge.register_similarity_strategy_noexcept(
query_embeddings[1], 0.4
).value(); // dog
// Embed a frame.
std::vector<float> image_embedding =
lightly_edge.embed_frame_noexcept(frame).value();
// Check if the frame is selected by any strategy.
SelectInfo select_info =
lightly_edge.should_select_noexcept(image_embedding, {}).value();
bool is_cat = select_info.similarity[0].should_select;
bool is_dog = select_info.similarity[1].should_select;
Parameters
query_embeddingQuery embedding vector. Can be obtained using embed_texts.
max_distanceMaximum distance to the query embedding for a frame to be selected. Must be between 0 and 1.
A higher value will result in more frames being selected.
Returns
Void or an error.
See also
embed_texts_noexcept
should_select_noexcept

◆ set_embedding_patches()

auto lightly_edge_sdk::LightlyEdge::set_embedding_patches ( const std::vector< PatchCoordsRel > & patches) const -> void
inline

Set the embedding patches.

Allows setting custom patches that are used for embedding parts of an image with embed_patches function.

Parameters
patchesA vector of PatchCoordsRel structs representing the patches. The PatchCoordsRel struct is a vector representing relative coordinates of the patch like {x, y, width, height}.

Example

try {
lightly_edge.set_embedding_patches({
{0.0, 0.0, 0.5, 0.5}, // Top-left quarter of the image, 0.5 width means
50% of the image width
});
} catch (const std::runtime_error& e) {
// Handle the exception, log or rethrow as needed
std::cerr << "Error: " << e.what() << std::endl;
}
Returns
Void or an exception is thrown on failure.

◆ set_embedding_patches_noexcept()

auto lightly_edge_sdk::LightlyEdge::set_embedding_patches_noexcept ( const std::vector< PatchCoordsRel > & patches) const -> tl::expected<void, Error>
inlinenoexcept

Set the embedding patches, noexcept version.

Allows setting custom patches that are used for embedding parts of an image with embed_patches function.

Parameters
patchesA vector of PatchCoordsRel structs representing the patches.
The PatchCoordsRel struct is a vector representing relative coordinates of the patch like {x, y, width, height}.

Example

auto result = lightly_edge.set_embedding_patches_noexcept({
{0.0, 0.0, 0.5, 0.5}, // Top-left quarter of the image, 0.5 width means 50%
of the image width
});
Returns
Void or an error.

◆ should_select()

auto lightly_edge_sdk::LightlyEdge::should_select ( const std::vector< float > & embedding,
const std::vector< ObjectDetection > & detections ) const -> SelectInfo
inline

Check if a frame should be selected.

This method checks if a frame should be selected based on its embedding and the registered strategies. In particular, it iterates over all registered diversity, similarity and detection strategies and creates a selection info object of for each strategy. The strategy selection info object contains the selection status (true or false) and metadata about the selection.

Example

// Register a diversity strategy.
lightly_edge.register_diversity_strategy(0.1);
// Register a detection strategy.
lightly_edge.register_detection_strategy(1, -1, 0.5);
// Embed a frame and run detections.
std::vector<float> image_embedding = lightly_edge.embed_frame(frame);
std::vector<ObjectDetection> detections = lightly_edge.detect(frame);
// Check if the frame is selected by any strategy.
SelectInfo select_info = lightly_edge.should_select(image_embedding, detections);
assert(select_info.diversity.size() == 1); // One diversity strategy registered.
assert(select_info.similarity.size() == 0); // No similarity strategies registered.
assert(select_info.detection.size() == 1); // One detection strategy registered.
bool is_selected = select_info.diversity[0].selected ||
select_info.detection[0].selected;
Parameters
embeddingEmbedding of the frame. Can be obtained using embed_frame.
detectionsDetections of the frame.
Returns
A selection info object or throw lightly_edge_sdk::Exception. The selection info object contains the selection status and metadata for each registered strategy.
See also
embed_frame
register_diversity_strategy
register_similarity_strategy
SelectInfo

◆ should_select_noexcept()

auto lightly_edge_sdk::LightlyEdge::should_select_noexcept ( const std::vector< float > & embedding,
const std::vector< ObjectDetection > & detections ) const -> tl::expected<SelectInfo, Error>
inlinenoexcept

Check if a frame should be selected.

This method checks if a frame should be selected based on its embedding and the registered strategies. In particular, it iterates over all registered diversity, similarity and detection strategies and creates a selection info object of for each strategy. The strategy selection info object contains the selection status (true or false) and metadata about the selection.

Example

// Warning: This is an example code. Don't use `.value()` in production code.
// Instead, test if the result has value and handle the error case.
// Register a diversity strategy.
lightly_edge.register_diversity_strategy_noexcept(0.1).value();
// Embed a frame and run detections.
std::vector<float> image_embedding =
lightly_edge.embed_frame_noexcept(frame).value();
std::vector<ObjectDetection> detections =
lightly_edge.detect_noexcept(frame).value();
// Check if the frame is selected by any strategy.
SelectInfo select_info =
lightly_edge.should_select_noexcept(image_embedding, detections).value();
assert(select_info.diversity.size() == 1); // One diversity strategy registered.
assert(select_info.similarity.size() == 0); // No similarity strategies registered.
assert(select_info.adaptive_diversity.size() == 0); // No adaptive_diversity
assert(select_info.detection.size() == 0); // No detection strategies registered.
bool is_selected = select_info.diversity[0].selected;
Parameters
embeddingEmbedding of the frame. Can be obtained using embed_frame.
detectionsDetections of the frame.
Returns
A selection info object or an error. The selection info object contains the selection status and metadata for each registered strategy.
See also
embed_frame_noexcept
register_diversity_strategy_noexcept
register_similarity_strategy_noexcept
SelectInfo

◆ should_select_with_patches()

auto lightly_edge_sdk::LightlyEdge::should_select_with_patches ( const std::vector< std::vector< float > > & embeddings,
const std::vector< ObjectDetection > & detections ) const -> SelectInfo
inline

Check if a frame should be selected taking patches into account.

This method checks if a frame should be selected based on patch embeddings and the registered strategies. In particular, it iterates over all registered strategies and creates a selection info object for each strategy. The strategy selection info object contains the selection status (true or false) and metadata about the selection.

Example

// Register a diversity strategy.
lightly_edge.register_diversity_strategy(0.1);
// Set patches.
lightly_edge.set_embedding_patches({
{0.0, 0.0, 0.2, 0.2},
{0.5, 0.5, 0.5, 0.5},
});
// Embed patches and run detections.
std::vector<std::vector<float>> embeddings = lightly_edge.embed_patches(frame);
std::vector<ObjectDetection> detections = lightly_edge.detect(frame);
// Check if the frame is selected by any strategy.
SelectInfo select_info =
lightly_edge.should_select_with_patches(embeddings, detections).value();
assert(select_info.diversity.size() == 1); // One diversity strategy registered.
bool is_selected = select_info.diversity[0].selected;
Parameters
embeddingsEmbeddings of the frame patches. Can be obtained using embed_patches.
detectionsDetections of the frame.
Returns
A selection info object or throw lightly_edge_sdk::Exception. The selection info object contains the selection status and metadata for each registered strategy.
See also
embed_patches
set_embedding_patches
register_diversity_strategy
SelectInfo

◆ should_select_with_patches_noexcept()

auto lightly_edge_sdk::LightlyEdge::should_select_with_patches_noexcept ( const std::vector< std::vector< float > > & embeddings,
const std::vector< ObjectDetection > & detections ) const -> tl::expected<SelectInfo, Error>
inlinenoexcept

Check if a frame should be selected taking patches into account.

This method checks if a frame should be selected based on patch embeddings and the registered strategies. In particular, it iterates over all registered strategies and creates a selection info object for each strategy. The strategy selection info object contains the selection status (true or false) and metadata about the selection.

Example

// Warning: This is an example code. Don't use `.value()` in production code.
// Instead, test if the result has value and handle the error case.
// Register a diversity strategy.
lightly_edge.register_diversity_strategy_noexcept(0.1).value();
// Set patches.
lightly_edge.set_embedding_patches_noexcept({
{0.0, 0.0, 0.2, 0.2},
{0.5, 0.5, 0.5, 0.5},
}).value();
// Embed patches and run detections.
std::vector<std::vector<float>> embeddings =
lightly_edge.embed_patches_noexcept(frame).value();
std::vector<ObjectDetection> detections =
lightly_edge.detect_noexcept(frame).value();
// Check if the frame is selected by any strategy.
SelectInfo select_info =
lightly_edge.should_select_with_patches_noexcept(embeddings, detections).value();
assert(select_info.diversity.size() == 1); // One diversity strategy registered.
bool is_selected = select_info.diversity[0].selected;
Parameters
embeddingsEmbeddings of the frame patches. Can be obtained using embed_patches_noexcept.
detectionsDetections of the frame.
Returns
A selection info object or an error. The selection info object contains the selection status and metadata for each registered strategy.
See also
embed_patches_noexcept
set_embedding_patches_noexcept
register_diversity_strategy_noexcept
SelectInfo

The documentation for this class was generated from the following file: