LightlyEdge C++ SDK
|
#include <lightly_edge_sdk.h>
Public Member Functions | |
~LightlyEdge () | |
Deallocate the memory associated with a LightlyEdge. | |
LightlyEdge (LightlyEdge &&other) noexcept | |
Move constructor. | |
auto | operator= (LightlyEdge &&other) noexcept -> LightlyEdge & |
Move assignment operator. | |
LightlyEdge (const LightlyEdge &other)=delete | |
Copy constructor (disallowed). | |
auto | operator= (const LightlyEdge &other) -> LightlyEdge &=delete |
Copy assignment operator (disallowed). | |
auto | embed_frame (const Frame &frame) const -> std::vector< float > |
Embed an RGB image. | |
auto | embed_frame_noexcept (const Frame &frame) const noexcept -> tl::expected< std::vector< float >, Error > |
Embed an RGB image. | |
auto | embedding_dimension () const noexcept -> size_t |
Get the embedding dimension. | |
auto | embed_texts (const std::vector< std::string > &texts) const -> std::vector< std::vector< float > > |
Embed a list of text strings. | |
auto | embed_texts_noexcept (const std::vector< std::string > &texts) const noexcept -> tl::expected< std::vector< std::vector< float > >, Error > |
Embed a list of text strings. | |
void | insert_into_embedding_database (const std::vector< float > &embedding) const |
Insert an embedding into the database. | |
auto | insert_into_embedding_database_noexcept (const std::vector< float > &embedding) const noexcept -> tl::expected< void, Error > |
Insert an embedding into the database. | |
auto | clear_embedding_database () const -> void |
Clear the embedding database. | |
auto | clear_embedding_database_noexcept () const noexcept -> tl::expected< void, Error > |
Clear the embedding database. | |
auto | num_diversity_strategies () const noexcept -> size_t |
Get the number of registered diversity strategies. | |
auto | num_similarity_strategies () const noexcept -> size_t |
Get the number of registered similarity strategies. | |
auto | num_adaptive_diversity_strategies () const noexcept -> size_t |
Get the number of registered adaptive diversity strategies. | |
auto | num_detection_strategies () const noexcept -> size_t |
Get the number of registered detection strategies. | |
auto | register_diversity_strategy (float min_distance) const -> void |
Register a diversity strategy. | |
auto | register_diversity_strategy_noexcept (float min_distance) const noexcept -> tl::expected< void, Error > |
Register a diversity strategy, noexcept version. | |
auto | register_similarity_strategy (const std::vector< float > &query_embedding, float max_distance) const -> void |
Register a similarity strategy. | |
auto | register_similarity_strategy_noexcept (const std::vector< float > &query_embedding, float max_distance) const noexcept -> tl::expected< void, Error > |
Register a similarity strategy, noexcept version. | |
auto | register_adaptive_diversity_strategy (float target_ratio, size_t buffer_max_length=BUFFER_MAX_LENGTH) const -> void |
Register an adaptive diversity strategy. | |
auto | register_adaptive_diversity_strategy_noexcept (float target_ratio, size_t buffer_max_length=BUFFER_MAX_LENGTH) const noexcept -> tl::expected< void, Error > |
Register an adaptive diversity strategy, no except version. | |
auto | register_detection_strategy (uint32_t class_id, int32_t subclass_id, float threshold) const -> void |
Register a detection strategy. | |
auto | register_detection_strategy_noexcept (uint32_t class_id, int32_t subclass_id, float threshold) const noexcept -> tl::expected< void, Error > |
Register a detection strategy, noexcept version. | |
void | clear_strategies () |
Clear all registered selection strategies. | |
auto | embed_patches (const Frame &frame) const -> std::vector< std::vector< float > > |
Embed patches of an image. | |
auto | embed_patches_noexcept (const Frame &frame) const noexcept -> tl::expected< std::vector< std::vector< float > >, Error > |
Embed patches of an image, noexcept version. | |
auto | set_embedding_patches (const std::vector< PatchCoordsRel > &patches) const -> void |
Set the embedding patches. | |
auto | set_embedding_patches_noexcept (const std::vector< PatchCoordsRel > &patches) const noexcept -> tl::expected< void, Error > |
Set the embedding patches, noexcept version. | |
void | reset_embedding_patches () |
Reset the embedding patches to full image patch. | |
auto | should_select (const std::vector< float > &embedding, const std::vector< ObjectDetection > &detections) const -> SelectInfo |
Check if a frame should be selected. | |
auto | should_select_noexcept (const std::vector< float > &embedding, const std::vector< ObjectDetection > &detections) const noexcept -> tl::expected< SelectInfo, Error > |
Check if a frame should be selected. | |
auto | should_select_with_patches (const std::vector< std::vector< float > > &embeddings, const std::vector< ObjectDetection > &detections) const -> SelectInfo |
Check if a frame should be selected taking patches into account. | |
auto | should_select_with_patches_noexcept (const std::vector< std::vector< float > > &embeddings, const std::vector< ObjectDetection > &detections) const noexcept -> tl::expected< SelectInfo, Error > |
Check if a frame should be selected taking patches into account. | |
auto | detect_noexcept (const Frame &frame) const noexcept -> tl::expected< std::vector< ObjectDetection >, Error > |
Run object detection on a frame. | |
auto | detect (const Frame &frame) const -> std::vector< ObjectDetection > |
Run object detection on a frame. | |
auto | detection_class_labels () const noexcept -> std::vector< std::string > |
Get labels of object detection classes. | |
auto | detection_subclass_labels (uint32_t class_id) const noexcept -> std::vector< std::string > |
Get the subclass labels for a specified detection class. | |
Static Public Member Functions | |
static auto | new_from_tar (const char *archive_path, LightlyEdgeConfig config) -> LightlyEdge |
Initialize LightlyEdge with an ort embedding inference backend. | |
static auto | new_from_tar_noexcept (const char *archive_path, LightlyEdgeConfig config) noexcept -> tl::expected< LightlyEdge, Error > |
Initialize LightlyEdge with an ort embedding inference backend. | |
This class provides access to the main functionalities of the LightlyEdge SDK. It provides methods to embed images and texts and make decisions about which frames to select.
|
inline |
Clear the embedding database.
|
inlinenoexcept |
Clear the embedding database.
|
inline |
Run object detection on a frame.
Run object detection on a frame.
|
inlinenoexcept |
Run object detection on a frame.
Run object detection on a frame.
|
inlinenoexcept |
Get labels of object detection classes.
Class IDs contained in ObjectDetection.class_id
also have names (or labels). This methods returns the list of class labels, starting from ID 0.
i
(0-indexed) of the vector corresponds to class ID equal to i
.
|
inlinenoexcept |
Get the subclass labels for a specified detection class.
Subclass IDs returned as ObjectDetection.subclass_id
are generated by a classification model specific for each detection class. This methods returns the list mapping the IDs to subclass labels.
class_id | Class ID of the detection class. |
i
(0-indexed) of the vector corresponds to class ID equal to i
. The list is empty if the class has no subclasses.
|
inline |
Embed an RGB image.
frame | Frame to embed. |
|
inline |
Embed patches of an image.
This method extracts patches from an image and returns embeddings.
frame | Frame to embed. |
num_patches | Number of patches to get |
Example
|
inlinenoexcept |
Embed patches of an image, noexcept version.
This method extracts patches from an image and returns embeddings.
frame | Frame to embed. |
num_patches | Number of patches to get |
Example
|
inline |
Embed a list of text strings.
This method processes a list of text strings, generating an embedding for each text. The embeddings are returned as a vector of vectors, where each inner vector represents the embedding of a single text.
texts | A vector of strings representing the texts to be embedded. |
|
inlinenoexcept |
Embed a list of text strings.
This method processes a list of text strings, generating an embedding for each text. The embeddings are returned as a vector of vectors, where each inner vector represents the embedding of a single text.
texts | A vector of strings representing the texts to be embedded. |
|
inlinenoexcept |
Get the embedding dimension.
|
inline |
Insert an embedding into the database.
Embedding database is used by the diversity strategy. If an embedding is inserted into the database the strategy will not anymore select frames with embeddings similar to it. You should call insert_into_embedding_database
for frames marked by should_select
with diversity strategy.
The function can fail if:
embedding | Embedding of a frame. Can be obtained using embed_frame . |
|
inlinenoexcept |
Insert an embedding into the database.
Embedding database is used by the diversity strategy. If an embedding is inserted into the database the strategy will not anymore select frames with embeddings similar to it. You should call insert_into_embedding_database_noexcept
for frames marked by should_select_noexcept
with diversity strategy.
The function can fail if:
embedding | Embedding of a frame. Can be obtained using embed_frame . |
|
inlinestatic |
Initialize LightlyEdge with an ort embedding inference backend.
archive_path | Path to an embedding model tar file. |
|
inlinestaticnoexcept |
Initialize LightlyEdge with an ort embedding inference backend.
archive_path | Path to an embedding model tar file. |
|
inlinenoexcept |
Get the number of registered adaptive diversity strategies.
|
inlinenoexcept |
Get the number of registered detection strategies.
|
inlinenoexcept |
Get the number of registered diversity strategies.
|
inlinenoexcept |
Get the number of registered similarity strategies.
|
inline |
Register an adaptive diversity strategy.
It selects frames based on the distance to the database and the target ratio of frames that should be selected.
Example
target_ratio | The ratio of frames that should be selected. Must be between 0 and 1. |
buffer_max_length | (Optional, Default 3000) The maximum number of embeddings that can be stored in the buffer. |
|
inlinenoexcept |
Register an adaptive diversity strategy, no except version.
It selects frames based on the distance to the database and the target ratio of frames that should be selected.
Example
target_ratio | The ratio of frames that should be selected. Must be between 0 and 1. |
buffer_max_length | (Optional, Default 3000) The maximum number of embeddings that can be stored in the buffer. |
|
inline |
Register a detection strategy.
A detection strategy selects frames that contain object detections that match the specified query. Only frames with a confidence higher than the threshold
will be selected.
Example
class_id | Class identifier of the search object. |
subclass_id | Subclass identifier, use -1 to ignore the subclass |
threshold | Frame matches the query if any detection has higher confidence than threshold |
|
inlinenoexcept |
Register a detection strategy, noexcept version.
A detection strategy selects frames that contain object detections that match the specified query. Only frames with a confidence higher than the threshold
will be selected.
Example
class_id | Class identifier of the search object. |
subclass_id | Subclass identifier, use -1 to ignore the subclass |
threshold | Frame matches the query if any detection has higher confidence than threshold |
|
inline |
Register a diversity strategy.
A diversity strategy selects frames that are different from previously selected frames. The minimum distance parameter must be between 0 and 1 and controls how different the new frame must be from the previously selected frames: Only frames with a distance greater than the minimum distance will be selected. To remember selected frames across multiple calls, the user must fill the database with embeddings by calling insert_into_embedding_database
.
Example
min_distance | Only frame embeddings with a distance greater than min_distance to all embedding in the database will be selected. Must be between 0 and 1. A higher value will result in fewer frames being selected. |
|
inlinenoexcept |
Register a diversity strategy, noexcept version.
A diversity strategy selects frames that are different from previously selected frames. The minimum distance parameter must be between 0 and 1 and controls how different the new frame must be from the previously selected frames: Only frames with a distance greater than the minimum distance will be selected. To remember selected frames across multiple calls, the user must fill the database with embeddings by calling insert_into_embedding_database_noexcept
.
Example
min_distance | Only frame embeddings with a distance greater than min_distance to all embedding in the database will be selected. Must be between 0 and 1. A higher value will result in fewer frames being selected. |
|
inline |
Register a similarity strategy.
A similarity strategy selects frames that are similar to a query embedding. The max_distance
parameter must be between 0 and 1 and controls how similar the embeddings must be for a frame to be selected: Only frames with a distance less than the max_distance
will be selected.
Example
query_embedding | Query embedding vector. Can be obtained using embed_texts . |
max_distance | Maximum distance to the query embedding for a frame to be selected. Must be between 0 and 1. A higher value will result in more frames being selected. |
|
inlinenoexcept |
Register a similarity strategy, noexcept version.
A similarity strategy selects frames that are similar to a query embedding. The max_distance
parameter must be between 0 and 1 and controls how similar the embeddings must be for a frame to be selected: Only frames with a distance less than the max_distance
will be selected.
Example
query_embedding | Query embedding vector. Can be obtained using embed_texts . |
max_distance | Maximum distance to the query embedding for a frame to be selected. Must be between 0 and 1. A higher value will result in more frames being selected. |
|
inline |
Set the embedding patches.
Allows setting custom patches that are used for embedding parts of an image with embed_patches function.
patches | A vector of PatchCoordsRel structs representing the patches. The PatchCoordsRel struct is a vector representing relative coordinates of the patch like {x, y, width, height}. |
Example
|
inlinenoexcept |
Set the embedding patches, noexcept version.
Allows setting custom patches that are used for embedding parts of an image with embed_patches function.
patches | A vector of PatchCoordsRel structs representing the patches. The PatchCoordsRel struct is a vector representing relative coordinates of the patch like {x, y, width, height}. |
Example
|
inline |
Check if a frame should be selected.
This method checks if a frame should be selected based on its embedding and the registered strategies. In particular, it iterates over all registered diversity, similarity and detection strategies and creates a selection info object of for each strategy. The strategy selection info object contains the selection status (true or false) and metadata about the selection.
Example
embedding | Embedding of the frame. Can be obtained using embed_frame . |
detections | Detections of the frame. |
|
inlinenoexcept |
Check if a frame should be selected.
This method checks if a frame should be selected based on its embedding and the registered strategies. In particular, it iterates over all registered diversity, similarity and detection strategies and creates a selection info object of for each strategy. The strategy selection info object contains the selection status (true or false) and metadata about the selection.
Example
embedding | Embedding of the frame. Can be obtained using embed_frame . |
detections | Detections of the frame. |
|
inline |
Check if a frame should be selected taking patches into account.
This method checks if a frame should be selected based on patch embeddings and the registered strategies. In particular, it iterates over all registered strategies and creates a selection info object for each strategy. The strategy selection info object contains the selection status (true or false) and metadata about the selection.
Example
embeddings | Embeddings of the frame patches. Can be obtained using embed_patches . |
detections | Detections of the frame. |
|
inlinenoexcept |
Check if a frame should be selected taking patches into account.
This method checks if a frame should be selected based on patch embeddings and the registered strategies. In particular, it iterates over all registered strategies and creates a selection info object for each strategy. The strategy selection info object contains the selection status (true or false) and metadata about the selection.
Example
embeddings | Embeddings of the frame patches. Can be obtained using embed_patches_noexcept . |
detections | Detections of the frame. |