#include <lightly_edge_sdk.h>

Public Member Functions
	~LightlyEdge ()
	Deallocate the memory associated with a LightlyEdge.

	LightlyEdge (LightlyEdge &&other) noexcept
	Move constructor.

auto	operator= (LightlyEdge &&other) noexcept -> LightlyEdge &
	Move assignment operator.

	LightlyEdge (const LightlyEdge &other)=delete
	Copy constructor (disallowed).

auto	operator= (const LightlyEdge &other) -> LightlyEdge &=delete
	Copy assignment operator (disallowed).

auto	embed_frame (const Frame &frame) const -> std::vector< float >
	Embed an RGB image.

auto	embed_frame_noexcept (const Frame &frame) const noexcept -> tl::expected< std::vector< float >, Error >
	Embed an RGB image.

auto	embedding_dimension () const noexcept -> size_t
	Get the embedding dimension.

auto	embed_texts (const std::vector< std::string > &texts) const -> std::vector< std::vector< float > >
	Embed a list of text strings.

auto	embed_texts_noexcept (const std::vector< std::string > &texts) const noexcept -> tl::expected< std::vector< std::vector< float > >, Error >
	Embed a list of text strings.

void	insert_into_embedding_database (const std::vector< float > &embedding) const
	Insert an embedding into the database.

auto	insert_into_embedding_database_noexcept (const std::vector< float > &embedding) const noexcept -> tl::expected< void, Error >
	Insert an embedding into the database.

auto	clear_embedding_database () const -> void
	Clear the embedding database.

auto	clear_embedding_database_noexcept () const noexcept -> tl::expected< void, Error >
	Clear the embedding database.

auto	num_diversity_strategies () const noexcept -> size_t
	Get the number of registered diversity strategies.

auto	num_similarity_strategies () const noexcept -> size_t
	Get the number of registered similarity strategies.

auto	num_adaptive_diversity_strategies () const noexcept -> size_t
	Get the number of registered adaptive diversity strategies.

auto	num_detection_strategies () const noexcept -> size_t
	Get the number of registered detection strategies.

auto	register_diversity_strategy (float min_distance) const -> void
	Register a diversity strategy.

auto	register_diversity_strategy_noexcept (float min_distance) const noexcept -> tl::expected< void, Error >
	Register a diversity strategy, noexcept version.

auto	register_similarity_strategy (const std::vector< float > &query_embedding, float max_distance) const -> void
	Register a similarity strategy.

auto	register_similarity_strategy_noexcept (const std::vector< float > &query_embedding, float max_distance) const noexcept -> tl::expected< void, Error >
	Register a similarity strategy, noexcept version.

auto	register_adaptive_diversity_strategy (float target_ratio, size_t buffer_max_length=BUFFER_MAX_LENGTH) const -> void
	Register an adaptive diversity strategy.

auto	register_adaptive_diversity_strategy_noexcept (float target_ratio, size_t buffer_max_length=BUFFER_MAX_LENGTH) const noexcept -> tl::expected< void, Error >
	Register an adaptive diversity strategy, no except version.

auto	register_detection_strategy (uint32_t class_id, int32_t subclass_id, float threshold) const -> void
	Register a detection strategy.

auto	register_detection_strategy_noexcept (uint32_t class_id, int32_t subclass_id, float threshold) const noexcept -> tl::expected< void, Error >
	Register a detection strategy, noexcept version.

void	clear_strategies ()
	Clear all registered selection strategies.

auto	embed_patches (const Frame &frame) const -> std::vector< std::vector< float > >
	Embed patches of an image.

auto	embed_patches_noexcept (const Frame &frame) const noexcept -> tl::expected< std::vector< std::vector< float > >, Error >
	Embed patches of an image, noexcept version.

auto	set_embedding_patches (const std::vector< PatchCoordsRel > &patches) const -> void
	Set the embedding patches.

auto	set_embedding_patches_noexcept (const std::vector< PatchCoordsRel > &patches) const noexcept -> tl::expected< void, Error >
	Set the embedding patches, noexcept version.

void	reset_embedding_patches ()
	Reset the embedding patches to full image patch.

auto	should_select (const std::vector< float > &embedding, const std::vector< ObjectDetection > &detections) const -> SelectInfo
	Check if a frame should be selected.

auto	should_select_noexcept (const std::vector< float > &embedding, const std::vector< ObjectDetection > &detections) const noexcept -> tl::expected< SelectInfo, Error >
	Check if a frame should be selected.

auto	should_select_with_patches (const std::vector< std::vector< float > > &embeddings, const std::vector< ObjectDetection > &detections) const -> SelectInfo
	Check if a frame should be selected taking patches into account.

auto	should_select_with_patches_noexcept (const std::vector< std::vector< float > > &embeddings, const std::vector< ObjectDetection > &detections) const noexcept -> tl::expected< SelectInfo, Error >
	Check if a frame should be selected taking patches into account.

auto	detect_noexcept (const Frame &frame) const noexcept -> tl::expected< std::vector< ObjectDetection >, Error >
	Run object detection on a frame.

auto	detect (const Frame &frame) const -> std::vector< ObjectDetection >
	Run object detection on a frame.

auto	detection_class_labels () const noexcept -> std::vector< std::string >
	Get labels of object detection classes.

auto	detection_subclass_labels (uint32_t class_id) const noexcept -> std::vector< std::string >
	Get the subclass labels for a specified detection class.

Static Public Member Functions
static auto	new_from_tar (const char *archive_path, LightlyEdgeConfig config) -> LightlyEdge
	Initialize LightlyEdge with an ort embedding inference backend.

static auto	new_from_tar_noexcept (const char *archive_path, LightlyEdgeConfig config) noexcept -> tl::expected< LightlyEdge, Error >
	Initialize LightlyEdge with an ort embedding inference backend.

Detailed Description

LightlyEdge.

This class provides access to the main functionalities of the LightlyEdge SDK. It provides methods to embed images and texts and make decisions about which frames to select.

Member Function Documentation

◆ clear_embedding_database()

auto lightly_edge_sdk::LightlyEdge::clear_embedding_database ( ) const -> void

inline

Clear the embedding database.

Returns: Void or throw an exception.

See also: clear_embedding_database_noexcept

◆ clear_embedding_database_noexcept()

auto lightly_edge_sdk::LightlyEdge::clear_embedding_database_noexcept ( ) const -> tl::expected<void, Error>

inlinenoexcept

Clear the embedding database.

Returns: Void or an error.

◆ detect()

auto lightly_edge_sdk::LightlyEdge::detect ( const Frame & frame ) const -> std::vector<ObjectDetection>

inline

Run object detection on a frame.

◆ detect_noexcept()

auto lightly_edge_sdk::LightlyEdge::detect_noexcept ( const Frame & frame ) const -> tl::expected<std::vector<ObjectDetection>, Error>

inlinenoexcept

Run object detection on a frame.

◆ detection_class_labels()

auto lightly_edge_sdk::LightlyEdge::detection_class_labels ( ) const -> std::vector<std::string>

inlinenoexcept

Get labels of object detection classes.

Class IDs contained in ObjectDetection.class_id also have names (or labels). This methods returns the list of class labels, starting from ID 0.

Returns: Labels of object detection classes. Label in position i (0-indexed) of the vector corresponds to class ID equal to i.

See also: detect; detect_noexcept

◆ detection_subclass_labels()

auto lightly_edge_sdk::LightlyEdge::detection_subclass_labels ( uint32_t class_id ) const -> std::vector<std::string>

inlinenoexcept

Get the subclass labels for a specified detection class.

Subclass IDs returned as ObjectDetection.subclass_id are generated by a classification model specific for each detection class. This methods returns the list mapping the IDs to subclass labels.

Parameters

class_id Class ID of the detection class.

Returns: Class labels for the detection class. Label in position i (0-indexed) of the vector corresponds to class ID equal to i. The list is empty if the class has no subclasses.

See also: detect; detect_noexcept

◆ embed_frame()

auto lightly_edge_sdk::LightlyEdge::embed_frame ( const Frame & frame ) const -> std::vector<float>

inline

Embed an RGB image.

Parameters

frame Frame to embed.

Returns: An embedding vector or throw an exception

◆ embed_frame_noexcept()

auto lightly_edge_sdk::LightlyEdge::embed_frame_noexcept ( const Frame & frame ) const -> tl::expected<std::vector<float>, Error>

inlinenoexcept

Embed an RGB image.

Parameters

frame Frame to embed.

Returns: An embedding vector or an error.

◆ embed_patches()

auto lightly_edge_sdk::LightlyEdge::embed_patches ( const Frame & frame ) const -> std::vector<std::vector<float>>

inline

Embed patches of an image.

This method extracts patches from an image and returns embeddings.

Parameters

frame	Frame to embed.
num_patches	Number of patches to get

Example

// Set the patches coordinates.
lightly_edge.set_embedding_patches({
    {0.0, 0.0, 0.2, 0.2},
    {0.5, 0.5, 0.5, 0.5},
});
 
// Embed patches and get embeddings.
embeddings = lightly_edge.embed_patches(
    imageRgb
);

Returns: A vector of vectors or throw an exception. Each inner vector is the embedding of one of the patches.

◆ embed_patches_noexcept()

auto lightly_edge_sdk::LightlyEdge::embed_patches_noexcept ( const Frame & frame ) const -> tl::expected<std::vector<std::vector<float>>, Error>

inlinenoexcept

Embed patches of an image, noexcept version.

This method extracts patches from an image and returns embeddings.

Parameters

frame	Frame to embed.
num_patches	Number of patches to get

Example

// Set the patches coordinates.
lightly_edge.set_embedding_patches({
    {0.0, 0.0, 0.2, 0.2},
    {0.5, 0.5, 0.5, 0.5},
});
 
// Embed patches and get embeddings.
embeddings = lightly_edge.embed_patches(
    imageRgb
);

Returns: A vector of vectors or an error. Each inner vector is the embedding of one of the patches.

◆ embed_texts()

auto lightly_edge_sdk::LightlyEdge::embed_texts ( const std::vector< std::string > & texts ) const -> std::vector<std::vector<float>>

inline

Embed a list of text strings.

This method processes a list of text strings, generating an embedding for each text. The embeddings are returned as a vector of vectors, where each inner vector represents the embedding of a single text.

Parameters

texts A vector of strings representing the texts to be embedded.

Returns: A vector of vectors or throw an exception. Each inner vector is the embedding of one of the input texts.

◆ embed_texts_noexcept()

auto lightly_edge_sdk::LightlyEdge::embed_texts_noexcept ( const std::vector< std::string > & texts ) const -> tl::expected<std::vector<std::vector<float>>, Error>

inlinenoexcept

Embed a list of text strings.

This method processes a list of text strings, generating an embedding for each text. The embeddings are returned as a vector of vectors, where each inner vector represents the embedding of a single text.

Parameters

texts A vector of strings representing the texts to be embedded.

Returns: A vector of vectors or an error. Each inner vector is the embedding of one of the input texts.

◆ embedding_dimension()

auto lightly_edge_sdk::LightlyEdge::embedding_dimension ( ) const -> size_t

inlinenoexcept

Get the embedding dimension.

Returns: The embedding dimension

◆ insert_into_embedding_database()

void lightly_edge_sdk::LightlyEdge::insert_into_embedding_database ( const std::vector< float > & embedding ) const

inline

Insert an embedding into the database.

Embedding database is used by the diversity strategy. If an embedding is inserted into the database the strategy will not anymore select frames with embeddings similar to it. You should call insert_into_embedding_database for frames marked by should_select with diversity strategy.

The function can fail if:

The database is full (maximum number of embeddings reached).
The embedding vector has a different dimension than other embeddings in the database.

Parameters

embedding Embedding of a frame. Can be obtained using embed_frame.

Returns: Void or throw an exception.

See also: embed_frame; register_diversity_strategy

◆ insert_into_embedding_database_noexcept()

auto lightly_edge_sdk::LightlyEdge::insert_into_embedding_database_noexcept ( const std::vector< float > & embedding ) const -> tl::expected<void, Error>

inlinenoexcept

Insert an embedding into the database.

Embedding database is used by the diversity strategy. If an embedding is inserted into the database the strategy will not anymore select frames with embeddings similar to it. You should call insert_into_embedding_database_noexcept for frames marked by should_select_noexcept with diversity strategy.

The function can fail if:

The database is full (maximum number of embeddings reached).
The embedding vector has a different dimension than other embeddings in the database.

Parameters

embedding Embedding of a frame. Can be obtained using embed_frame.

Returns: Void or an error.

See also: embed_frame_noexcept; register_diversity_strategy_noexcept

◆ new_from_tar()

static auto lightly_edge_sdk::LightlyEdge::new_from_tar	(	const char *	archive_path,
		LightlyEdgeConfig	config ) -> LightlyEdge

inlinestatic

Initialize LightlyEdge with an ort embedding inference backend.

Parameters

archive_path Path to an embedding model tar file.

Returns: A new LightlyEdge or throw an error.

◆ new_from_tar_noexcept()

static auto lightly_edge_sdk::LightlyEdge::new_from_tar_noexcept	(	const char *	archive_path,
		LightlyEdgeConfig	config ) -> tl::expected<LightlyEdge, Error>

inlinestaticnoexcept

Initialize LightlyEdge with an ort embedding inference backend.

Parameters

archive_path Path to an embedding model tar file.

Returns: A new LightlyEdge or an error.

◆ num_adaptive_diversity_strategies()

auto lightly_edge_sdk::LightlyEdge::num_adaptive_diversity_strategies ( ) const -> size_t

inlinenoexcept

Get the number of registered adaptive diversity strategies.

Returns: The number of adaptive diversity strategies.

◆ num_detection_strategies()

auto lightly_edge_sdk::LightlyEdge::num_detection_strategies ( ) const -> size_t

inlinenoexcept

Get the number of registered detection strategies.

Returns: The number of detection strategies.

◆ num_diversity_strategies()

auto lightly_edge_sdk::LightlyEdge::num_diversity_strategies ( ) const -> size_t

inlinenoexcept

Get the number of registered diversity strategies.

Returns: The number of diversity strategies.

◆ num_similarity_strategies()

auto lightly_edge_sdk::LightlyEdge::num_similarity_strategies ( ) const -> size_t

inlinenoexcept

Get the number of registered similarity strategies.

Returns: The number of similarity strategies.

◆ register_adaptive_diversity_strategy()

auto lightly_edge_sdk::LightlyEdge::register_adaptive_diversity_strategy	(	float	target_ratio,
		size_t	buffer_max_length = BUFFER_MAX_LENGTH ) const -> void

inline

Register an adaptive diversity strategy.

It selects frames based on the distance to the database and the target ratio of frames that should be selected.

Note: You can register multiple adaptive diversity strategies.

Example

// Register an adaptive diversity strategy.
lightly_edge.register_adaptive_diversity_strategy(0.3, 500);
lightly_edge.register_adaptive_diversity_strategy(0.4, 3000);
 
// Embed a frame.
std::vector<float> image_embedding = lightly_edge.embed_frame(frame);
 
// Check if the frame is selected by any strategy.
SelectInfo select_info = lightly_edge.should_select(image_embedding, {});
bool first_selected = select_info.adaptive_diversity[0].should_select;
bool second_selected = select_info.adaptive_diversity[1].should_select;

Parameters

target_ratio	The ratio of frames that should be selected. Must be between 0 and 1.
buffer_max_length	(Optional, Default 3000) The maximum number of embeddings that can be stored in the buffer.

Returns: Void or throw an exception.

See also: should_select

◆ register_adaptive_diversity_strategy_noexcept()

auto lightly_edge_sdk::LightlyEdge::register_adaptive_diversity_strategy_noexcept	(	float	target_ratio,
		size_t	buffer_max_length = BUFFER_MAX_LENGTH ) const -> tl::expected<void, Error>

inlinenoexcept

Register an adaptive diversity strategy, no except version.

It selects frames based on the distance to the database and the target ratio of frames that should be selected.

Note: You can register multiple adaptive diversity strategies.

Example

// Register an adaptive diversity strategy.
lightly_edge.register_adaptive_diversity_strategy_noexcept(0.3, 500).value();
lightly_edge.register_adaptive_diversity_strategy_noexcept(0.4, 3000).value();
 
// Embed a frame.
std::vector<float> image_embedding =
lightly_edge.embed_frame_noexcept(frame).value();
 
// Check if the frame is selected by any strategy.
SelectInfo select_info =
lightly_edge.should_select_noexcept(image_embedding, {}).value();
 
bool first_selected = select_info.adaptive_diversity[0].should_select;
bool second_selcted = select_info.adaptive_diversity[1].should_select;

Parameters

target_ratio	The ratio of frames that should be selected. Must be between 0 and 1.
buffer_max_length	(Optional, Default 3000) The maximum number of embeddings that can be stored in the buffer.

Returns: Void or an error.

See also: should_select_noexcept

◆ register_detection_strategy()

auto lightly_edge_sdk::LightlyEdge::register_detection_strategy	(	uint32_t	class_id,
		int32_t	subclass_id,
		float	threshold ) const -> void

inline

Register a detection strategy.

A detection strategy selects frames that contain object detections that match the specified query. Only frames with a confidence higher than the threshold will be selected.

Note: You can register multiple detection strategies.

Example

lightly_edge.register_detection_strategy(1, -1, 0.5);
lightly_edge.register_detection_strategy(2, 1, 0.5);
 
// Embed a frame and run detections.
std::vector<float> image_embedding = lightly_edge.embed_frame(frame);
std::vector<ObjectDetection> detections = lightly_edge.detect(frame);
 
// Check if the frame is selected by any strategy.
SelectInfo select_info = lightly_edge.should_select(image_embedding,
detections).value();
bool is_selected = select_info.detection[0].should_select;

Parameters

class_id	Class identifier of the search object.
subclass_id	Subclass identifier, use -1 to ignore the subclass
threshold	Frame matches the query if any detection has higher confidence than threshold

Returns: Void or throw an exception.

See also: should_select

◆ register_detection_strategy_noexcept()

auto lightly_edge_sdk::LightlyEdge::register_detection_strategy_noexcept	(	uint32_t	class_id,
		int32_t	subclass_id,
		float	threshold ) const -> tl::expected<void, Error>

inlinenoexcept

Register a detection strategy, noexcept version.

A detection strategy selects frames that contain object detections that match the specified query. Only frames with a confidence higher than the threshold will be selected.

Note: You can register multiple detection strategies.

Example

lightly_edge.register_detection_strategy_noexcept(1, -1, 0.5);
lightly_edge.register_detection_strategy_noexcept(2, 1, 0.5);
 
// Embed a frame and run detections.
std::vector<float> image_embedding = lightly_edge.embed_frame_noexcept(frame);
std::vector<ObjectDetection> detections = lightly_edge.detect_noexcept(frame);
 
// Check if the frame is selected by any strategy.
SelectInfo select_info = lightly_edge.should_select_noexcept(image_embedding,
detections).value();
bool is_selected = select_info.detection[0].should_select;

Parameters

class_id	Class identifier of the search object.
subclass_id	Subclass identifier, use -1 to ignore the subclass
threshold	Frame matches the query if any detection has higher confidence than threshold

Returns: Void or an error.

See also: should_select_noexcept

◆ register_diversity_strategy()

auto lightly_edge_sdk::LightlyEdge::register_diversity_strategy ( float min_distance ) const -> void

inline

Register a diversity strategy.

A diversity strategy selects frames that are different from previously selected frames. The minimum distance parameter must be between 0 and 1 and controls how different the new frame must be from the previously selected frames: Only frames with a distance greater than the minimum distance will be selected. To remember selected frames across multiple calls, the user must fill the database with embeddings by calling insert_into_embedding_database.

Example

// Register a diversity strategy.
lightly_edge.register_diversity_strategy(0.1);
 
// Embed a frame.
std::vector<float> image_embedding = lightly_edge.embed_frame(frame);
 
// Check if the frame is selected by any strategy.
SelectInfo select_info = lightly_edge.should_select(image_embedding, {}).value();
 
// Add to the embedding database if the image is selected.
DiversitySelectInfo diversity_select_info = select_info.diversity[0];
if (diversity_select_info.should_select) {
 lightly_edge.insert_into_embedding_database(image_embedding);
}

Parameters

min_distance Only frame embeddings with a distance greater than min_distance to all embedding in the database will be selected. Must be between 0 and 1.
A higher value will result in fewer frames being selected.

Returns: Void or throws an exception.

See also: embed_frame; insert_into_embedding_database; should_select

◆ register_diversity_strategy_noexcept()

auto lightly_edge_sdk::LightlyEdge::register_diversity_strategy_noexcept ( float min_distance ) const -> tl::expected<void, Error>

inlinenoexcept

Register a diversity strategy, noexcept version.

A diversity strategy selects frames that are different from previously selected frames. The minimum distance parameter must be between 0 and 1 and controls how different the new frame must be from the previously selected frames: Only frames with a distance greater than the minimum distance will be selected. To remember selected frames across multiple calls, the user must fill the database with embeddings by calling insert_into_embedding_database_noexcept.

Example

// Warning: This is an example code. Don't use `.value()` in production code.
// Instead, test if the result has value and handle the error case.
 
// Register a diversity strategy.
lightly_edge.register_diversity_strategy_noexcept(0.1).value();
 
// Embed a frame.
std::vector<float> image_embedding =
lightly_edge.embed_frame_noexcept(frame).value();
 
// Check if the frame is selected by any strategy.
SelectInfo select_info =
lightly_edge.should_select_noexcept(image_embedding, {}).value();
 
// Add to the embedding database if the image is selected.
DiversitySelectInfo diversity_select_info = select_info.diversity[0];
if (diversity_select_info.should_select) {
 lightly_edge.insert_into_embedding_database_noexcept(image_embedding).value();
}

Parameters

min_distance Only frame embeddings with a distance greater than min_distance to all embedding in the database will be selected. Must be between 0 and 1.
A higher value will result in fewer frames being selected.

Returns: Void or an error.

See also: embed_frame_noexcept; insert_into_embedding_database_noexcept; should_select_noexcept

◆ register_similarity_strategy()

auto lightly_edge_sdk::LightlyEdge::register_similarity_strategy	(	const std::vector< float > &	query_embedding,
		float	max_distance ) const -> void

inline

Register a similarity strategy.

A similarity strategy selects frames that are similar to a query embedding. The max_distance parameter must be between 0 and 1 and controls how similar the embeddings must be for a frame to be selected: Only frames with a distance less than the max_distance will be selected.

Note: You can register multiple similarity strategies.

Example

// Embed query texts.
std::vector<std::vector<float>> query_embeddings =
lightly_edge.embed_texts({"cat", "dog"});
 
// Register a similarity strategy for each query.
lightly_edge.register_similarity_strategy(query_embeddings[0], 0.3); // cat
lightly_edge.register_similarity_strategy(query_embeddings[1], 0.4); // dog
 
// Embed a frame.
std::vector<float> image_embedding = lightly_edge.embed_frame(frame);
 
// Check if the frame is selected by any strategy.
SelectInfo select_info = lightly_edge.should_select(image_embedding, {}).value();
bool is_cat = select_info.similarity[0].should_select;
bool is_dog = select_info.similarity[1].should_select;

Parameters

query_embedding	Query embedding vector. Can be obtained using `embed_texts`.
max_distance	Maximum distance to the query embedding for a frame to be selected. Must be between 0 and 1. A higher value will result in more frames being selected.

Returns: Void or throw an exception.

See also: embed_texts; should_select

◆ register_similarity_strategy_noexcept()

auto lightly_edge_sdk::LightlyEdge::register_similarity_strategy_noexcept	(	const std::vector< float > &	query_embedding,
		float	max_distance ) const -> tl::expected<void, Error>

inlinenoexcept

Register a similarity strategy, noexcept version.

A similarity strategy selects frames that are similar to a query embedding. The max_distance parameter must be between 0 and 1 and controls how similar the embeddings must be for a frame to be selected: Only frames with a distance less than the max_distance will be selected.

Note: You can register multiple similarity strategies.

Example

// Warning: This is an example code. Don't use `.value()` in production code.
// Instead, test if the result has value and handle the error case.
 
// Embed query texts.
std::vector<std::vector<float>> query_embeddings =
lightly_edge.embed_texts_noexcept({"cat", "dog"}).value();
 
// Register a similarity strategy for each query.
lightly_edge.register_similarity_strategy_noexcept(
  query_embeddings[0], 0.3
).value(); // cat
lightly_edge.register_similarity_strategy_noexcept(
  query_embeddings[1], 0.4
).value(); // dog
 
// Embed a frame.
std::vector<float> image_embedding =
   lightly_edge.embed_frame_noexcept(frame).value();
 
// Check if the frame is selected by any strategy.
SelectInfo select_info =
lightly_edge.should_select_noexcept(image_embedding, {}).value();
bool is_cat = select_info.similarity[0].should_select;
bool is_dog = select_info.similarity[1].should_select;

Parameters

query_embedding	Query embedding vector. Can be obtained using `embed_texts`.
max_distance	Maximum distance to the query embedding for a frame to be selected. Must be between 0 and 1. A higher value will result in more frames being selected.

Returns: Void or an error.

See also: embed_texts_noexcept; should_select_noexcept

◆ set_embedding_patches()

auto lightly_edge_sdk::LightlyEdge::set_embedding_patches ( const std::vector< PatchCoordsRel > & patches ) const -> void

inline

Set the embedding patches.

Allows setting custom patches that are used for embedding parts of an image with embed_patches function.

Parameters

patches A vector of PatchCoordsRel structs representing the patches. The PatchCoordsRel struct is a vector representing relative coordinates of the patch like {x, y, width, height}.

Example

   try {
       lightly_edge.set_embedding_patches({
           {0.0, 0.0, 0.5, 0.5}, // Top-left quarter of the image, 0.5 width means
50% of the image width
       });
   } catch (const std::runtime_error& e) {
       // Handle the exception, log or rethrow as needed
       std::cerr << "Error: " << e.what() << std::endl;
   }

Returns: Void or an exception is thrown on failure.

◆ set_embedding_patches_noexcept()

auto lightly_edge_sdk::LightlyEdge::set_embedding_patches_noexcept ( const std::vector< PatchCoordsRel > & patches ) const -> tl::expected<void, Error>

inlinenoexcept

Set the embedding patches, noexcept version.

Allows setting custom patches that are used for embedding parts of an image with embed_patches function.

Parameters

patches A vector of PatchCoordsRel structs representing the patches.
The PatchCoordsRel struct is a vector representing relative coordinates of the patch like {x, y, width, height}.

Example

   auto result = lightly_edge.set_embedding_patches_noexcept({
       {0.0, 0.0, 0.5, 0.5}, // Top-left quarter of the image, 0.5 width means 50%
of the image width
   });

Returns: Void or an error.

◆ should_select()

auto lightly_edge_sdk::LightlyEdge::should_select	(	const std::vector< float > &	embedding,
		const std::vector< ObjectDetection > &	detections ) const -> SelectInfo

inline

Check if a frame should be selected.

This method checks if a frame should be selected based on its embedding and the registered strategies. In particular, it iterates over all registered diversity, similarity and detection strategies and creates a selection info object of for each strategy. The strategy selection info object contains the selection status (true or false) and metadata about the selection.

Example

// Register a diversity strategy.
lightly_edge.register_diversity_strategy(0.1);
 
// Register a detection strategy.
lightly_edge.register_detection_strategy(1, -1, 0.5);
 
// Embed a frame and run detections.
std::vector<float> image_embedding = lightly_edge.embed_frame(frame);
std::vector<ObjectDetection> detections = lightly_edge.detect(frame);
 
// Check if the frame is selected by any strategy.
SelectInfo select_info = lightly_edge.should_select(image_embedding, detections);
 
assert(select_info.diversity.size() == 1); // One diversity strategy registered.
assert(select_info.similarity.size() == 0); // No similarity strategies registered.
assert(select_info.detection.size() == 1); // One detection strategy registered.
bool is_selected = select_info.diversity[0].selected ||
                   select_info.detection[0].selected;

Parameters

embedding	Embedding of the frame. Can be obtained using `embed_frame`.
detections	Detections of the frame.

Returns: A selection info object or throw lightly_edge_sdk::Exception. The selection info object contains the selection status and metadata for each registered strategy.

See also: embed_frame; register_diversity_strategy; register_similarity_strategy; SelectInfo

◆ should_select_noexcept()

auto lightly_edge_sdk::LightlyEdge::should_select_noexcept	(	const std::vector< float > &	embedding,
		const std::vector< ObjectDetection > &	detections ) const -> tl::expected<SelectInfo, Error>

inlinenoexcept

Check if a frame should be selected.

This method checks if a frame should be selected based on its embedding and the registered strategies. In particular, it iterates over all registered diversity, similarity and detection strategies and creates a selection info object of for each strategy. The strategy selection info object contains the selection status (true or false) and metadata about the selection.

Example

// Warning: This is an example code. Don't use `.value()` in production code.
// Instead, test if the result has value and handle the error case.
 
// Register a diversity strategy.
lightly_edge.register_diversity_strategy_noexcept(0.1).value();
 
// Embed a frame and run detections.
std::vector<float> image_embedding =
lightly_edge.embed_frame_noexcept(frame).value();
std::vector<ObjectDetection> detections =
lightly_edge.detect_noexcept(frame).value();
 
// Check if the frame is selected by any strategy.
SelectInfo select_info =
lightly_edge.should_select_noexcept(image_embedding, detections).value();
 
assert(select_info.diversity.size() == 1); // One diversity strategy registered.
assert(select_info.similarity.size() == 0); // No similarity strategies registered.
assert(select_info.adaptive_diversity.size() == 0); // No adaptive_diversity
assert(select_info.detection.size() == 0); // No detection strategies registered.
 bool is_selected = select_info.diversity[0].selected;

Parameters

embedding	Embedding of the frame. Can be obtained using `embed_frame`.
detections	Detections of the frame.

Returns: A selection info object or an error. The selection info object contains the selection status and metadata for each registered strategy.

See also: embed_frame_noexcept; register_diversity_strategy_noexcept; register_similarity_strategy_noexcept; SelectInfo

◆ should_select_with_patches()

auto lightly_edge_sdk::LightlyEdge::should_select_with_patches	(	const std::vector< std::vector< float > > &	embeddings,
		const std::vector< ObjectDetection > &	detections ) const -> SelectInfo

inline

Check if a frame should be selected taking patches into account.

This method checks if a frame should be selected based on patch embeddings and the registered strategies. In particular, it iterates over all registered strategies and creates a selection info object for each strategy. The strategy selection info object contains the selection status (true or false) and metadata about the selection.

Example

// Register a diversity strategy.
lightly_edge.register_diversity_strategy(0.1);
 
// Set patches.
lightly_edge.set_embedding_patches({
    {0.0, 0.0, 0.2, 0.2},
    {0.5, 0.5, 0.5, 0.5},
});
 
// Embed patches and run detections.
std::vector<std::vector<float>> embeddings = lightly_edge.embed_patches(frame);
std::vector<ObjectDetection> detections = lightly_edge.detect(frame);
 
// Check if the frame is selected by any strategy.
SelectInfo select_info =
lightly_edge.should_select_with_patches(embeddings, detections).value();
 
assert(select_info.diversity.size() == 1); // One diversity strategy registered.
bool is_selected = select_info.diversity[0].selected;

Parameters

embeddings	Embeddings of the frame patches. Can be obtained using `embed_patches`.
detections	Detections of the frame.

Returns: A selection info object or throw lightly_edge_sdk::Exception. The selection info object contains the selection status and metadata for each registered strategy.

See also: embed_patches; set_embedding_patches; register_diversity_strategy; SelectInfo

◆ should_select_with_patches_noexcept()

auto lightly_edge_sdk::LightlyEdge::should_select_with_patches_noexcept	(	const std::vector< std::vector< float > > &	embeddings,
		const std::vector< ObjectDetection > &	detections ) const -> tl::expected<SelectInfo, Error>

inlinenoexcept

Check if a frame should be selected taking patches into account.

This method checks if a frame should be selected based on patch embeddings and the registered strategies. In particular, it iterates over all registered strategies and creates a selection info object for each strategy. The strategy selection info object contains the selection status (true or false) and metadata about the selection.

Example

// Warning: This is an example code. Don't use `.value()` in production code.
// Instead, test if the result has value and handle the error case.
 
// Register a diversity strategy.
lightly_edge.register_diversity_strategy_noexcept(0.1).value();
 
// Set patches.
lightly_edge.set_embedding_patches_noexcept({
    {0.0, 0.0, 0.2, 0.2},
    {0.5, 0.5, 0.5, 0.5},
}).value();
 
// Embed patches and run detections.
std::vector<std::vector<float>> embeddings =
    lightly_edge.embed_patches_noexcept(frame).value();
std::vector<ObjectDetection> detections =
lightly_edge.detect_noexcept(frame).value();
 
// Check if the frame is selected by any strategy.
SelectInfo select_info =
lightly_edge.should_select_with_patches_noexcept(embeddings, detections).value();
 
assert(select_info.diversity.size() == 1); // One diversity strategy registered.
bool is_selected = select_info.diversity[0].selected;

Parameters

embeddings	Embeddings of the frame patches. Can be obtained using `embed_patches_noexcept`.
detections	Detections of the frame.

Returns: A selection info object or an error. The selection info object contains the selection status and metadata for each registered strategy.

See also: embed_patches_noexcept; set_embedding_patches_noexcept; register_diversity_strategy_noexcept; SelectInfo

The documentation for this class was generated from the following file:

lightly_edge_sdk.h

Public Member Functions

Static Public Member Functions

Detailed Description

Member Function Documentation

◆ clear_embedding_database()

◆ clear_embedding_database_noexcept()

◆ detect()

◆ detect_noexcept()

◆ detection_class_labels()

◆ detection_subclass_labels()

◆ embed_frame()

◆ embed_frame_noexcept()

◆ embed_patches()

◆ embed_patches_noexcept()

◆ embed_texts()

◆ embed_texts_noexcept()

◆ embedding_dimension()

◆ insert_into_embedding_database()

◆ insert_into_embedding_database_noexcept()

◆ new_from_tar()

◆ new_from_tar_noexcept()

◆ num_adaptive_diversity_strategies()

◆ num_detection_strategies()

◆ num_diversity_strategies()

◆ num_similarity_strategies()

◆ register_adaptive_diversity_strategy()

◆ register_adaptive_diversity_strategy_noexcept()

◆ register_detection_strategy()

◆ register_detection_strategy_noexcept()

◆ register_diversity_strategy()

◆ register_diversity_strategy_noexcept()

◆ register_similarity_strategy()

◆ register_similarity_strategy_noexcept()

◆ set_embedding_patches()

◆ set_embedding_patches_noexcept()

◆ should_select()

◆ should_select_noexcept()

◆ should_select_with_patches()

◆ should_select_with_patches_noexcept()