Unlocking the Power of Multimodal Embeddings

1
# Import the necessary packages
2
import os
3
import base64
4
5
6
# Defining the function to convert an image to a base 64 Data URL
7
def image_to_base64_data_url(image_path):
8
    _, file_extension = os.path.splitext(image_path)
9
    file_type = file_extension[1:]
10
11
    with open(image_path, "rb") as f:
12
        enc_img = base64.b64encode(f.read()).decode("utf-8")
13
        enc_img = f"data:image/{file_type};base64,{enc_img}"
14
    return enc_img
15
16
17
image_path = "<YOUR IMAGE PATH>"
18
processed_image = image_to_base64_data_url(image_path)
19
20
21
# format the input_object
22
image_input = [
23
    {"content": [{"type": "image", "image": processed_image}]}
24
]
25
26
res = co.embed(
27
    model="embed-v4.0",
28
    inputs=image_input,
29
    input_type="search_document",
30
    output_dimension=1024,
31
    embedding_types=["float"],
32
)
33
res.embeddings.float

1
# Import the necessary packages
2
import cohere
3
4
co = cohere.ClientV2(api_key="<YOUR API KEY>")
5
6
# format the input_object
7
image_input = [
8
    {"content": [{"type": "image", "image": processed_image}]}
9
]
10
11
co.embed(
12
    model="embed-v4.0",
13
    inputs=image_input,
14
    input_type="search_document",
15
    embedding_types=["float"],
16
)

1
{
2
    'id': '0d9bb922-f15f-4b8b-9a2f-72577324528f', 
3
    'texts': [], 
4
    'images': [{'width': 1080, 'height': 1350, 'format': 'jpeg', 'bit_depth': 24}], 
5
    'embeddings': {'float': [[-0.035369873, 0.040740967, 0.008262634, -0.008766174, .....]]}, 
6
    'meta': {
7
    'api_version': {'version': '1'},
8
    'billed_units': {'images': 1}
9
    }, 
10
    'response_type': 'embeddings_by_type'
11
}