Skip to main content

Rank

Search your data based on concepts or visual similarity


You can rank order your search results with the intuitive insights of an AI. Your model can identify concepts in your data and rank search results by how confident it is that a given concept is present.

You can even rank search results by how similar one input is to another input or region of the input model detected. The search results will return the input and also the annotation, which includes the region.

In annotation search, Rank is a list of Annotation objects.

info

The initialization code used in the following examples is outlined in detail on the client installation page.

tip

You can learn how to paginate your API requests results here.

Search by Concepts

Once your inputs are indexed, you can search for them by concepts.

By Clarifai/main App Concepts

When you add an input, it automatically gets predictions from the workflow in your base workflow, which is typically from the clarifai/main app, such as the Universal workflow. You can search by those predictions.

Click here to learn how to get a list of concepts available in the app.

##################################################################################
# In this section, we set the user authentication, app ID, and the concept ID we
# we want to rank by. Change these strings to run your own example.
##################################################################################

USER_ID = 'YOUR_USER_ID_HERE'
# Your PAT (Personal Access Token) can be found in the Account's Security section
PAT = 'YOUR_PAT_HERE'
APP_ID = 'YOUR_APP_ID_HERE'
# Change this to rank by a Clarifai/main concept
CONCEPT_ID = 'ai_fvlBqXZR'

##########################################################################
# YOU DO NOT NEED TO CHANGE ANYTHING BELOW THIS LINE TO RUN THIS EXAMPLE
##########################################################################

from clarifai_grpc.channel.clarifai_channel import ClarifaiChannel
from clarifai_grpc.grpc.api import resources_pb2, service_pb2, service_pb2_grpc
from clarifai_grpc.grpc.api.status import status_code_pb2

channel = ClarifaiChannel.get_grpc_channel()
stub = service_pb2_grpc.V2Stub(channel)

metadata = (('authorization', 'Key ' + PAT),)

userDataObject = resources_pb2.UserAppIDSet(user_id=USER_ID, app_id=APP_ID) # The userDataObject is required when using a PAT

post_annotations_searches_response = stub.PostAnnotationsSearches(
service_pb2.PostAnnotationsSearchesRequest(
user_app_id=userDataObject,
searches = [
resources_pb2.Search(
query=resources_pb2.Query(
ranks=[
resources_pb2.Rank(
annotation=resources_pb2.Annotation(
data=resources_pb2.Data(
concepts=[ # You can search by multiple concepts.
resources_pb2.Concept(
id=CONCEPT_ID, # You could search by concept Name as well.
value=1 # Value of 0 will search for images that don't have the concept.
)
]
)
)
)
]
)
)
],
pagination=service_pb2.Pagination(per_page=2, page=1)
),
metadata=metadata
)

if post_annotations_searches_response.status.code != status_code_pb2.SUCCESS:
raise Exception("Post searches failed, status: " + post_annotations_searches_response.status.description)

print("Search result:")
for hit in post_annotations_searches_response.hits:
print("\tScore %.2f for annotation: %s off input: %s" % (hit.score, hit.annotation.id, hit.input.id))

By Custom Concepts

After you have added inputs, annotated the inputs, and trained a custom model, you can search by those concepts.

########################################################################################
# In this section, we set the user authentication, app ID, and the custom concept we
# we want to rank by. Change these strings to run your own example.
########################################################################################

USER_ID = 'YOUR_USER_ID_HERE'
# Your PAT (Personal Access Token) can be found in the Account's Security section
PAT = 'YOUR_PAT_HERE'
APP_ID = 'YOUR_APP_ID_HERE'
# Change this to rank by your own custom concept
CONCEPT_ID = 'people'

##########################################################################
# YOU DO NOT NEED TO CHANGE ANYTHING BELOW THIS LINE TO RUN THIS EXAMPLE
##########################################################################

from clarifai_grpc.channel.clarifai_channel import ClarifaiChannel
from clarifai_grpc.grpc.api import resources_pb2, service_pb2, service_pb2_grpc
from clarifai_grpc.grpc.api.status import status_code_pb2

channel = ClarifaiChannel.get_grpc_channel()
stub = service_pb2_grpc.V2Stub(channel)

metadata = (('authorization', 'Key ' + PAT),)

userDataObject = resources_pb2.UserAppIDSet(user_id=USER_ID, app_id=APP_ID) # The userDataObject is required when using a PAT

post_annotations_searches_response = stub.PostAnnotationsSearches(
service_pb2.PostAnnotationsSearchesRequest(
user_app_id=userDataObject,
searches = [
resources_pb2.Search(
query=resources_pb2.Query(
ranks=[
resources_pb2.Rank(
annotation=resources_pb2.Annotation(
data=resources_pb2.Data(
concepts=[ # You can search by multiple concepts.
resources_pb2.Concept(
id=CONCEPT_ID, # You could search by concept Name as well.
value=1 # Value of 0 will search for images that don't have the concept.
)
]
)
)
)
]
)
)
],
pagination=service_pb2.Pagination(per_page=2, page=1)
),
metadata=metadata
)

if post_annotations_searches_response.status.code != status_code_pb2.SUCCESS:
raise Exception("Post searches failed, status: " + post_annotations_searches_response.status.description)

print("Search result:")
for hit in post_annotations_searches_response.hits:
print("\tScore %.2f for annotation: %s off input: %s" % (hit.score, hit.annotation.id, hit.input.id))

By Clarifai/main and Custom Concepts

You can combine a search to find inputs that have concepts you have supplied as well as predictions from your model.

################################################################################
# In this section, we set the user authentication, app ID, and the concepts we
# we want to rank by. Change these strings to run your own example.
################################################################################

USER_ID = 'YOUR_USER_ID_HERE'
# Your PAT (Personal Access Token) can be found in the Account's Security section
PAT = 'YOUR_PAT_HERE'
APP_ID = 'YOUR_APP_ID_HERE'
# Change this to a Clarifai/main concept
CONCEPT_ID_1 = 'ai_fvlBqXZR'
# Change this to your own custom concept
CONCEPT_ID_2 = 'people'

##########################################################################
# YOU DO NOT NEED TO CHANGE ANYTHING BELOW THIS LINE TO RUN THIS EXAMPLE
##########################################################################

from clarifai_grpc.channel.clarifai_channel import ClarifaiChannel
from clarifai_grpc.grpc.api import resources_pb2, service_pb2, service_pb2_grpc
from clarifai_grpc.grpc.api.status import status_code_pb2

channel = ClarifaiChannel.get_grpc_channel()
stub = service_pb2_grpc.V2Stub(channel)

metadata = (('authorization', 'Key ' + PAT),)

userDataObject = resources_pb2.UserAppIDSet(user_id=USER_ID, app_id=APP_ID) # The userDataObject is required when using a PAT

# Here we search for images labeled with 'ai_fvlBqXZR' and for which the General prediction model does not find
# a 'people' concept
post_annotations_searches_response = stub.PostAnnotationsSearches(
service_pb2.PostAnnotationsSearchesRequest(
user_app_id=userDataObject,
searches = [
resources_pb2.Search(
query=resources_pb2.Query(
ranks=[
resources_pb2.Rank(
annotation=resources_pb2.Annotation(
data=resources_pb2.Data(
concepts=[ # You can search by multiple concepts
resources_pb2.Concept(
id=CONCEPT_ID_1, # You could search by concept Name as well
value=1 # Value of 0 will search for images that don't have the concept
)
]
)
)
)
],
filters=[
resources_pb2.Filter(
annotation=resources_pb2.Annotation(
data=resources_pb2.Data(
concepts=[ # You can search by multiple concepts
resources_pb2.Concept(
id=CONCEPT_ID_2, # You could search by concept Name as well
value=0 # Value of 0 will search for images that don't have the concept
)
]
)
)
)
]
)
)
],
pagination=service_pb2.Pagination(per_page=2, page=1)
),
metadata=metadata
)

if post_annotations_searches_response.status.code != status_code_pb2.SUCCESS:
raise Exception("Post searches failed, status: " + post_annotations_searches_response.status.description)

print("Search result:")
for hit in post_annotations_searches_response.hits:
print("\tScore %.2f for annotation: %s off input: %s" % (hit.score, hit.annotation.id, hit.input.id))

By Concept in Another Language

Concepts that have a translation into another language can be searched for in that language, even without having the default language for your app being in that language. This uses the Clarifai's knowledge graph to lookup the translation and then perform the search.

For example, if your app is in English and you want to search for "dog" in Japanese, then you could search with language="ja" and name="犬".

################################################################################
# In this section, we set the user authentication, app ID, concept name, and
# language ID. Change these strings to run your own example.
################################################################################

USER_ID = 'YOUR_USER_ID_HERE'
# Your PAT (Personal Access Token) can be found in the Account's Security section
PAT = 'YOUR_PAT_HERE'
APP_ID = 'YOUR_APP_ID_HERE'
# Change these to what you want to search by
CONCEPT_NAME = '犬'
LANGUAGE_ID = 'ja' # Japanese

##########################################################################
# YOU DO NOT NEED TO CHANGE ANYTHING BELOW THIS LINE TO RUN THIS EXAMPLE
##########################################################################

from clarifai_grpc.channel.clarifai_channel import ClarifaiChannel
from clarifai_grpc.grpc.api import resources_pb2, service_pb2, service_pb2_grpc
from clarifai_grpc.grpc.api.status import status_code_pb2

channel = ClarifaiChannel.get_grpc_channel()
stub = service_pb2_grpc.V2Stub(channel)

metadata = (('authorization', 'Key ' + PAT),)

userDataObject = resources_pb2.UserAppIDSet(user_id=USER_ID, app_id=APP_ID)

post_annotations_searches_response = stub.PostAnnotationsSearches(
service_pb2.PostAnnotationsSearchesRequest(
user_app_id=userDataObject,
searches = [
resources_pb2.Search(
query=resources_pb2.Query(
ranks=[
resources_pb2.Rank(
annotation=resources_pb2.Annotation(
data=resources_pb2.Data(
concepts=[ # You can search by multiple concepts
resources_pb2.Concept(
name=CONCEPT_NAME, # You could search by concept ID as well
value=1 # Value of 0 will search for images that don't have the concept
)
]
)
)
)
],
language=LANGUAGE_ID
)
)
],
pagination=service_pb2.Pagination(per_page=2, page=1)
),
metadata=metadata
)

if post_annotations_searches_response.status.code != status_code_pb2.SUCCESS:
print(post_annotations_searches_response.status)
raise Exception("Post searches failed, status: " + post_annotations_searches_response.status.description)

print("Search result:")
for hit in post_annotations_searches_response.hits:
print("\tScore %.2f for annotation: %s off input: %s" % (hit.score, hit.annotation.id, hit.input.id))

Search by Visual Similarity

You can use images to search through your collection. The API will return ranked results based on how similar the results are to the image you provided in your query.

Search by Image URL

##################################################################
# In this section, we set the user authentication, app ID, and
# image URL. Change these strings to run your own example.
##################################################################

USER_ID = 'YOUR_USER_ID_HERE'
# Your PAT (Personal Access Token) can be found in the Account's Security section
PAT = 'YOUR_PAT_HERE'
APP_ID = 'YOUR_APP_ID_HERE'
# Change this to the image URL you want to search by
IMAGE_URL = 'https://samples.clarifai.com/metro-north.jpg'

##########################################################################
# YOU DO NOT NEED TO CHANGE ANYTHING BELOW THIS LINE TO RUN THIS EXAMPLE
##########################################################################

from clarifai_grpc.channel.clarifai_channel import ClarifaiChannel
from clarifai_grpc.grpc.api import resources_pb2, service_pb2, service_pb2_grpc
from clarifai_grpc.grpc.api.status import status_code_pb2

channel = ClarifaiChannel.get_grpc_channel()
stub = service_pb2_grpc.V2Stub(channel)

metadata = (('authorization', 'Key ' + PAT),)

userDataObject = resources_pb2.UserAppIDSet(user_id=USER_ID, app_id=APP_ID)

post_inputs_searches_response = stub.PostInputsSearches(
service_pb2.PostInputsSearchesRequest(
user_app_id=userDataObject,
searches = [
resources_pb2.Search(
query=resources_pb2.Query(
ranks=[
resources_pb2.Rank(
annotation=resources_pb2.Annotation(
data=resources_pb2.Data(
image=resources_pb2.Image(
url=IMAGE_URL
)
)
)
)
]
)
)
],
pagination=service_pb2.Pagination(per_page=2, page=1)
),
metadata=metadata
)

if post_inputs_searches_response.status.code != status_code_pb2.SUCCESS:
print(post_inputs_searches_response.status)
raise Exception("Post searches failed, status: " + post_inputs_searches_response.status.description)

print("Search result:")
for hit in post_inputs_searches_response.hits:
print("\tScore %.2f for input: %s" % (hit.score, hit.input.id))

Search by Image Bytes

You can also search for an input by bytes, with the bytes being from local storage.

#######################################################################
# In this section, we set the user authentication, app ID, and image
# file location. Change these strings to run your own example.
#######################################################################

USER_ID = 'YOUR_USER_ID_HERE'
# Your PAT (Personal Access Token) can be found in the Account's Security section
PAT = 'YOUR_PAT_HERE'
APP_ID = 'YOUR_APP_ID_HERE'
# Change this to the image file location you want to search by
IMAGE_FILE_LOCATION = 'YOUR_IMAGE_FILE_LOCATION'

##########################################################################
# YOU DO NOT NEED TO CHANGE ANYTHING BELOW THIS LINE TO RUN THIS EXAMPLE
##########################################################################

from clarifai_grpc.channel.clarifai_channel import ClarifaiChannel
from clarifai_grpc.grpc.api import resources_pb2, service_pb2, service_pb2_grpc
from clarifai_grpc.grpc.api.status import status_code_pb2

channel = ClarifaiChannel.get_grpc_channel()
stub = service_pb2_grpc.V2Stub(channel)

metadata = (('authorization', 'Key ' + PAT),)

userDataObject = resources_pb2.UserAppIDSet(user_id=USER_ID, app_id=APP_ID)

with open(IMAGE_FILE_LOCATION, "rb") as f:
file_bytes = f.read()

post_inputs_searches_response = stub.PostInputsSearches(
service_pb2.PostInputsSearchesRequest(
user_app_id=userDataObject,
searches = [
resources_pb2.Search(
query=resources_pb2.Query(
ranks=[
resources_pb2.Rank(
input=resources_pb2.Annotation(
data=resources_pb2.Data(
image=resources_pb2.Image(
base64=file_bytes
)
)
)
)
]
)
)
],
pagination=service_pb2.Pagination(per_page=2, page=1)
),
metadata=metadata
)

if post_inputs_searches_response.status.code != status_code_pb2.SUCCESS:
print(post_inputs_searches_response.status)
raise Exception("Post searches failed, status: " + post_inputs_searches_response.status.description)

print("Search result:")
for hit in post_inputs_searches_response.hits:
print("\tScore %.2f for input: %s" % (hit.score, hit.input.id))

By Input ID

If the input has been indexed, we can use the input ID. If there are multiple embeddings (for example multiple regions), we will average the embeddings.

##################################################################
# In this section, we set the user authentication, app ID, and
# input ID. Change these strings to run your own example.
##################################################################

USER_ID = 'YOUR_USER_ID_HERE'
# Your PAT (Personal Access Token) can be found in the Account's Security section
PAT = 'YOUR_PAT_HERE'
APP_ID = 'YOUR_APP_ID_HERE'
# Change this to the input ID you want to search by
INPUT_ID = 'c021c670357e4083b197abe80bda82b0'

##########################################################################
# YOU DO NOT NEED TO CHANGE ANYTHING BELOW THIS LINE TO RUN THIS EXAMPLE
##########################################################################

from clarifai_grpc.channel.clarifai_channel import ClarifaiChannel
from clarifai_grpc.grpc.api import resources_pb2, service_pb2, service_pb2_grpc
from clarifai_grpc.grpc.api.status import status_code_pb2

channel = ClarifaiChannel.get_grpc_channel()
stub = service_pb2_grpc.V2Stub(channel)

metadata = (('authorization', 'Key ' + PAT),)

userDataObject = resources_pb2.UserAppIDSet(user_id=USER_ID, app_id=APP_ID)

post_annotations_searches_response = stub.PostAnnotationsSearches(
service_pb2.PostAnnotationsSearchesRequest(
user_app_id=userDataObject,
searches = [
resources_pb2.Search(
query=resources_pb2.Query(
ranks=[
resources_pb2.Rank(
annotation=resources_pb2.Annotation(
input_id=INPUT_ID
)
)
]
)
)
],
pagination=service_pb2.Pagination(per_page=2, page=1)
),
metadata=metadata
)

if post_annotations_searches_response.status.code != status_code_pb2.SUCCESS:
print(post_annotations_searches_response.status)
raise Exception("Post searches failed, status: " + post_annotations_searches_response.status.description)

print("Search result:")
for hit in post_annotations_searches_response.hits:
print("\tScore %.2f for annotation: %s off input: %s" % (hit.score, hit.annotation.id, hit.input.id))

Search by Text Similarity

You can use texts to search through your collection of texts. The text-to-text search will return ranked results based on how similar the results are to the text you provided in your query.

tip
  • To perform text-to-text searches, you could choose a workflow that includes a text embedder and a clusterer, such as the Text workflow, as the base workflow for your application.
  • To perform text-to-image searches, you could choose Universal as the base workflow, which allows you to use texts to search through your collection of images.
####################################################################################
# In this section, we set the user authentication, app ID, and the raw text we
# we want to search by. Change these strings to run your own example.
###################################################################################

USER_ID = 'YOUR_USER_ID_HERE'
# Your PAT (Personal Access Token) can be found in the Account's Security section
PAT = 'YOUR_PAT_HERE'
APP_ID = 'YOUR_APP_ID_HERE'
# Change this to the raw text you want to search by
RAW_TEXT = 'black dress with white polka dots'

##########################################################################
# YOU DO NOT NEED TO CHANGE ANYTHING BELOW THIS LINE TO RUN THIS EXAMPLE
##########################################################################

from clarifai_grpc.channel.clarifai_channel import ClarifaiChannel
from clarifai_grpc.grpc.api import resources_pb2, service_pb2, service_pb2_grpc
from clarifai_grpc.grpc.api.status import status_code_pb2

channel = ClarifaiChannel.get_grpc_channel()
stub = service_pb2_grpc.V2Stub(channel)

metadata = (('authorization', 'Key ' + PAT),)

userDataObject = resources_pb2.UserAppIDSet(user_id=USER_ID, app_id=APP_ID)

post_annotations_searches_response = stub.PostAnnotationsSearches(
service_pb2.PostAnnotationsSearchesRequest(
user_app_id=userDataObject,
searches = [
resources_pb2.Search(
query=resources_pb2.Query(
ranks=[
resources_pb2.Rank(
annotation=resources_pb2.Annotation(
data=resources_pb2.Data(
text=resources_pb2.Text(
raw=RAW_TEXT
)
)
)
)
]
)
)
],
pagination=service_pb2.Pagination(per_page=2, page=1)
),
metadata=metadata
)

if post_annotations_searches_response.status.code != status_code_pb2.SUCCESS:
print(post_annotations_searches_response.status)
raise Exception("Post searches failed, status: " + post_annotations_searches_response.status.description)

print("Search result:")
for hit in post_annotations_searches_response.hits:
print("\tScore %.2f for annotation: %s off input: %s" % (hit.score, hit.annotation.id, hit.input.id))