Skip to main content

Interpreting Evaluations

Learn to interpret model evaluations


Model evaluation takes some time—depending on the amount of data the model has. After the process is complete, you could get the results and use them to assess the performance of your model.

Get Evaluation Results

Below are examples of how you would use different methods to get the evaluation results.

info

The initialization code used in the following examples is outlined in detail on the client installation page.

GetEvaluation

Get the evaluation results by using the evaluation_id returned after starting an evaluation.

#############################################################################################
# In this section, we set the user authentication, app ID, and the model evaluation ID.
# Change these strings to run your own example.
############################################################################################

USER_ID = "YOUR_USER_ID_HERE"
# Your PAT (Personal Access Token) can be found in the Account's Security section
PAT = "YOUR_PAT_HERE"
APP_ID = "YOUR_APP_ID_HERE"
# Change this to get your model evaluation results
EVALUATION_ID = "YOUR_EVALUATION_ID_HERE"

##########################################################################
# YOU DO NOT NEED TO CHANGE ANYTHING BELOW THIS LINE TO RUN THIS EXAMPLE
##########################################################################

from clarifai_grpc.channel.clarifai_channel import ClarifaiChannel
from clarifai_grpc.grpc.api import resources_pb2, service_pb2, service_pb2_grpc
from clarifai_grpc.grpc.api.status import status_code_pb2

channel = ClarifaiChannel.get_grpc_channel()
stub = service_pb2_grpc.V2Stub(channel)

metadata = (("authorization", "Key " + PAT),)

userDataObject = resources_pb2.UserAppIDSet(user_id=USER_ID, app_id=APP_ID)

get_evaluation_response = stub.GetEvaluation(
service_pb2.GetEvaluationRequest(
user_app_id=userDataObject,
evaluation_id=EVALUATION_ID, # returned after starting an evaluation
fields=resources_pb2.FieldsValue(
confusion_matrix=True,
cooccurrence_matrix=True,
label_counts=True,
binary_metrics=True,
test_set=True,
metrics_by_area=True,
metrics_by_class=True,
),
),
metadata=metadata,
)

if get_evaluation_response.status.code != status_code_pb2.SUCCESS:
print(get_evaluation_response.status)
raise Exception(
"Get model metrics failed, status: "
+ get_evaluation_response.status.description
)

print(get_evaluation_response)
Output Example
status {
code: SUCCESS
description: "Ok"
req_id: "0251012185ab3e3c76dd0b31262b78f0"
}
eval_metrics {
status {
code: MODEL_EVALUATED
description: "Model was successfully evaluated."
}
summary {
macro_avg_roc_auc: 1.0
macro_avg_f1_score: 0.8809523582458496
macro_std_f1_score: 0.13677529990673065
macro_avg_precision: 0.9375
macro_avg_recall: 0.875
}
confusion_matrix {
matrix {
predicted: "positive"
actual: "positive"
value: 0.7497637867927551
predicted_concept {
id: "positive"
name: "positive"
value: 0.7497637867927551
app_id: "text-search-app"
}
actual_concept {
id: "positive"
name: "positive"
value: 1.0
app_id: "text-search-app"
}
}
matrix {
predicted: "negative"
actual: "positive"
value: 0.2502362132072449
predicted_concept {
id: "negative"
name: "negative"
value: 0.2502362132072449
app_id: "text-search-app"
}
actual_concept {
id: "positive"
name: "positive"
value: 1.0
app_id: "text-search-app"
}
}
matrix {
predicted: "positive"
actual: "negative"
value: 3.033356961168465e-07
predicted_concept {
id: "positive"
name: "positive"
value: 3.033356961168465e-07
app_id: "text-search-app"
}
actual_concept {
id: "negative"
name: "negative"
value: 1.0
app_id: "text-search-app"
}
}
matrix {
predicted: "negative"
actual: "negative"
value: 0.9999997019767761
predicted_concept {
id: "negative"
name: "negative"
value: 0.9999997019767761
app_id: "text-search-app"
}
actual_concept {
id: "negative"
name: "negative"
value: 1.0
app_id: "text-search-app"
}
}
concept_ids: "positive"
concept_ids: "negative"
}
cooccurrence_matrix {
matrix {
row: "positive"
col: "positive"
count: 10
}
matrix {
row: "negative"
col: "negative"
count: 11
}
concept_ids: "positive"
concept_ids: "negative"
}
label_counts {
positive_label_counts {
concept_name: "positive"
count: 10
concept {
id: "positive"
name: "positive"
value: 1.0
app_id: "text-search-app"
}
}
positive_label_counts {
concept_name: "negative"
count: 11
concept {
id: "negative"
name: "negative"
value: 1.0
app_id: "text-search-app"
}
}
}
binary_metrics {
num_pos: 2
num_neg: 2
num_tot: 4
roc_auc: 1.0
f1: 0.8333333730697632
concept {
id: "positive"
name: "positive"
value: 1.0
app_id: "text-search-app"
}
roc_curve {
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 0.0
fpr: 1.0
tpr: 0.0
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 0.75
tpr: 1.0
thresholds: 1.0
thresholds: 0.9900000095367432
thresholds: 0.9800000190734863
thresholds: 0.9700000286102295
thresholds: 0.9599999785423279
thresholds: 0.949999988079071
thresholds: 0.9399999976158142
thresholds: 0.9300000071525574
thresholds: 0.9200000166893005
thresholds: 0.9100000262260437
thresholds: 0.8999999761581421
thresholds: 0.8899999856948853
thresholds: 0.8799999952316284
thresholds: 0.8700000047683716
thresholds: 0.8600000143051147
thresholds: 0.8500000238418579
thresholds: 0.8399999737739563
thresholds: 0.8299999833106995
thresholds: 0.8199999928474426
thresholds: 0.8100000023841858
thresholds: 0.800000011920929
thresholds: 0.7900000214576721
thresholds: 0.7799999713897705
thresholds: 0.7699999809265137
thresholds: 0.7599999904632568
thresholds: 0.75
thresholds: 0.7400000095367432
thresholds: 0.7300000190734863
thresholds: 0.7200000286102295
thresholds: 0.7099999785423279
thresholds: 0.699999988079071
thresholds: 0.6899999976158142
thresholds: 0.6800000071525574
thresholds: 0.6700000166893005
thresholds: 0.6600000262260437
thresholds: 0.6499999761581421
thresholds: 0.6399999856948853
thresholds: 0.6299999952316284
thresholds: 0.6200000047683716
thresholds: 0.6100000143051147
thresholds: 0.6000000238418579
thresholds: 0.5899999737739563
thresholds: 0.5799999833106995
thresholds: 0.5699999928474426
thresholds: 0.5600000023841858
thresholds: 0.550000011920929
thresholds: 0.5400000214576721
thresholds: 0.5299999713897705
thresholds: 0.5199999809265137
thresholds: 0.5099999904632568
thresholds: 0.5
thresholds: 0.49000000953674316
thresholds: 0.47999998927116394
thresholds: 0.4699999988079071
thresholds: 0.46000000834465027
thresholds: 0.44999998807907104
thresholds: 0.4399999976158142
thresholds: 0.4300000071525574
thresholds: 0.41999998688697815
thresholds: 0.4099999964237213
thresholds: 0.4000000059604645
thresholds: 0.38999998569488525
thresholds: 0.3799999952316284
thresholds: 0.3700000047683716
thresholds: 0.36000001430511475
thresholds: 0.3499999940395355
thresholds: 0.3400000035762787
thresholds: 0.33000001311302185
thresholds: 0.3199999928474426
thresholds: 0.3100000023841858
thresholds: 0.30000001192092896
thresholds: 0.28999999165534973
thresholds: 0.2800000011920929
thresholds: 0.27000001072883606
thresholds: 0.25999999046325684
thresholds: 0.25
thresholds: 0.23999999463558197
thresholds: 0.23000000417232513
thresholds: 0.2199999988079071
thresholds: 0.20999999344348907
thresholds: 0.20000000298023224
thresholds: 0.1899999976158142
thresholds: 0.18000000715255737
thresholds: 0.17000000178813934
thresholds: 0.1599999964237213
thresholds: 0.15000000596046448
thresholds: 0.14000000059604645
thresholds: 0.12999999523162842
thresholds: 0.11999999731779099
thresholds: 0.10999999940395355
thresholds: 0.10000000149011612
thresholds: 0.09000000357627869
thresholds: 0.07999999821186066
thresholds: 0.07000000029802322
thresholds: 0.05999999865889549
thresholds: 0.05000000074505806
thresholds: 0.03999999910593033
thresholds: 0.029999999329447746
thresholds: 0.019999999552965164
thresholds: 0.009999999776482582
thresholds: 0.0
}
precision_recall_curve {
recall: 1.0
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.75
recall: 0.0
precision: 0.5
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
precision: 1.0
thresholds: 0.0
thresholds: 0.009999999776482582
thresholds: 0.019999999552965164
thresholds: 0.029999999329447746
thresholds: 0.03999999910593033
thresholds: 0.05000000074505806
thresholds: 0.05999999865889549
thresholds: 0.07000000029802322
thresholds: 0.07999999821186066
thresholds: 0.09000000357627869
thresholds: 0.10000000149011612
thresholds: 0.10999999940395355
thresholds: 0.11999999731779099
thresholds: 0.12999999523162842
thresholds: 0.14000000059604645
thresholds: 0.15000000596046448
thresholds: 0.1599999964237213
thresholds: 0.17000000178813934
thresholds: 0.18000000715255737
thresholds: 0.1899999976158142
thresholds: 0.20000000298023224
thresholds: 0.20999999344348907
thresholds: 0.2199999988079071
thresholds: 0.23000000417232513
thresholds: 0.23999999463558197
thresholds: 0.25
thresholds: 0.25999999046325684
thresholds: 0.27000001072883606
thresholds: 0.2800000011920929
thresholds: 0.28999999165534973
thresholds: 0.30000001192092896
thresholds: 0.3100000023841858
thresholds: 0.3199999928474426
thresholds: 0.33000001311302185
thresholds: 0.3400000035762787
thresholds: 0.3499999940395355
thresholds: 0.36000001430511475
thresholds: 0.3700000047683716
thresholds: 0.3799999952316284
thresholds: 0.38999998569488525
thresholds: 0.4000000059604645
thresholds: 0.4099999964237213
thresholds: 0.41999998688697815
thresholds: 0.4300000071525574
thresholds: 0.4399999976158142
thresholds: 0.44999998807907104
thresholds: 0.46000000834465027
thresholds: 0.4699999988079071
thresholds: 0.47999998927116394
thresholds: 0.49000000953674316
thresholds: 0.5
thresholds: 0.5099999904632568
thresholds: 0.5199999809265137
thresholds: 0.5299999713897705
thresholds: 0.5400000214576721
thresholds: 0.550000011920929
thresholds: 0.5600000023841858
thresholds: 0.5699999928474426
thresholds: 0.5799999833106995
thresholds: 0.5899999737739563
thresholds: 0.6000000238418579
thresholds: 0.6100000143051147
thresholds: 0.6200000047683716
thresholds: 0.6299999952316284
thresholds: 0.6399999856948853
thresholds: 0.6499999761581421
thresholds: 0.6600000262260437
thresholds: 0.6700000166893005
thresholds: 0.6800000071525574
thresholds: 0.6899999976158142
thresholds: 0.699999988079071
thresholds: 0.7099999785423279
thresholds: 0.7200000286102295
thresholds: 0.7300000190734863
thresholds: 0.7400000095367432
thresholds: 0.75
thresholds: 0.7599999904632568
thresholds: 0.7699999809265137
thresholds: 0.7799999713897705
thresholds: 0.7900000214576721
thresholds: 0.800000011920929
thresholds: 0.8100000023841858
thresholds: 0.8199999928474426
thresholds: 0.8299999833106995
thresholds: 0.8399999737739563
thresholds: 0.8500000238418579
thresholds: 0.8600000143051147
thresholds: 0.8700000047683716
thresholds: 0.8799999952316284
thresholds: 0.8899999856948853
thresholds: 0.8999999761581421
thresholds: 0.9100000262260437
thresholds: 0.9200000166893005
thresholds: 0.9300000071525574
thresholds: 0.9399999976158142
thresholds: 0.949999988079071
thresholds: 0.9599999785423279
thresholds: 0.9700000286102295
thresholds: 0.9800000190734863
thresholds: 0.9900000095367432
thresholds: 1.0
}
}
binary_metrics {
num_pos: 2
num_neg: 2
num_tot: 4
roc_auc: 1.0
f1: 0.9285714626312256
concept {
id: "negative"
name: "negative"
value: 1.0
app_id: "text-search-app"
}
roc_curve {
fpr: 0.0
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 0.25
fpr: 1.0
tpr: 0.5
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
tpr: 1.0
thresholds: 1.0
thresholds: 0.9900000095367432
thresholds: 0.9800000190734863
thresholds: 0.9700000286102295
thresholds: 0.9599999785423279
thresholds: 0.949999988079071
thresholds: 0.9399999976158142
thresholds: 0.9300000071525574
thresholds: 0.9200000166893005
thresholds: 0.9100000262260437
thresholds: 0.8999999761581421
thresholds: 0.8899999856948853
thresholds: 0.8799999952316284
thresholds: 0.8700000047683716
thresholds: 0.8600000143051147
thresholds: 0.8500000238418579
thresholds: 0.8399999737739563
thresholds: 0.8299999833106995
thresholds: 0.8199999928474426
thresholds: 0.8100000023841858
thresholds: 0.800000011920929
thresholds: 0.7900000214576721
thresholds: 0.7799999713897705
thresholds: 0.7699999809265137
thresholds: 0.7599999904632568
thresholds: 0.75
thresholds: 0.7400000095367432
thresholds: 0.7300000190734863
thresholds: 0.7200000286102295
thresholds: 0.7099999785423279
thresholds: 0.699999988079071
thresholds: 0.6899999976158142
thresholds: 0.6800000071525574
thresholds: 0.6700000166893005
thresholds: 0.6600000262260437
thresholds: 0.6499999761581421
thresholds: 0.6399999856948853
thresholds: 0.6299999952316284
thresholds: 0.6200000047683716
thresholds: 0.6100000143051147
thresholds: 0.6000000238418579
thresholds: 0.5899999737739563
thresholds: 0.5799999833106995
thresholds: 0.5699999928474426
thresholds: 0.5600000023841858
thresholds: 0.550000011920929
thresholds: 0.5400000214576721
thresholds: 0.5299999713897705
thresholds: 0.5199999809265137
thresholds: 0.5099999904632568
thresholds: 0.5
thresholds: 0.49000000953674316
thresholds: 0.47999998927116394
thresholds: 0.4699999988079071
thresholds: 0.46000000834465027
thresholds: 0.44999998807907104
thresholds: 0.4399999976158142
thresholds: 0.4300000071525574
thresholds: 0.41999998688697815
thresholds: 0.4099999964237213
thresholds: 0.4000000059604645
thresholds: 0.38999998569488525
thresholds: 0.3799999952316284
thresholds: 0.3700000047683716
thresholds: 0.36000001430511475
thresholds: 0.3499999940395355
thresholds: 0.3400000035762787
thresholds: 0.33000001311302185
thresholds: 0.3199999928474426
thresholds: 0.3100000023841858
thresholds: 0.30000001192092896
thresholds: 0.28999999165534973
thresholds: 0.2800000011920929
thresholds: 0.27000001072883606
thresholds: 0.25999999046325684
thresholds: 0.25
thresholds: 0.23999999463558197
thresholds: 0.23000000417232513
thresholds: 0.2199999988079071
thresholds: 0.20999999344348907
thresholds: 0.20000000298023224
thresholds: 0.1899999976158142
thresholds: 0.18000000715255737
thresholds: 0.17000000178813934
thresholds: 0.1599999964237213
thresholds: 0.15000000596046448
thresholds: 0.14000000059604645
thresholds: 0.12999999523162842
thresholds: 0.11999999731779099
thresholds: 0.10999999940395355
thresholds: 0.10000000149011612
thresholds: 0.09000000357627869
thresholds: 0.07999999821186066
thresholds: 0.07000000029802322
thresholds: 0.05999999865889549
thresholds: 0.05000000074505806
thresholds: 0.03999999910593033
thresholds: 0.029999999329447746
thresholds: 0.019999999552965164
thresholds: 0.009999999776482582
thresholds: 0.0
}
precision_recall_curve {
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 1.0
recall: 0.5
precision: 0.5
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 0.875
precision: 1.0
thresholds: 0.0
thresholds: 0.009999999776482582
thresholds: 0.019999999552965164
thresholds: 0.029999999329447746
thresholds: 0.03999999910593033
thresholds: 0.05000000074505806
thresholds: 0.05999999865889549
thresholds: 0.07000000029802322
thresholds: 0.07999999821186066
thresholds: 0.09000000357627869
thresholds: 0.10000000149011612
thresholds: 0.10999999940395355
thresholds: 0.11999999731779099
thresholds: 0.12999999523162842
thresholds: 0.14000000059604645
thresholds: 0.15000000596046448
thresholds: 0.1599999964237213
thresholds: 0.17000000178813934
thresholds: 0.18000000715255737
thresholds: 0.1899999976158142
thresholds: 0.20000000298023224
thresholds: 0.20999999344348907
thresholds: 0.2199999988079071
thresholds: 0.23000000417232513
thresholds: 0.23999999463558197
thresholds: 0.25
thresholds: 0.25999999046325684
thresholds: 0.27000001072883606
thresholds: 0.2800000011920929
thresholds: 0.28999999165534973
thresholds: 0.30000001192092896
thresholds: 0.3100000023841858
thresholds: 0.3199999928474426
thresholds: 0.33000001311302185
thresholds: 0.3400000035762787
thresholds: 0.3499999940395355
thresholds: 0.36000001430511475
thresholds: 0.3700000047683716
thresholds: 0.3799999952316284
thresholds: 0.38999998569488525
thresholds: 0.4000000059604645
thresholds: 0.4099999964237213
thresholds: 0.41999998688697815
thresholds: 0.4300000071525574
thresholds: 0.4399999976158142
thresholds: 0.44999998807907104
thresholds: 0.46000000834465027
thresholds: 0.4699999988079071
thresholds: 0.47999998927116394
thresholds: 0.49000000953674316
thresholds: 0.5
thresholds: 0.5099999904632568
thresholds: 0.5199999809265137
thresholds: 0.5299999713897705
thresholds: 0.5400000214576721
thresholds: 0.550000011920929
thresholds: 0.5600000023841858
thresholds: 0.5699999928474426
thresholds: 0.5799999833106995
thresholds: 0.5899999737739563
thresholds: 0.6000000238418579
thresholds: 0.6100000143051147
thresholds: 0.6200000047683716
thresholds: 0.6299999952316284
thresholds: 0.6399999856948853
thresholds: 0.6499999761581421
thresholds: 0.6600000262260437
thresholds: 0.6700000166893005
thresholds: 0.6800000071525574
thresholds: 0.6899999976158142
thresholds: 0.699999988079071
thresholds: 0.7099999785423279
thresholds: 0.7200000286102295
thresholds: 0.7300000190734863
thresholds: 0.7400000095367432
thresholds: 0.75
thresholds: 0.7599999904632568
thresholds: 0.7699999809265137
thresholds: 0.7799999713897705
thresholds: 0.7900000214576721
thresholds: 0.800000011920929
thresholds: 0.8100000023841858
thresholds: 0.8199999928474426
thresholds: 0.8299999833106995
thresholds: 0.8399999737739563
thresholds: 0.8500000238418579
thresholds: 0.8600000143051147
thresholds: 0.8700000047683716
thresholds: 0.8799999952316284
thresholds: 0.8899999856948853
thresholds: 0.8999999761581421
thresholds: 0.9100000262260437
thresholds: 0.9200000166893005
thresholds: 0.9300000071525574
thresholds: 0.9399999976158142
thresholds: 0.949999988079071
thresholds: 0.9599999785423279
thresholds: 0.9700000286102295
thresholds: 0.9800000190734863
thresholds: 0.9900000095367432
thresholds: 1.0
}
}
test_set {
predicted_concepts {
id: "positive"
name: "positive"
value: 0.9999601244926453
app_id: "text-search-app"
}
predicted_concepts {
id: "negative"
name: "negative"
value: 3.9902104617794976e-05
app_id: "text-search-app"
}
ground_truth_concepts {
id: "positive"
name: "positive"
value: 1.0
app_id: "text-search-app"
}
input {
id: "UyZACEDqN6WhAQOO"
data {
text {
url: "https://data.clarifai.com/orig/users/alfrick/apps/text-search-app/inputs/text/e9dc85e0585a7a6d32da791e6cfa2c52"
hosted {
prefix: "https://data.clarifai.com"
suffix: "users/alfrick/apps/text-search-app/inputs/text/e9dc85e0585a7a6d32da791e6cfa2c52"
sizes: "orig"
crossorigin: "use-credentials"
}
text_info {
char_count: 507
encoding: "UTF8"
}
}
}
created_at {
seconds: 1687435506
nanos: 969900000
}
modified_at {
seconds: 1690525794
nanos: 876378000
}
status {
code: INPUT_DOWNLOAD_SUCCESS
description: "Download complete"
}
}
}
test_set {
predicted_concepts {
id: "negative"
name: "negative"
value: 0.9999999403953552
app_id: "text-search-app"
}
predicted_concepts {
id: "positive"
name: "positive"
value: 6.925120743517255e-08
app_id: "text-search-app"
}
ground_truth_concepts {
id: "negative"
name: "negative"
value: 1.0
app_id: "text-search-app"
}
input {
id: "A2SsbMJrHqiAUmnr"
data {
text {
url: "https://data.clarifai.com/orig/users/alfrick/apps/text-search-app/inputs/text/77fe0c1ff92bcb6d876ec8e551e9268f"
hosted {
prefix: "https://data.clarifai.com"
suffix: "users/alfrick/apps/text-search-app/inputs/text/77fe0c1ff92bcb6d876ec8e551e9268f"
sizes: "orig"
crossorigin: "use-credentials"
}
text_info {
char_count: 440
encoding: "UTF8"
}
}
}
created_at {
seconds: 1687436288
nanos: 370201000
}
modified_at {
seconds: 1690525794
nanos: 876378000
}
status {
code: INPUT_DOWNLOAD_SUCCESS
description: "Download complete"
}
}
}
test_set {
predicted_concepts {
id: "negative"
name: "negative"
value: 0.9999988675117493
app_id: "text-search-app"
}
predicted_concepts {
id: "positive"
name: "positive"
value: 1.1424209560573217e-06
app_id: "text-search-app"
}
ground_truth_concepts {
id: "negative"
name: "negative"
value: 1.0
app_id: "text-search-app"
}
input {
id: "NnIBViXRFrXAGKkd"
data {
text {
url: "https://data.clarifai.com/orig/users/alfrick/apps/text-search-app/inputs/text/65d33e371bbdd16cbcf7a2826ab0bcc1"
hosted {
prefix: "https://data.clarifai.com"
suffix: "users/alfrick/apps/text-search-app/inputs/text/65d33e371bbdd16cbcf7a2826ab0bcc1"
sizes: "orig"
crossorigin: "use-credentials"
}
text_info {
char_count: 359
encoding: "UTF8"
}
}
}
created_at {
seconds: 1687435598
nanos: 758239000
}
modified_at {
seconds: 1690525794
nanos: 876378000
}
status {
code: INPUT_DOWNLOAD_SUCCESS
description: "Download complete"
}
}
}
test_set {
predicted_concepts {
id: "positive"
name: "positive"
value: 0.9999937415122986
app_id: "text-search-app"
}
predicted_concepts {
id: "negative"
name: "negative"
value: 6.254503659874899e-06
app_id: "text-search-app"
}
ground_truth_concepts {
id: "positive"
name: "positive"
value: 1.0
app_id: "text-search-app"
}
input {
id: "OK2cJALgpFQYafF2"
data {
text {
url: "https://data.clarifai.com/orig/users/alfrick/apps/text-search-app/inputs/text/90fd9b5c2f9af536520add1a2da13db8"
hosted {
prefix: "https://data.clarifai.com"
suffix: "users/alfrick/apps/text-search-app/inputs/text/90fd9b5c2f9af536520add1a2da13db8"
sizes: "orig"
crossorigin: "use-credentials"
}
text_info {
char_count: 399
encoding: "UTF8"
}
}
}
created_at {
seconds: 1687435419
nanos: 661211000
}
modified_at {
seconds: 1690525794
nanos: 810009000
}
status {
code: INPUT_DOWNLOAD_SUCCESS
description: "Download complete"
}
}
}
test_set {
predicted_concepts {
id: "positive"
name: "positive"
value: 0.9985570907592773
app_id: "text-search-app"
}
predicted_concepts {
id: "negative"
name: "negative"
value: 0.001442914130166173
app_id: "text-search-app"
}
ground_truth_concepts {
id: "positive"
name: "positive"
value: 1.0
app_id: "text-search-app"
}
input {
id: "YViTPPBzt3pTlmgY"
data {
text {
url: "https://data.clarifai.com/orig/users/alfrick/apps/text-search-app/inputs/text/5bccdbc8c9f8fb9a673d61dc2e32e40b"
hosted {
prefix: "https://data.clarifai.com"
suffix: "users/alfrick/apps/text-search-app/inputs/text/5bccdbc8c9f8fb9a673d61dc2e32e40b"
sizes: "orig"
crossorigin: "use-credentials"
}
text_info {
char_count: 413
encoding: "UTF8"
}
}
}
created_at {
seconds: 1687436129
nanos: 772735000
}
modified_at {
seconds: 1690525794
nanos: 876378000
}
status {
code: INPUT_DOWNLOAD_SUCCESS
description: "Download complete"
}
}
}
id: "e223fa4ac14b4784b223cd31cc545f34"
eval_info {
params {
fields {
key: "dataset_id"
value {
string_value: ""
}
}
fields {
key: "dataset_version_id"
value {
string_value: ""
}
}
fields {
key: "use_kfold"
value {
bool_value: true
}
}
}
}
model {
id: "text-model-1"
app_id: "text-search-app"
model_version {
id: "3ad2c152232e46ebb16ed31f67dc54d8"
created_at {
seconds: 1693564041
nanos: 515456000
}
status {
code: MODEL_TRAINED
description: "Model is trained and ready"
}
active_concept_count: 2
metrics {
status {
code: MODEL_EVALUATED
description: "Model was successfully evaluated."
}
summary {
macro_avg_roc_auc: 1.0
macro_avg_f1_score: 0.8809523582458496
macro_std_f1_score: 0.13677529990673065
macro_avg_precision: 0.9375
macro_avg_recall: 0.875
}
}
total_input_count: 21
completed_at {
seconds: 1693564044
nanos: 915680000
}
visibility {
gettable: PRIVATE
}
app_id: "text-search-app"
user_id: "alfrick"
metadata {
}
output_info {
output_config {
}
message: "Show output_info with: GET /models/{model_id}/output_info"
params {
fields {
key: "max_concepts"
value {
number_value: 20.0
}
}
fields {
key: "min_value"
value {
number_value: 0.0
}
}
fields {
key: "select_concepts"
value {
list_value {
}
}
}
}
}
input_info {
base_embed_model {
id: "multilingual-text-embedding"
app_id: "main"
model_version {
id: "9b33adf15280465b857163ddaaacdcb1"
}
user_id: "clarifai"
model_type_id: "text-embedder"
}
}
train_info {
params {
fields {
key: "dataset_id"
value {
string_value: ""
}
}
fields {
key: "dataset_version_id"
value {
string_value: ""
}
}
fields {
key: "enrich_dataset"
value {
string_value: "Automatic"
}
}
}
}
import_info {
}
}
user_id: "alfrick"
model_type_id: "embedding-classifier"
}
user_id: "alfrick"
app_id: "text-search-app"
}

ListEvaluations

List the evaluation results of all models in your app.

#################################################################################
# In this section, we set the user authentication and app ID.
# Change these strings to run your own example.
#################################################################################

USER_ID = "YOUR_USER_ID_HERE"
# Your PAT (Personal Access Token) can be found in the Account's Security section
PAT = "YOUR_PAT_HERE"
APP_ID = "YOUR_APP_ID_HERE"

##########################################################################
# YOU DO NOT NEED TO CHANGE ANYTHING BELOW THIS LINE TO RUN THIS EXAMPLE
##########################################################################

from clarifai_grpc.channel.clarifai_channel import ClarifaiChannel
from clarifai_grpc.grpc.api import resources_pb2, service_pb2, service_pb2_grpc
from clarifai_grpc.grpc.api.status import status_code_pb2

channel = ClarifaiChannel.get_grpc_channel()
stub = service_pb2_grpc.V2Stub(channel)

metadata = (("authorization", "Key " + PAT),)

userDataObject = resources_pb2.UserAppIDSet(user_id=USER_ID, app_id=APP_ID)

list_evaluation_response = stub.ListEvaluations(
service_pb2.ListEvaluationsRequest(user_app_id=userDataObject),
metadata=metadata,
)

if list_evaluation_response.status.code != status_code_pb2.SUCCESS:
print(list_evaluation_response.status)
raise Exception("Get model metrics failed, status: " + list_evaluation_response.status.description)

print(list_evaluation_response)

GetModelVersionEvaluation

Get the evaluation results of a specific version of a custom model by using the evaluation_id returned after starting an evaluation.

###################################################################################################
# In this section, we set the user authentication, app ID, and the model evaluation details.
# Change these strings to run your own example.
##################################################################################################

USER_ID = "YOUR_USER_ID_HERE"
# Your PAT (Personal Access Token) can be found in the Account's Security section
PAT = "YOUR_PAT_HERE"
APP_ID = "YOUR_APP_ID_HERE"
# Change these to get your model evaluation results
MODEL_ID = "YOUR_MODEL_ID_HERE"
MODEL_VERSION_ID = "YOUR_MODEL_VERSION_ID_HERE"
EVALUATION_ID = "YOUR_EVALUATION_ID_HERE"

##########################################################################
# YOU DO NOT NEED TO CHANGE ANYTHING BELOW THIS LINE TO RUN THIS EXAMPLE
##########################################################################

from clarifai_grpc.channel.clarifai_channel import ClarifaiChannel
from clarifai_grpc.grpc.api import resources_pb2, service_pb2, service_pb2_grpc
from clarifai_grpc.grpc.api.status import status_code_pb2

channel = ClarifaiChannel.get_grpc_channel()
stub = service_pb2_grpc.V2Stub(channel)

metadata = (("authorization", "Key " + PAT),)

userDataObject = resources_pb2.UserAppIDSet(user_id=USER_ID, app_id=APP_ID)

get_evaluation_response = stub.GetModelVersionEvaluation(
service_pb2.GetModelVersionEvaluationRequest(
user_app_id=userDataObject,
model_id=MODEL_ID,
model_version_id=MODEL_VERSION_ID,
evaluation_id=EVALUATION_ID,
),
metadata=metadata,
)

if get_evaluation_response.status.code != status_code_pb2.SUCCESS:
print(get_evaluation_response.status)
raise Exception("Get model metrics failed, status: " + get_evaluation_response.status.description)

print(get_evaluation_response)

ListModelVersionEvaluations

List the evaluation results of a model version in your app.

###################################################################################################
# In this section, we set the user authentication, app ID, and the model evaluation details.
# Change these strings to run your own example.
##################################################################################################

USER_ID = "YOUR_USER_ID_HERE"
# Your PAT (Personal Access Token) can be found in the Account's Security section
PAT = "YOUR_PAT_HERE"
APP_ID = "YOUR_APP_ID_HERE"
# Change these to get your model evaluation results
MODEL_ID = "YOUR_MODEL_ID_HERE"
MODEL_VERSION_ID = "YOUR_MODEL_VERSION_ID_HERE"

##########################################################################
# YOU DO NOT NEED TO CHANGE ANYTHING BELOW THIS LINE TO RUN THIS EXAMPLE
##########################################################################

from clarifai_grpc.channel.clarifai_channel import ClarifaiChannel
from clarifai_grpc.grpc.api import resources_pb2, service_pb2, service_pb2_grpc
from clarifai_grpc.grpc.api.status import status_code_pb2

channel = ClarifaiChannel.get_grpc_channel()
stub = service_pb2_grpc.V2Stub(channel)

metadata = (("authorization", "Key " + PAT),)

userDataObject = resources_pb2.UserAppIDSet(user_id=USER_ID, app_id=APP_ID)

list_evaluation_response = stub.ListModelVersionEvaluations(
service_pb2.ListModelVersionEvaluationsRequest(
user_app_id=userDataObject,
model_id=MODEL_ID,
model_version_id=MODEL_VERSION_ID
),
metadata=metadata,
)

if list_evaluation_response.status.code != status_code_pb2.SUCCESS:
print(list_evaluation_response.status)
raise Exception("Get model metrics failed, status: " + list_evaluation_response.status.description)

print(list_evaluation_response)
tip

You can also learn how to interpret a model's evaluation results via the Portal here.