InvalidExpectationConfigurationError when running an expectation on big query data source

GX OSS Support
1

Hi,
Could you please help me with the below issue.
I am connecting GX to a Big Query data source. I using 0.18.9 version of GX.

Below is my code:

from db_metadata_conn import
from datetime import datetime
import pandas as pd
from great_expectations.checkpoint import SimpleCheckpoint
from great_expectations.core.batch import BatchRequest, RuntimeBatchRequest
import json
from configparser import ConfigParser
from google.api_core.exceptions import NotFound

import great_expectations as gx
import os

os.environ[
‘GOOGLE_APPLICATION_CREDENTIALS’] = r’key.json’
projectID = ‘my_project_id’
bigquery_dataset = ‘my_dataset’
context = gx.get_context()
my_connection_string = f"bigquery://{projectID}/{bigquery_dataset}"
datasource_name = “my_bigquery_datasource”
datasource = context.sources.add_or_update_sql(
name=datasource_name,
connection_string=my_connection_string,
)
id=1

SET RUN_NAME & RUN TIME

run_name = f"{id}"
print(f"Run Name: {run_name}")

run_time = datetime.now()

run_time = datetime.now()

run_id = {
“run_name”: run_name,
“run_time”: run_time
}
i=100
expectation_suite_name = f"{id}.{i}"
expectation_code = ‘expect_column_values_to_not_be_in_set(“mortv_adj_rate_index”, [“6-MO-CMT”, “1-YR-TBILL”, “5-YR-CMT”])’
query_filter = "select * from dataset_id.table_name1 where date_col = 20231003 "

asset_name = f"{run_name}my_query_asset{i}"
query_asset = datasource.add_query_asset(name=asset_name, query=query_filter)
data_asset = context.get_datasource(datasource_name).get_asset(asset_name)
batch_request = data_asset.build_batch_request()

context.add_or_update_expectation_suite(expectation_suite_name)

validator = context.get_validator(
batch_request=batch_request,
expectation_suite_name=expectation_suite_name,
)

s = “validator.” + expectation_code
eval(s)
print(“saving to expectation suite…”)
validator.save_expectation_suite(
discard_failed_expectations=False)

checkpoint = context.add_or_update_checkpoint(
name=“my_checkpoint”,
validations=[
{
“batch_request”: batch_request,
“expectation_suite_name”: expectation_suite_name,
},
],
)

checkpoint_result = checkpoint.run()

print(checkpoint_result)

When i am running the same code on one table i am not getting any error while for another table I am getting the following error:

Traceback (most recent call last):
File “/root/ddq/DDQ_2/DDQ_GCP_02232024/DDQ_GE_GCP_Python/test_valid.py”, line 55, in
eval(s)
File “”, line 1, in
File “/usr/local/lib/python3.10/dist-packages/great_expectations/validator/validator.py”, line 591, in inst_expectation
raise err
File “/usr/local/lib/python3.10/dist-packages/great_expectations/validator/validator.py”, line 554, in inst_expectation
validation_result = expectation.validate(
File “/usr/local/lib/python3.10/dist-packages/great_expectations/expectations/expectation.py”, line 1287, in validate
] = validator.graph_validate(
File “/usr/local/lib/python3.10/dist-packages/great_expectations/validator/validator.py”, line 1090, in graph_validate
raise err
File “/usr/local/lib/python3.10/dist-packages/great_expectations/validator/validator.py”, line 1074, in graph_validate
result = configuration.metrics_validate(
File “/usr/local/lib/python3.10/dist-packages/great_expectations/core/expectation_configuration.py”, line 1494, in metrics_validate
return expectation_impl(self).metrics_validate(
File “/usr/local/lib/python3.10/dist-packages/great_expectations/expectations/expectation.py”, line 1061, in metrics_validate
_validate_dependencies_against_available_metrics(
File “/usr/local/lib/python3.10/dist-packages/great_expectations/expectations/expectation.py”, line 3739, in _validate_dependencies_against_available_metrics
raise InvalidExpectationConfigurationError(
great_expectations.exceptions.exceptions.InvalidExpectationConfigurationError: Metric (‘column_values.nonnull.unexpected_count’, ‘22a65771b06bd1c00f3b5a38eb5e1761’, ()) is not available for validation of {
“expectation_type”: “expect_column_values_to_not_be_in_set”,
“kwargs”: {
“column”: “mortv_adj_rate_index”,
“value_set”: [
“6-MO-CMT”,
“1-YR-TBILL”,
“5-YR-CMT”
],
“batch_id”: “my_bigquery_datasource-1_my_query_asset_100”
},
“meta”: {}
}. Please check your configuration.