Below is the sample code and the error corresponding to that:
Code Snip:
“id=1
run_name=f”{id}"
run_time=datetime.datetime.now()
run_id = {
“run_name”: run_name,
“run_time”: run_time
}
query_filter = “select * from <snowflake_table> where <filter_condition>”
batch_request=RuntimeBatchRequest(
datasource_name=“my_spark_datasource_config”,
data_connector_name=“default_runtime_data_connector_name”,
data_asset_name=“my_data_asset_name”,
runtime_parameters={
“query”: query_filter
},
batch_identifiers={“d”: “batch_run_id”}
)
expectation_suite_name = “dry_run_2”
suite = context.add_or_update_expectation_suite(expectation_suite_name)
validator = context.get_validator(
batch_request=batch_request,
expectation_suite_name=expectation_suite_name,
)
expectation_code = ‘expect_column_values_to_be_in_set(value_set= [1, 2, 3, 4, 5, 134, 7, 8, 12, 13, 14, 15, 16, 17, 18, 24, 25, 26, 27, 28, 29, 30, 31, 157, 36, 37, 38, 39, 40, 41, 42, 48, 49, 50, 51, 180, 181, 182, 183, 59, 60, 61, 62, 63, 64, 65, 66, 67, 69, 71, 73, 74, 81, 84, 85, 86, 87, 88, 89, 90, 95, 97, 98, 101, 109, 120, 121, 122, 123], column= “first_change_period_n” , mostly= 1.0)’
s=“validator.”+expectation_code
eval(s)
validator.save_expectation_suite(discard_failed_expectations=False)
checkpoint_config = {
“class_name”: “SimpleCheckpoint”,
“validations”: [
{
“batch_request”: batch_request,
“expectation_suite_name”: expectation_suite_name
}
],
“runtime_configuration”: {
“result_format”: {
“result_format”: “SUMMARY”,
“include_unexpected_count”: True,
“include_element_count”: True
}
}
}
checkpoint = SimpleCheckpoint(f"{validator.active_batch_definition.data_asset_name}_{expectation_suite_name}", context, **checkpoint_config)
checkpoint_result = checkpoint.run(run_id=run_id)"
Error:
2024-01-02T13:25:19+0000 - INFO - 1 expectation(s) included in expectation_suite.
Calculating Metrics: 0%| | 0/13 [00:00<?, ?it/s]
2024-01-02T13:25:19+0000 - INFO - While introspecting columns NoSuchTableError(); attempting reflection fallback
2024-01-02T13:25:27+0000 - ERROR - Error running action with name store_evaluation_params
Traceback (most recent call last):
File “/databricks/python/lib/python3.8/site-packages/great_expectations/validation_operators/validation_operators.py”, line 478, in _run_actions
action_result = self.actions[name].run(
File “/databricks/python/lib/python3.8/site-packages/great_expectations/checkpoint/actions.py”, line 100, in run
return self._run(
File “/databricks/python/lib/python3.8/site-packages/great_expectations/checkpoint/actions.py”, line 1015, in _run
self.data_context.store_evaluation_parameters(validation_result_suite)
File “/databricks/python/lib/python3.8/site-packages/great_expectations/data_context/data_context/abstract_data_context.py”, line 2169, in store_evaluation_parameters
self._compile_evaluation_parameter_dependencies()
File “/databricks/python/lib/python3.8/site-packages/great_expectations/data_context/data_context/abstract_data_context.py”, line 4742, in _compile_evaluation_parameter_dependencies
expectation_suite_dict: dict = cast(dict, self.expectations_store.get(key))
File “/databricks/python/lib/python3.8/site-packages/great_expectations/data_context/store/expectations_store.py”, line 210, in get
return super().get(key) # type: ignore[return-value]
File “/databricks/python/lib/python3.8/site-packages/great_expectations/data_context/store/store.py”, line 210, in get
return self.deserialize(value)
File “/databricks/python/lib/python3.8/site-packages/great_expectations/data_context/store/expectations_store.py”, line 236, in deserialize
return self._expectationSuiteSchema.loads(value)
File “/databricks/python/lib/python3.8/site-packages/marshmallow/schema.py”, line 756, in loads
return self.load(data, many=many, partial=partial, unknown=unknown)
File “/databricks/python/lib/python3.8/site-packages/marshmallow/schema.py”, line 722, in load
return self._do_load(
File “/databricks/python/lib/python3.8/site-packages/marshmallow/schema.py”, line 909, in _do_load
raise exc
marshmallow.exceptions.ValidationError: {‘expectations’: {0: {‘expectation_type’: [‘expectation_type missing in expectation configuration’]}}}