Thank you @joshstauffer , following is the code snippet
# ============================
# General Config
# ============================
# Define output folder path under /kaggle/working/
output_base_path =
# Define Data-Source
source_folder =
data_source_name = “gx_sample_data_source”
Scan all CSV files in source-folder & store their filenames in a dict [file: filename]
csv_names = {}
for f in glob.glob(os.path.join(source_folder, “*.csv”)):
csv_names\[os.path.splitext(os.path.basename(f))\[0\]\] = os.path.basename(f)
# ============================
# GX Config
# ============================
# Define Context
context = gx.get_context(mode=“file”, project_root_dir=output_base_path)
# Add the Data-Source to the Data Context
try:
data_source = context.data_sources.get(data_source_name)
except KeyError:
data_source = context.data_sources.add_pandas_filesystem(
name=data_source_name,
base_directory=source_folder)
Define Data Asset
assets = {}
for csv in csv_names.keys():
dynamic_key = f"{csv}\_asset"
try:
assets\[dynamic_key\] = data_source.add_csv_asset(name=dynamic_key)
except ValueError:
assets\[dynamic_key\] = data_source.get_asset(dynamic_key)
Verification
for k in assets.keys():
print(f"Asset {k} created.")
============================
# GX Batch Load
# ============================
# Retrieve Data Asset
bugs_asset = assets[“BUGS_asset”]
Add Batch-Definition to Data Asset
batch_definition_name = “bugs_data”
batch_definition_path = list(csv_names.values())[0] # ← BUGS.csv
try:
batch_definition = bugs_asset.add_batch_definition_path(
name=batch_definition_name,
path=batch_definition_path
)
except ValueError:
batch_definition = bugs_asset.get_batch_definition(batch_definition_name)
Fetch batch
batch = batch_definition.get_batch()
batch.head()
=====================================================
# Define Expectations (data validation business rules)
# =====================================================
“”"
EXPECTATION#1: Column = workitem_type (type = Bug only)
-------------------------------------------------------------------------
Scope : Table-Specific (BUGS)
Business Rule : workitem_type column must strictly contain ONLY ‘Bug’.
Target Column : workitem_type
Validation : Value set = [‘Bug’]
Failure Impact: Critical - Unexpected workitem types present in dataset.
-------------------------------------------------------------------------
“”"
exp_workitem_type_is_bug = gx.expectations.ExpectColumnDistinctValuesToEqualSet(
column="workitem_type",
value_set=\["Bug"\]
)
“”"
EXPECTATION#2: Column = created_by (only MOJ internal users)
-------------------------------------------------------------------------
Scope : Generic
Business Rule : created_by must contain an email with a ‘@…justice’ domain.
Target Column : created_by
Validation : Matches regex r"@.*justice" (Case-insensitive)
Failure Impact: Warning — Invalid email format or non-justice domain detected.
-------------------------------------------------------------------------
“”"
exp_created_by_justice_domain = gx.expectations.ExpectColumnValuesToMatchRegex(
column="created_by",
regex=r"(?i)@.\*justice", # (?i) makes it case-insensitive
severity="warning"
)
“”"
EXPECTATION#3: Column = severity (non null)
-------------------------------------------------------------------------
Scope : Generic
Business Rule : severity must be non-null
Target Column : severity
Validation : values to be not null
Failure Impact: Critical - Null/blank present in severity column
-------------------------------------------------------------------------
“”"
exp_severity_notnull = gx.expectations.ExpectColumnValuesToNotBeNull(
column="severity"
)
“”"
EXPECTATION#4: Column = id (value length between 4 & 6)
-------------------------------------------------------------------------
Scope : Generic
Business Rule : id value length must be between 4 & 6
Target Column : id
Validation : min_value = 5 & max_value = 9
Failure Impact: Warning - Unknown Id values in the dataset
-------------------------------------------------------------------------
“”"
exp_id_length = gx.expectations.ExpectColumnValueLengthsToBeBetween(
column="id",
min_value=5,
max_value=9,
severity="warning"
)
“”"
EXPECTATION#5: Column = id (duplicate check)
-------------------------------------------------------------------------
Scope : Generic
Business Rule : primary key (id) should be unique
Target Column : id
Validation : no duplicates
Failure Impact: Critical
-------------------------------------------------------------------------
“”"
exp_unique_id = gx.expectations.ExpectColumnValuesToBeUnique(
column="id"
)
Expectations List
expectations_list = [
exp_workitem_type_is_bug, # <- workitemtype must be Bug
exp_created_by_justice_domain, # <- created_by email domain must be "justice"
exp_severity_notnull, # <- severity must non null
exp_id_length, # <- id length must be between 4 & 6
exp_unique_id # <- id must be unique
]
============================
# Expectation Suite
# ============================
# Define a Base Expectation Suite
suite_name = “base_expectation_suite”
Add suite to the context
try:
suite = context.suites.add(gx.ExpectationSuite(name=suite_name))
except gx.exceptions.DataContextError:
suite = context.suites.get(name=suite_name)
Add Expectations to Suite
for exp in expectations_list:
suite.add_expectation(exp)
Verification
print(f"Total expectations added: {len(suite.expectations)}")
============================
# Validation Definition
# ============================
Define unique validation definition name
val_definition_name = “bugs_validation_definition”
# Define validation definition & add to context
try:
validation_definition = context.validation_definitions.add(
gx.ValidationDefinition(
data=batch_definition,
suite=suite,
name=val_definition_name,
)
)
except (gx.exceptions.DataContextError, ValueError):
validation_definition = context.validation_definitions.get(
name=val_definition_name
)
============================
# Run Validation Definition
# ============================
Execute the validation run
validation_results = validation_definition.run()
# Check overall pass/fail status
print(f"Validation Result: {validation_results}")