Hello GE team!
I have an AzureML pipeline that runs 2 expectation suites on a PythonScriptStep (https://docs.microsoft.com/en-us/python/api/azureml-pipeline-steps/azureml.pipeline.steps.python_script_step.pythonscriptstep?view=azure-ml-py)
When I run it I get an InternalServerError after the script finalizes, could this be a GE error?
The experiment completed successfully. Finalizing run…
Cleaning up all outstanding Run operations, waiting 300.0 seconds
2 items cleaning up…
Exception encountered while calling history_context.exit . Exception will be ignored to avoid a run failure because the step is optional.
Traceback (most recent call last):
File “/azureml-envs/azureml_d1e873111d8d718983a7789de3851c09/lib/python3.6/site-packages/azureml/_restclient/workspace_client.py”, line 87, in _execute_with_arguments
return self._call_api(func, *args_list, **kwargs)
File “/azureml-envs/azureml_d1e873111d8d718983a7789de3851c09/lib/python3.6/site-packages/azureml/_restclient/clientbase.py”, line 227, in _call_api
return self._execute_with_base_arguments(func, *args, **kwargs)
File “/azureml-envs/azureml_d1e873111d8d718983a7789de3851c09/lib/python3.6/site-packages/azureml/_restclient/clientbase.py”, line 280, in _execute_with_base_arguments
back_off, total_retry, self._logger, func, _noop_reset, *args, **kwargs)
File “/azureml-envs/azureml_d1e873111d8d718983a7789de3851c09/lib/python3.6/site-packages/azureml/_restclient/clientbase.py”, line 295, in _execute_func_internal
left_retry = cls._handle_retry(back_off, left_retry, total_retry, error, logger, func)
File “/azureml-envs/azureml_d1e873111d8d718983a7789de3851c09/lib/python3.6/site-packages/azureml/_restclient/clientbase.py”, line 325, in _handle_retry
raise error
File “/azureml-envs/azureml_d1e873111d8d718983a7789de3851c09/lib/python3.6/site-packages/azureml/_restclient/clientbase.py”, line 293, in _execute_func_internal
return func(*args, **kwargs)
File “/azureml-envs/azureml_d1e873111d8d718983a7789de3851c09/lib/python3.6/site-packages/azureml/_restclient/operations/run_metric_operations.py”, line 161, in post_batch
raise models.ErrorResponseException(self._deserialize, response)
azureml._restclient.models.error_response.ErrorResponseException: (ServiceError) InternalServerError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File “/mnt/batch/tasks/shared/LS_root/jobs/avadevitsmlsvc/azureml/4d38e953-0f95-4c68-8f65-50761287ae2b/mounts/workspaceblobstore/azureml/4d38e953-0f95-4c68-8f65-50761287ae2b/azureml-setup/context_managers.py”, line 155, in exit
self.history_context.exit(*args)
File “/azureml-envs/azureml_d1e873111d8d718983a7789de3851c09/lib/python3.6/site-packages/azureml/_history/utils/context_managers.py”, line 57, in exit
return self._exit_stack.exit(*args)
File “/azureml-envs/azureml_d1e873111d8d718983a7789de3851c09/lib/python3.6/contextlib.py”, line 380, in exit
raise exc_details[1]
File “/azureml-envs/azureml_d1e873111d8d718983a7789de3851c09/lib/python3.6/contextlib.py”, line 365, in exit
if cb(*exc_details):
File “/azureml-envs/azureml_d1e873111d8d718983a7789de3851c09/lib/python3.6/contextlib.py”, line 284, in _exit_wrapper
return cm_exit(cm, *exc_details)
File “/azureml-envs/azureml_d1e873111d8d718983a7789de3851c09/lib/python3.6/site-packages/azureml/_history/utils/context_managers.py”, line 165, in exit
_RunBase._kill(timeout=self._kill_timeout)
File “/azureml-envs/azureml_d1e873111d8d718983a7789de3851c09/lib/python3.6/site-packages/azureml/_run_impl/run_base.py”, line 159, in _kill
handler(timeout)
File “/azureml-envs/azureml_d1e873111d8d718983a7789de3851c09/lib/python3.6/site-packages/azureml/_run_impl/run_base.py”, line 169, in _cleanup
self._client.flush(timeout)
File “/azureml-envs/azureml_d1e873111d8d718983a7789de3851c09/lib/python3.6/site-packages/azureml/_run_impl/run_history_facade.py”, line 569, in flush
self.metrics.flush(timeout_seconds=timeout_seconds)
File “/azureml-envs/azureml_d1e873111d8d718983a7789de3851c09/lib/python3.6/site-packages/azureml/_restclient/metrics_client.py”, line 240, in flush
self._task_queue.flush(self.identity, timeout_seconds=timeout_seconds)
File “/azureml-envs/azureml_d1e873111d8d718983a7789de3851c09/lib/python3.6/site-packages/azureml/_common/async_utils/batch_task_queue.py”, line 89, in flush
super(BatchTaskQueue, self).flush(*args, **kwargs)
File “/azureml-envs/azureml_d1e873111d8d718983a7789de3851c09/lib/python3.6/site-packages/azureml/_common/async_utils/task_queue.py”, line 112, in flush
self._results.extend((task.wait(awaiter_name=self.identity) for task in completed_tasks))
File “/azureml-envs/azureml_d1e873111d8d718983a7789de3851c09/lib/python3.6/site-packages/azureml/_common/async_utils/task_queue.py”, line 112, in
self._results.extend((task.wait(awaiter_name=self.identity) for task in completed_tasks))
File “/azureml-envs/azureml_d1e873111d8d718983a7789de3851c09/lib/python3.6/site-packages/azureml/_common/async_utils/async_task.py”, line 59, in wait
res = self._handler(self._future, self._logger)
File “/azureml-envs/azureml_d1e873111d8d718983a7789de3851c09/lib/python3.6/site-packages/azureml/_common/async_utils/async_task.py”, line 16, in basic_handler
return future.result()
File “/azureml-envs/azureml_d1e873111d8d718983a7789de3851c09/lib/python3.6/concurrent/futures/_base.py”, line 398, in result
return self.__get_result()
File “/azureml-envs/azureml_d1e873111d8d718983a7789de3851c09/lib/python3.6/concurrent/futures/_base.py”, line 357, in __get_result
raise self._exception
File “/azureml-envs/azureml_d1e873111d8d718983a7789de3851c09/lib/python3.6/concurrent/futures/thread.py”, line 55, in run
result = self.fn(*self.args, **self.kwargs)
File “/azureml-envs/azureml_d1e873111d8d718983a7789de3851c09/lib/python3.6/site-packages/azureml/_restclient/metrics_client.py”, line 271, in _log_batch
is_async=is_async)
File “/azureml-envs/azureml_d1e873111d8d718983a7789de3851c09/lib/python3.6/site-packages/azureml/_restclient/run_client.py”, line 542, in _execute_with_run_arguments
return self._execute_with_arguments(func, copy.deepcopy(self._run_arguments), *args, **kwargs)
File “/azureml-envs/azureml_d1e873111d8d718983a7789de3851c09/lib/python3.6/site-packages/azureml/_restclient/workspace_client.py”, line 89, in _execute_with_arguments
raise ServiceException(e)
azureml._restclient.exceptions.ServiceException: ServiceException:
Code: 500
Message: (ServiceError) InternalServerError
Details:
Headers: {
"Date": "Mon, 01 Jun 2020 17:35:09 GMT",
"Content-Type": "application/json; charset=utf-8",
"Transfer-Encoding": "chunked",
"Connection": "keep-alive",
"Request-Context": "appId=cid-v1:2d2e8e63-272e-4b3c-8598-4ee570a0e70d",
"x-ms-response-type": "error",
"x-ms-client-request-id": "a53222ab-eefa-40a6-8236-b49d85e94471",
"x-ms-client-session-id": "",
"Strict-Transport-Security": "max-age=15724800; includeSubDomains; preload"
}
InnerException: {
"additional_properties": {},
"error": {
"additional_properties": {},
"code": "ServiceError",
"message": "InternalServerError",
"message_format": null,
"message_parameters": null,
"reference_code": null,
"details_uri": null,
"target": null,
"details": [],
"inner_error": null,
"debug_info": null
},
"correlation": {
"operation": "064efa70e138c140b8110cb1ae0e4091",
"request": "8879d1c2fbf0f377"
},
"environment": "westus2",
"location": "westus2",
"time": {}
}