Skip to main content

Real-Time Intelligence

New

Improving the error message when a data type conversion issue occurs writing onto a lakehouse

Vote (1) Share
Pasquale Massimo Scorca's profile image

Pasquale Massimo Scorca on 15 Oct 2024 16:34:50

Hi,

for an evenstream that reads from an Azure Event Hubs to write the events onto a lakehouse I've encountered a data type conversion error and looking to the Run-time logs I've seen a such generic error message:


{

  "ErrorCode": {

    "value": "Warning",

    "type": "string"

  },

  "TimeGenerated": {

    "value": "2024-10-14 13:39:48Z",

    "type": "datetime"

  },

  "Level": {

    "value": "Warning",

    "type": "string"

  },

  "Category": {

    "value": "Warning",

    "type": "string"

  },

  "ResourceId": {

    "value": "",

    "type": "string"

  },

  "SubscriptionId": {

    "value": "",

    "type": "string"

  },

  "ResourceGroup": {

    "value": "",

    "type": "string"

  },

  "JobName": {

    "value": "",

    "type": "string"

  },

  "OperationName": {

    "value": "",

    "type": "string"

  },

  "Status": {

    "value": "Warning",

    "type": "string"

  },

  "Source": {

    "value": "",

    "type": "string"

  },

  "TypeProperty": {

    "value": "Warning",

    "type": "string"

  },

  "DataErrorType": {

    "value": "",

    "type": "string"

  },

  "ErrorCategory": {

    "value": "Warning",

    "type": "string"

  },

  "BriefMessage": {

    "value": "",

    "type": "string"

  },

  "Message": {

    "value": "First Occurred: 10/14/2024 1:39:46 PM UTC | Resource Name: 234dbbd1-4cb0-ab3b-7e66-a02a98fe0732 | Message: Source '234dbbd1-4cb0-ab3b-7e66-a02a98fe0732' had 1 occurrences of kind 'OutputDataConversionError.TypeConversionError' between processing times '2024-10-14T13:39:46.5109807Z' and '2024-10-14T13:39:46.7587399Z'. \r\n",

    "type": "string"

  },

  "FromTimestamp": {

    "value": null,

    "type": "datetime"

  },

  "ToTimestamp": {

    "value": null,

    "type": "datetime"

  },

  "EventCount": {

    "value": 0,

    "type": "long"

  },

  "GroupSize": {

    "value": 1,

    "type": "long"

  }

}


It could be very useful to have a major detail in order to detect the specific event data that causes the issue. Thanks