Skip to main content

Data Records

Every source connector will emit data records into a stream. Every event will be in the following JSON format:

{ "schema": <schema>, payload: <event data> }

The data record has two parts: a schema and a payload. The payload is defined by the source connector. The schema is defined below. For an example, see Example Data Record.

Schema#

Meroxa will automatically record the schema of the payload within the Data Record and capture its changes over time. More specifically, the schema describes the format of all the fields within the payload.

For example, given a payload with the field before the following schema will be captured:

{
"type": "struct",
"fields": [
{
"type": "int32",
"optional": false,
"field": "id"
},
...
],
"optional": true,
"name": "dbserver1.inventory.customers.Value",
"field": "before"
}

For each payload field, its type (type), optionality (optional), and name (field) will be captured.

Example Data Record#

The following example is an event from a Postgres using Logical Replication source connector.

{
"schema": {
"type": "struct",
"fields": [
{
"type": "struct",
"fields": [
{
"type": "int32",
"optional": false,
"field": "id"
},
{
"type": "string",
"optional": false,
"field": "email"
},
{
"type": "string",
"optional": true,
"field": "name"
},
{
"type": "string",
"optional": true,
"field": "birthday"
},
{
"type": "int64",
"optional": false,
"name": "io.debezium.time.Timestamp",
"version": 1,
"field": "createdAt"
},
{
"type": "int64",
"optional": false,
"name": "io.debezium.time.Timestamp",
"version": 1,
"field": "updatedAt"
}
],
"optional": true,
"name": "resource_217_139903.public.User.Value",
"field": "before"
},
{
"type": "struct",
"fields": [
...
],
"optional": true,
"name": "resource_217_139903.public.User.Value",
"field": "after"
},
{
"type": "struct",
"fields": [
{
"type": "string",
"optional": false,
"field": "version"
},
{
"type": "string",
"optional": false,
"field": "connector"
},
{
"type": "string",
"optional": false,
"field": "name"
},
{
"type": "int64",
"optional": false,
"field": "ts_ms"
},
{
"type": "string",
"optional": true,
"name": "io.debezium.data.Enum",
"version": 1,
"parameters": {
"allowed": "true,last,false"
},
"default": "false",
"field": "snapshot"
},
{
"type": "string",
"optional": false,
"field": "db"
},
{
"type": "string",
"optional": false,
"field": "schema"
},
{
"type": "string",
"optional": false,
"field": "table"
},
{
"type": "int64",
"optional": true,
"field": "txId"
},
{
"type": "int64",
"optional": true,
"field": "lsn"
},
{
"type": "int64",
"optional": true,
"field": "xmin"
}
],
"optional": false,
"name": "io.debezium.connector.postgresql.Source",
"field": "source"
},
{
"type": "string",
"optional": false,
"field": "op"
},
{
"type": "int64",
"optional": true,
"field": "ts_ms"
},
{
"type": "struct",
"fields": [
{
"type": "string",
"optional": false,
"field": "id"
},
{
"type": "int64",
"optional": false,
"field": "total_order"
},
{
"type": "int64",
"optional": false,
"field": "data_collection_order"
}
],
"optional": true,
"field": "transaction"
}
],
"optional": false,
"name": "resource_217_139903.public.User.Envelope"
},
"payload": {
"before": {
"id": 11,
"email": "ec@example.com",
"name": "Nell Abbott",
"birthday": "12/21/1959",
"createdAt": 1618255874536,
"updatedAt": 1618255874537
},
"after": {
"id": 11,
"email": "nell-abbott@example.com",
"name": "Nell Abbott",
"birthday": "12/21/1959",
"createdAt": 1618255874536,
"updatedAt": 1618255874537
},
"source": {
"version": "1.2.5.Final",
"connector": "postgresql",
"name": "resource-217-139903",
"ts_ms": 1618255875129,
"snapshot": "false",
"db": "my_database",
"schema": "public",
"table": "User",
"txId": 8355,
"lsn": 478419097256,
"xmin": null
},
"op": "u",
"ts_ms": 1618255875392,
"transaction": null
}
}