External Ingestion Batch Json Schema

Scan Endpoint

Request Body

1
2
{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "External Ingestion Scan Endpoint Response",
  "description": "Schema for Scan Endpoints Response",
  "required": ["entityIds"],
  "type": "object",
  "properties": {
    "entityIds": {
      "type": "array",
      "items" : {"$ref": "#/$defs/entityId"}
      "description": "List of identifiers that will be used to call Query on. The page-size returned should around 1000 entities per page."
    },
    "pageToken": {
      "type": "string",
      "description": "Page token (usually of the last record) of the current page or null if no more pages."
    }
  },
  "$defs": {
    "entityId": {
      "type": "string"
    }
  },
  "additionalProperties": false
}

Request Example

1
2
curl --request POST \
  --url 'https://localhost:8080/ref/scan' \
  --header 'Content-Type: application/json' \
  --data '{
  "pageToken": "some-entity-id-1" 
}'

Response Body

1
2
{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "External Ingestion Query Endpoint Request",
  "description": "Schema for Query Endpoints Request",
  "required": ["entityIds"],
  "type": "object",
  "properties": {
    "wrenchJobId": {
      "type": "string",
      "description": "Associated Wrench Job Id. This is an opaque, passed through field."
    },
    "entityIds": {
      "type": "array",
      "items": {"$ref": "#/$defs/entityId"},
      "description": "List of entities to be processed. Each EntityId is a string of the Entities' Id."
    }
  },
  "$defs": {
    "entityId": {
      "type": "string"
    }
  },
  "additionalProperties": false
}

Response Example

1
2
{
  "entityIds": [
    "some-entity-id-1",
    "some-entity-id-2",
    "some-entity-id-3"
  ],
  "pageToken": "some-entity-id-3"
}

Query Endpoint

Request Body

1
2
{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "External Ingestion Query Endpoint Request",
  "description": "Schema for Query Endpoints Request",
  "required": ["entityIds"],
  "type": "object",
  "properties": {
    "wrenchJobId": {
      "type": "string",
      "description": "Associated Wrench Job Id. This is an opaque, passed through field."
    },
    "entityIds": {
      "type": "array",
      "items": {"$ref": "#/$defs/entityId"},
      "description": "List of entities to be processed. Each EntityId is a string of the Entities' Id."
    }
  },
  "$defs": {
    "entityId": {
      "type": "string"
    }
  },
  "additionalProperties": false
}

Request Example

1
2
curl --request POST \
  --url 'https://localhost:8080/ref/query' \
  --header 'Content-Type: application/json' \
  --data '{
  "wrenchJobId": "some-job-id",
  "entityIds": [
    "some-entity-id-1",
    "some-entity-id-2",
    "some-entity-id-3"
  ]
}'

Response Body

1
2
{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "External Ingestion Query Endpoint Response",
  "description": "Schema for Query Endpoints Response",
  "type": "object",
  "properties": {
    "operationStatusResults": {
      "type": "object",
      "patternProperties": {
        "^(SUCCESS|DELETE_SUCCESS|RETRYABLE_FAILURE|PERMANENT_FAILURE|VALIDATION_FAILURE|PROCESSING_ERROR|SKIPPED|MESSAGE_QUEUE_FAILURE|NOT_FOUND)$": {
          "type": "array",
          "items": {
            "type": "string"
          },
          "description": "An array of Entity Ids which correspond with the status."
        }
      },
      "description": "The Operation Status of each Entity Id processed."
    },
    "entities": {
      "type": "array",
      "items": {
        "type": "object",
        "properties": {
          "entityId": {
            "type": "string",
            "description": "EntityId that was operated on"
          },
          "entityType": {
            "type": "string",
            "description": "Type of entity that was operated on. Note that there is an allowed list of event producers to entityType and events not matching this list will be ignored"
          },
          "version": {
            "type": "integer",
            "description": "Version number of the ingestion record. This can be used to ensure ordering of events, as it is strictly monotonically increasing, though not necessarily by only 1 each time"
          },
          "lastUpdateMillis": {
            "type": "integer",
            "description": "Timestamp when the ingestion record is updated in the provider's storage."
          },
          "deleted": {
            "type": "boolean",
            "description": "Determine if the record should be deleted."
          },
          "content": {
            "type": "string",
            "description": "The string-serialised content object to be operated on."
          },
          "base64GzipContent": {
            "type": "string",
            "description": "The base64 gzipped of string-serialised content object to be operated on."
          }
        },
        "oneOf": [
          {
            "required": [
              "entityId",
              "entityType",
              "version",
              "lastUpdateMillis",
              "deleted",
              "content"
            ]
          },
          {
            "required": [
              "entityId",
              "entityType",
              "version",
              "lastUpdateMillis",
              "deleted",
              "base64GzipContent"
            ]
          }
        ],
        "additionalProperties": false
      },
      "description": "A list of External Entities to be ingested."
    },
    "timestamp": {
      "type": "integer",
      "description": "Timestamp when the backfilling is requested"
    }
  },
  "required": [
    "operationStatusResults",
    "entities",
    "timestamp"
  ],
  "additionalProperties": false
}

Response Example

1
2
{
  "operationStatusResults": {
      "SUCCESS": [
        "success-id-1",
        "success-id-2"
      ]
  },
  "entities": [
    {
      "entityId": "success-id-1",
      "entityType": "SOME_ENTITY_TYPE",
      "version": 123,
      "lastUpdateMillis": 1656406528,
      "deleted": false,
      "content": "{\"data\":\"somecontent\"}"
    },
    {
      "entityId": "success-id-2",
      "entityType": "SOME_ENTITY_TYPE",
      "version": 123456,
      "lastUpdateMillis": 1656406528,
      "deleted": false,
      "content": "{\"data\":\"somecontent\"}"
    }
  ],
  "timestamp": 1656406530
}