curl --request POST \
  --url https://application_id.applications.lucidworks.com/ai/tokenization/{MODEL_ID} \
  --header 'Content-Type: application/json' \
  --data '{
  "batch": [
    {
      "text": "Mr. and Mrs. Dursley and O'\''\\'\'''\''Malley, of number four, Privet Drive, were proud to say that they were perfectly normal, thank you very much"
    }
  ],
  "useCaseConfig": {
    "dataType": "query or passage"
  },
  "modelConfig": {
    "vectorQuantizationMethod": "max-scale",
    "dimReductionSize": 256
  }
}'

{
  "generatedTokens": [
    {
      "tokens": [
        "\"[CLS]\",                 \"query\",                 \":\",                 \"mr\",                 \".\",                 \"and\",                 \"mrs\",                 \".\",                 \"du\",                 \"##rs\",                 \"##ley\",                 \"and\",                 \"o\",                 \"'\",                 \"malley\",                 \",\",                 \"of\",                 \"number\",                 \"four\",                 \",\",                 \"pri\",                 \"##vet\",                 \"drive\",                 \",\",                 \"were\",                 \"proud\",                 \"to\",                 \"say\",                 \"that\",                 \"they\",                 \"were\",                 \"perfectly\",                 \"normal\",                 \",\",                 \"thank\",                 \"you\",                 \"very\",                 \"much\",                 \".\",                 \"[SEP]\""
      ]
    }
  ],
  "tokensUsed": {
    "inputTokens": 40,
    "promptTokens": 148,
    "completionTokens": 0,
    "totalTokens": 175
  }
}