import requests

url = "https://application_id.applications.lucidworks.com/ai/tokenization/{MODEL_ID}"

payload = {
    "batch": [{ "text": "Mr. and Mrs. Dursley and O'\''Malley, of number four, Privet Drive, were proud to say that they were perfectly normal, thank you very much" }],
    "useCaseConfig": { "dataType": "query or passage" },
    "modelConfig": {
        "vectorQuantizationMethod": "max-scale",
        "dimReductionSize": 256
    }
}
headers = {"Content-Type": "application/json"}

response = requests.post(url, json=payload, headers=headers)

print(response.json())

{
  "generatedTokens": [
    {
      "tokens": [
        "\"[CLS]\",                 \"query\",                 \":\",                 \"mr\",                 \".\",                 \"and\",                 \"mrs\",                 \".\",                 \"du\",                 \"##rs\",                 \"##ley\",                 \"and\",                 \"o\",                 \"'\",                 \"malley\",                 \",\",                 \"of\",                 \"number\",                 \"four\",                 \",\",                 \"pri\",                 \"##vet\",                 \"drive\",                 \",\",                 \"were\",                 \"proud\",                 \"to\",                 \"say\",                 \"that\",                 \"they\",                 \"were\",                 \"perfectly\",                 \"normal\",                 \",\",                 \"thank\",                 \"you\",                 \"very\",                 \"much\",                 \".\",                 \"[SEP]\""
      ]
    }
  ],
  "tokensUsed": {
    "inputTokens": 40,
    "promptTokens": 148,
    "completionTokens": 0,
    "totalTokens": 175
  }
}