> ## Documentation Index > Fetch the complete documentation index at: https://doc.lucidworks.com/llms.txt > Use this file to discover all available pages before exploring further. # Machine Learning > Query pipeline stage configuration specifications export const schema = { "type": "object", "title": "Machine Learning", "description": "Generate predictions from a machine learning model", "required": ["serviceId", "modelId", "inputScript"], "properties": { "skip": { "type": "boolean", "title": "Skip This Stage", "description": "Set to true to skip this stage.", "default": false, "hints": ["advanced"] }, "label": { "type": "string", "title": "Label", "description": "A unique label for this stage.", "hints": ["advanced"], "maxLength": 255 }, "condition": { "type": "string", "title": "Condition", "description": "Define a conditional script that must result in true or false. This can be used to determine if the stage should process or not.", "hints": ["code", "code/javascript", "advanced"] }, "legacy": { "type": "boolean", "title": "Legacy", "description": "True if this stage only supports legacy mode", "hints": ["readonly", "hidden"] }, "serviceId": { "type": "string", "title": "Service ID of ML Model Service", "description": "Service ID of ML Model Service", "default": "ml-model-service", "hints": ["advanced", "hidden"] }, "modelId": { "type": "string", "title": "Model ID", "description": "Model ID" }, "contextKey": { "type": "string", "title": "Name of context key to store prediction", "description": "Name of context key to store prediction" }, "failOnError": { "type": "boolean", "title": "Fail on Error", "description": "Flag to indicate if this stage should throw an exception if an error occurs while generating a prediction for a document.", "default": false }, "inputScript": { "type": "string", "title": "Model input transformation script", "description": "Javascript code that returns a HashMap contains fields and values to send to ML model service. Refer to examples.", "default": "/*\n\nThis script must contruct a HashMap containing fields and values to be sent to the ML model service.\nThe field names and values will depend on the input schema of the model.\nGenerally, you'll be reading fields and values from the request/context/response and placing them into a HashMap.\n\nValue types supported are:\n- String\n- Double\n- String[]\n- double[]\n- List\n- List\n\nThis script receives these objects and can be referenced in your script:\n- request\n- response\n- context\n- log (Logger useful for debugging)\n\nThe last line of the script must be a reference to the HashMap object you created.\n\nExample 1: Single string parameter from request to modelInput HashMap\nvar modelInput = new java.util.HashMap()\nmodelInput.put(\"input_1\", request.getFirstParam(\"q\"))\nmodelInput\n\nExample 2: List of strings from request to modelInput HashMap\nvar modelInput = new java.util.HashMap()\nmodelInput.put(\"input_1\", request.getParam(\"q\")) // request.getParam returns a Collection\nmodelInput\n\nExample 3: List of numeric values from request to modelInput HashMap\nvar modelInput = new java.util.HashMap()\nvar list = new java.util.ArrayList()\nlist.add(Double.parseDouble(request.getFirstParam(\"numeric_1\")))\nlist.add(Double.parseDouble(request.getFirstParam(\"numeric_2\")))\nmodelInput.put(\"input_1\", list)\nmodelInput\n\nExample 4: To use the spark ml based models, use the below code\nvar modelInput = new java.util.HashMap()\nmodelInput.put(\"concatField\", request.getFirstParam(\"q\"))\nmodelInput\n\n*/\n", "hints": ["lengthy", "code", "code/javascript"] }, "outputScript": { "type": "string", "title": "Model output transformation script", "description": "Javascript code that receives output from ML service as a HashMap called \"modelOutput\". Most of the time this is used to place prediction results in the request or context. Refer to examples.", "default": "/*\n\nThis output script receives the output prediction from the ML model service as a HashMap called \"modelOutput\".\nMost of the time this is used to place prediction results in the request or context for downstream pipeline stages\nto consume.\n\nThis script receives these objects and can be referenced in your script:\n- modelOutput (a HashMap containing fields/values returned from ML model service)\n- request\n- response\n- context\n- log (Logger useful for debugging)\n\nExample: Place predictedLabel (string) on request\nrequest.putSingleParam(\"sentiment\", modelOutput.get(\"predictedLabel\"))\n\n*/\n", "hints": ["lengthy", "code", "code/javascript"] } }, "category": "AI", "categoryPriority": 10, "unsafe": false }; export const SchemaParamFields = ({schema}) => { const sanitize = str => { if (typeof str !== "string") return str; return str.replace(/^"(.*)"$/s, "$1").replace(/\\/g, "").replace(/"/g, "'"); }; const formatDescription = str => { const s = sanitize(str); return (/[.!?]\)*$/).test(s) ? s : `${s}.`; }; const {description, properties = {}, required: requiredProps = []} = schema; const visibleProps = useMemo(() => Object.entries(properties).filter(([, prop]) => !prop.hints?.includes("hidden")), [properties]); return
{description &&

{formatDescription(description)}

} {visibleProps.map(([name, prop]) => { const isRequired = requiredProps.includes(name); const hasDefault = prop.default !== undefined; const rawDefault = prop.default; const isComplexDefault = hasDefault && (typeof rawDefault === "object" || typeof rawDefault === "string" && (rawDefault.length > 20 || rawDefault.includes('"'))); const fieldProps = { key: name, body: prop.title || name, type: prop.type, ...prop.title && ({ post: [<>API property: {name}] }), ...isRequired && ({ required: true }), ...!isComplexDefault && hasDefault ? { default: sanitize(String(rawDefault)) } : {} }; const isObject = prop.type === "object" && prop.properties; const isArrayOfObjects = prop.type === "array" && prop.items?.type === "object" && prop.items.properties; return {prop.description &&

{formatDescription(prop.description)}

} {isComplexDefault &&

Default: 

                  
                    {JSON.stringify(rawDefault, null, 2)}
} {isArrayOfObjects &&

Object attributes: 

                
                  {'{\n'}
                  {Object.entries(prop.items.properties).map(([iname, iprop]) => <>
                      {`  ${iname}`}
                      {prop.items?.required?.includes(iname) &&  required}
                      {`: {\n    display name: ${sanitize(iprop.title || '')}\n    type: ${iprop.type}\n  }\n`}
                    )}
                  {'}'}
} {isObject && } ; })}
; }; export const LwTemplate = ({title = "Key questions to get you started", icon = "sparkles", cta = "Powered by Agent Studio", linkHref = "https://lucidworks.com/demo/?utm_source=docs&utm_medium=referral&utm_campaign=docs_cta_ai"}) => { const [isLoaded, setIsLoaded] = useState(false); useEffect(() => { const timer = setTimeout(() => { setIsLoaded(true); }, 500); return () => clearTimeout(timer); }, []); return
{isLoaded && ` }} />} Powered by Lucidworks Agent Studio
; }; [localhost link]: http://localhost:3000/docs/lucidworks-search/09-developer-documentation/config-specs/query-pipeline-stages/machine-learning [mintlify link]: https://doc.lucidworks.com/docs/lucidworks-search/09-developer-documentation/config-specs/query-pipeline-stages/machine-learning [old doc.lw link]: https://doc.lucidworks.com/managed-fusion/5.9/43gkel The Machine Learning query pipeline stage uses a trained machine learning model to analyze a field or fields of a [Request](https://javadoc.lucidworks.com/fusion-pipeline-javadocs/5.9/?com/lucidworks/apollo/pipeline/query/Request.html) object and stores the results of analysis in a new field added to either the Request or the [Context](https://javadoc.lucidworks.com/fusion-pipeline-javadocs/4.2/com/lucidworks/apollo/pipeline/Context.html) object. In order to use the Machine Learning Stage, you must train a machine learning model. There are two different ways to train a model: * Use a Lucidworks Search job that trains a model, like [Classification](/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/classification) * **Develop and Deploy a Machine Learning Model** This tutorial walks you through deploying your own model to Fusion with Seldon Core. ## Prerequisites * A Fusion instance with an app and indexed data * An understanding of Python and the ability to write Python code * [Docker](https://docs.docker.com/get-docker/) installed locally, plus a private or public Docker repository * Seldon-core installed locally: `pip install seldon-core` * Code editor; you can use any editor, but Visual Studio Code is used in the example * Model: [paraphrase-multilingual-MiniLM-L12-v2 from Hugging Face](https://huggingface.co/sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2) * Docker image: [example\_sbert\_model](https://hub.docker.com/r/jstrmec/example_sbert_model/) ## Tips * Always test your Python code locally before uploading to Docker and then Fusion. This simplifies troubleshooting significantly. * Once you’ve created your Docker you can also test locally by doing `docker run` with a specified port, like 9000, which you can then `curl` to confirm functionality in Fusion. See the testing example below. The course for **Intro to Machine Learning in Fusion** focuses on using machine learning to infer the goals of customers and users in order to deliver a more sophisticated search experience. ## Local testing example The examples in this section use the following models: * [multilingual miniLM model](https://colab.research.google.com/drive/1LeCGl57_Evn_PCL25NJAt1MzP6GYH4j2) * [e5-model](https://colab.research.google.com/drive/15WOHqqMBng5abUsWJYdElxgVLdAoJdn2) 1. Docker command: ```bash theme={"dark"} docker run -p 127.0.0.1:9000:9000 ``` 2. Curl to hit Docker: ```bash theme={"dark"} curl -X POST -H 'Content-Type: application/json' -d '{"data": { "ndarray": ["Sentence to test"], "names":["text"]} }' https://localhost:9000/api/v1.0/predictions ``` 3. Curl model in Fusion: ```bash theme={"dark"} curl -u $FUSION_USER:$FUSION_PASSWORD -X POST -H 'Content-Type: application/json' -d '{"text": "i love fusion"}' https://.lucidworks.com:6764/api/ai/ml-models//prediction ``` 4. See all your deployed models: ```bash theme={"dark"} curl -u USERNAME:PASSWORD http://FUSION_HOST:FUSION_PORT/api/ai/ml-models ``` ## Download the model This tutorial uses the [paraphrase-multilingual-MiniLM-L12-v2](https://huggingface.co/sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2) model from Hugging Face, but any pre-trained model from [https://huggingface.co](https://huggingface.co) will work with this tutorial. If you want to use your own model instead, you can do so, but your model must have been trained and then saved though a function similar to the PyTorch’s `torch.save(model, PATH)` function. See [Saving and Loading Models](https://pytorch.org/tutorials/beginner/saving_loading_models.html) in the PyTorch documentation. ## Format a Python class The next step is to format a Python class which will be invoked by Fusion to get the results from your model. The skeleton below represents the format that you should follow. See also [Packaging a Python model for Seldon Core using Docker](https://docs.seldon.ai/seldon-core-1/configuration/wrappers-and-sdks/python-language-wrapper/python_wrapping_docker) in the Seldon Core documentation. ```python wrap expandable theme={"dark"} class MyModel(object): """ Model template. You can load your model parameters in __init__ from a location accessible at runtime """ def __init__(self): """ Add any initialization parameters. These will be passed at runtime from the graph definition parameters defined in your seldondeployment kubernetes resource manifest. """ print("Initializing") def predict(self,X,features_names,**kwargs): """ Return a prediction. Parameters ---------- X : array-like feature_names : array of feature names (optional) """ print("Predict called - will run identity function") return X def class_names(self): return ["X_name"] ``` A real instance of this class with the Paraphrase Multilingual MiniLM L12 v2 model is as follows: ```python wrap expandable theme={"dark"} import logging import os from transformers import AutoTokenizer, AutoModel from torch.nn import functional as F from typing import Iterable import numpy as np import torch log = logging.getLogger() class mini(): def __init__(self): self.tokenizer = AutoTokenizer.from_pretrained('sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2') self.model= AutoModel.from_pretrained('sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2') #Mean Pooling def mean_pooling(self, model_output, attention_mask): token_embeddings = model_output[0] #First element of model_output contains all token embeddings input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float() return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9) def predict(self, X:np.ndarray, names=None, **kwargs): # In Fusion there are several variables passed in the numpy array with the Milvus Query stage, # Encode to Milvus index stage, and Vectorize Seldon index and query stage: # [pipeline, bool, and text]. Text is what variable will be encoded, so that is what will be set to 'text' # When using the Machine Learning stage, the input map keys should match what what is in this file. model_input = dict(zip(names, X)) text = model_input["text"] with torch.inference_mode(): # Allows torch to run more quickly # Tokenize sentences encoded_input = self.tokenizer(text, padding=True, truncation=True, return_tensors='pt') log.debug('encoded input',str(encoded_input)) model_output = self.model(**encoded_input) log.debug('model output',str(model_output)) # Perform pooling. In this case, max pooling. sentence_embeddings = self.mean_pooling(model_output, encoded_input['attention_mask']) # Normalize embeddings, because Fusion likes it that way. sentence_embeddings = F.normalize(sentence_embeddings, p=2, dim=-1) # Fixing the shape of the emebbedings to match (1, 384). final = [sentence_embeddings.squeeze().cpu().detach().numpy().tolist()] return final def class_names(self) -> Iterable[str]: return ["vector"] ``` In the above code, an additional function has been added in the class; this is completely fine to do. Logging has also been added for debugging purposes. Two functions are non-negotiable: * `init`: The `init` function is where models, tokenizers, vectorizers, and the like should be set to self for invoking.\ It is recommended that you include your model’s trained parameters directly into the Docker container rather than reaching out to external storage inside `init`. * `predict`: The `predict` function processes the field or query that Fusion passes to the model.\ The `predict` function must be able to handle any text processing needed for the model to accept input invoked in its `model.evaluate()`, `model.predict()`, or equivalent function to get the expected model result. If the output needs additional manipulation, that should be done before the result is returned.\ For embedding models the return value must have the shape of (1, DIM), where DIM (dimension) is a consistent integer, to enable Fusion to handle the vector encoding into Milvus or Solr. Use the *exact* name of the class when naming this file. For the example, above the Python file is named `mini.py` and the class name is `mini()`. ## Create a Dockerfile The next step is to create a Dockerfile. The Dockerfile should follow this general outline; read the comments for additional details: ```dockerfile wrap expandable theme={"dark"} #It is important that python version is 3.x-slim for seldon-core FROM python:3.10-slim # Whatever directory(folder)the python file for your python class, Dockerfile, and # requirements.txt is in should be copied then denoted as the work directory. COPY . /app WORKDIR /app # The requirements file for the Docker container COPY requirements.txt requirements.txt RUN pip install --upgrade pip && pip install --no-cache-dir -r requirements.txt # Copy source code COPY . . # GRPC - Allows Fusion to do a Remote Procedure Call EXPOSE 5000 # Define environment variable for seldon-core # !!!MODEL_NAME must be the EXACT same as the python file & python class name!!! ENV MODEL_NAME=mini ENV SERVICE_TYPE=MODEL ENV PERSISTENCE=0 # Changing active directory folder (same one as above on lines 5 & 6) to default user, required for Fusion RUN chown -R 8888 /app # Command to wrap python class with seldon-core to allow it to be usable in Fusion CMD ["sh", "-c", "seldon-core-microservice $MODEL_NAME --service-type $SERVICE_TYPE --persistence $PERSISTENCE"] # You can use the following if You need shell features like environment variable expansion or # You need to use shell constructs like pipes, redirects, etc. # See https://docs.docker.com/reference/dockerfile/#cmd for more details. # CMD exec seldon-core-microservice $MODEL_NAME --service-type $SERVICE_TYPE --persistence $PERSISTENCE ``` ## Create a requirements file The `requirements.txt` file is a list of installs for the `Dockerfile` to run to ensure the Docker container has the right resources to run the model.\ For the Paraphrase Multilingual MiniLM L12 v2 model, the requirements are as follows: ```text theme={"dark"} seldon-core torch transformers numpy ``` In general, if an item was used in an `import` statement in your Python file, it should be included in the requirements file. An easy way to populate the requirements is by using in the following command in the terminal, inside the directory that contains your code: ```bash theme={"dark"} pip freeze > requirements.txt ``` If you use `pip freeze`, you must manually add `seldon-core` to the requirements file because it is not invoked in the Python file but is required for containerization. ## Build and push the Docker image After creating the `.py`, `Dockerfile`, and `requirements.txt` files, you need to run a few Docker commands. Run the commands below in order: ```bash theme={"dark"} DOCKER_DEFAULT_PLATFORM=linux/amd64 docker build . -t [DOCKERHUB-USERNAME]/[REPOSITORY]:[VERSION-TAG] ``` ```bash theme={"dark"} docker push [DOCKERHUB USERNAME]/[REPOSITORY]:[VERSION-TAG] ``` Using the example model, the terminal commands would be as follows: ```bash theme={"dark"} DOCKER_DEFAULT_PLATFORM=linux/amd64 docker build . -t jstrmec/example_sbert_model:0.14; docker push jstrmec/example_sbert_model:0.14 ``` This repository is public and you can visit it here: [example\_sbert\_model](https://hub.docker.com/r/jstrmec/example_sbert_model/) ## Deploy the model in Fusion Now you can go to Fusion to deploy your model. 1. In Fusion, navigate to **Collections** > **Jobs**. 2. Add a job by clicking the **Add+** Button and selecting **Create Seldon Core Model Deployment**. 3. Fill in each of the text fields: Create a Seldon Core model deployment job | Parameter | Description | | ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Job ID | A string used by the Fusion API to reference the job after its creation. | | Model name | A name for the deployed model. This is used to generate the deployment name in Seldon Core. It is also the name that you reference as a `model-id` when making predictions with the ML Service. | | Model replicas | The number of load-balanced replicas of the model to deploy; specify multiple replicas for a higher-volume intake. | | Docker Repository | The public or private repository where the Docker image is located. If you’re using Docker Hub, fill in the Docker Hub username here. | | Image name | The name of the image with an optional tag. If no tag is given, `latest` is used. | | Kubernetes secret | If you’re using a private repository, supply the name of the Kubernetes secret used for access. | | Output columns | A list of column names that the model’s `predict` method returns. | 4. Click **Save**, then **Run** and **Start**. Start a Seldon Core model deployment job When the job finishes successfully, you can proceed to the next section. Now that the model is in Fusion, it can be utilized in either index or query pipelines, depending on the model’s purpose. In this case the model is a word vectorizer or semantic vector search implementation, so both pipelines must invoke the model. ## Apply an API key to the deployment These steps are only needed if your model utilizes any kind of secret, such as an API key. If not, skip this section and proceed to the next. 1. Create and modify a `_sdep.yaml` file.\ In the first line, `kubectl get sdep` gets the details for the currently running Seldon Deployment job and saves those details to a YAML file. `kubectl apply -f open_sdep.yaml` adds the key to the Seldon Deployment job the next time it launches. ```yaml theme={"dark"} kubectl get sdep -o yaml > _sdep.yaml # Modify _sdep.yaml to add - env: - name: API_KEY value: "your-api-key-here" kubectl apply -f _sdep.yaml ``` 2. Delete `sdep` before redeploying the model. The currently running Seldon Deployment job does not have the key applied to it. Delete it before redeploying and the new job will have the key. ```bash theme={"dark"} kubectl delete sdep ``` 3. Lastly, you can [encode into Milvus](/docs/5/fusion/intro/fusion-stack/milvus). ## Create a Milvus collection 1. In Fusion, navigate to **Collections** > **Jobs**. 2. Click the **Add+** Button and select **Create Collections in Milvus**.\ This job creates a collection in Milvus for storing the vectors sent to it. The job is needed because a collection does not automatically spawn at indexing or query time if it does not already exist. 3. Name the job and the collection. 4. Click **Add** on the right side of the job panel.\ The key to creating the collection is the **Dimension** text field; this must exactly match the shape value your output prediction has.\ In our example the shape is (1,384), so 384 will be in the collections **Dimension** field: Create a Milvus collection The **Metric** field should typically be left at the default of `Inner Product`, but this also depends on use case and model type. 5. Click **Save**, then **Run** and **Start**. ## Configure the Fusion pipelines Your real-world pipeline configuration depends on your use case and model, but for our example we will configure the index pipeline and then the query pipeline. **Configure the index pipeline** 1. Create a new index pipeline or load an existing one for editing. 2. Click **Add a Stage** and then **Encode to Milvus**. 3. In the new stage, fill in these fields: * The name of your model * The output name you have for your model job * The field you’d like to encode * The collection name 4. Save the stage in the pipeline and index your data with it. **Configure the query pipeline** 1. Create a new query pipeline or load an existing one for editing. 2. Click **Add a Stage** and then **Milvus Query**. 3. Fill in the configuration fields, then save the stage. 4. Add a **Milvus Ensemble Query** stage.\ This stage is necessary to have the Milvus collection scores taken into account in ranking and to weight multiple collections. The **Milvus Results Context Key** from the Milvus Query Stage is used in this stage to preform math on the Milvus result scores. One (1) is a typical multiplier for the Milvus results but any number can be used. 5. Save the stage and then run a query by typing a search term. 6. To verify the Milvus results are correct, use the **Compare+** button to see another pipeline without the model implementation and compare the number of results. You have now successfully uploaded a Seldon Core model to Fusion and deployed it. This stage requires that you use JavaScript to construct a model input object from the Request and/or Context. This JavaScript is defined in the "Model input transformation script" property. This script must construct a HashMap containing fields and values to be sent to the model. The field names and values will depend on the input schema of the model. Value types supported are: * `String` * `Double` * `String[]` * `double[]` * `List` * `List` The JavaScript interpreter that executes the script will have the following variables available in scope: * [`request`](https://javadoc.lucidworks.com/fusion-pipeline-javadocs/5.9/?com/lucidworks/apollo/pipeline/query/Request.html) * [`response`](https://javadoc.lucidworks.com/fusion-pipeline-javadocs/5.9/?com/lucidworks/apollo/pipeline/query/Response.html) * [`context`](https://javadoc.lucidworks.com/fusion-pipeline-javadocs/4.2/com/lucidworks/apollo/pipeline/Context.html) The last line of the script must be a reference to the HashMap object you created. **Example 1: Single string parameter from request to modelInput HashMap** ```js wrap theme={"dark"} var modelInput = new java.util.HashMap() modelInput.put("input_1", request.getFirstParam("q")) modelInput ``` **Example 2: List of strings from request to modelInput HashMap** ```js wrap theme={"dark"} var modelInput = new java.util.HashMap() modelInput.put("input_1", request.getParam("q")) // request.getParam returns a Collection modelInput ``` **Example 3: List of numeric values from request to modelInput HashMap** ```js wrap theme={"dark"} var modelInput = new java.util.HashMap() var list = new java.util.ArrayList() list.add(Double.parseDouble(request.getFirstParam("numeric_1"))) list.add(Double.parseDouble(request.getFirstParam("numeric_2"))) modelInput.put("input_1", list) modelInput ``` Similarly, you will need to use JavaScript to store the predictions into the Request and/or Context from the model output object. The model output object is a HashMap containing fields and values produced by the model. The JavaScript interpreter that executes the script will have the following variables available in scope: * modelOutput (a HashMap containing fields/values returned from ML model service) * [`request`](https://javadoc.lucidworks.com/fusion-pipeline-javadocs/5.9/?com/lucidworks/apollo/pipeline/query/Request.html) * [`response`](https://javadoc.lucidworks.com/fusion-pipeline-javadocs/5.9/?com/lucidworks/apollo/pipeline/query/Response.html) * [`context`](https://javadoc.lucidworks.com/fusion-pipeline-javadocs/4.2/com/lucidworks/apollo/pipeline/Context.html) **Example: Place predictedLabel (string) on request** ```js wrap theme={"dark"} request.putSingleParam("sentiment", modelOutput.get("predictedLabel")) ``` The course for **Intro to Machine Learning in Fusion** focuses on using machine learning to infer the goals of customers and users in order to deliver a more sophisticated search experience. ## Configuration