> ## Documentation Index
> Fetch the complete documentation index at: https://doc.lucidworks.com/llms.txt
> Use this file to discover all available pages before exploring further.

# Ranking Metrics

> Job configuration specifications

export const schema = {
  "type": "object",
  "title": "Ranking Metrics",
  "description": "use this job to calculate relevance metrics (nDCG etc..) by replaying ground truth queries (see ground truth job) against catalog data using variants from an experiment.",
  "required": ["id", "groundTruthConfig", "rankingExperimentConfig", "outputCollection", "type"],
  "properties": {
    "id": {
      "type": "string",
      "title": "Spark Job ID",
      "description": "The ID for this Spark job. Used in the API to reference this job. Allowed characters: a-z, A-Z, dash (-) and underscore (_). Maximum length: 63 characters.",
      "maxLength": 63,
      "pattern": "[a-zA-Z][_\\-a-zA-Z0-9]*[a-zA-Z0-9]?"
    },
    "sparkConfig": {
      "type": "array",
      "title": "Spark Settings",
      "description": "Spark configuration settings.",
      "hints": ["advanced"],
      "items": {
        "type": "object",
        "required": ["key"],
        "properties": {
          "key": {
            "type": "string",
            "title": "Parameter Name"
          },
          "value": {
            "type": "string",
            "title": "Parameter Value"
          }
        }
      }
    },
    "groundTruthConfig": {
      "type": "object",
      "title": "Configure ground truth dataset",
      "description": "Configure properties for Ground truth dataset",
      "required": ["inputCollection"],
      "properties": {
        "inputCollection": {
          "type": "string",
          "title": "Ground Truth Input Collection",
          "description": "Input collection representing ground truth dataset",
          "minLength": 1
        },
        "filterQueries": {
          "type": "array",
          "title": "Filter queries",
          "description": "Solr filter queries to apply against Ground truth collection",
          "default": ["type:ground_truth"],
          "hints": ["advanced"],
          "items": {
            "type": "string",
            "default": "[\"type:ground_truth\"]"
          }
        },
        "queryField": {
          "type": "string",
          "title": "Query field",
          "description": "Query field in the collection",
          "default": "query",
          "hints": ["advanced"]
        },
        "docIdField": {
          "type": "string",
          "title": "Doc ID field",
          "description": "Field containing ranked doc id's",
          "default": "docId",
          "hints": ["advanced"]
        },
        "weightField": {
          "type": "string",
          "title": "Weight Field",
          "description": "Field representing the weight of document to the query",
          "default": "weight",
          "hints": ["advanced"]
        }
      }
    },
    "rankingExperimentConfig": {
      "type": "object",
      "title": "Configure experiment",
      "description": "Configure properties for the experiment",
      "properties": {
        "inputCollection": {
          "type": "string",
          "title": "Input Collection",
          "description": "Collection to run the experiment on",
          "hints": ["advanced"],
          "minLength": 1
        },
        "queryPipelines": {
          "type": "array",
          "title": "Query pipelines",
          "description": "Pipeline variants for experiment",
          "hints": ["advanced"],
          "items": {
            "type": "string"
          }
        },
        "docIdField": {
          "type": "string",
          "title": "Doc Id Field",
          "description": "Doc id field to retrieve values (Must return values that match the ground truth data)",
          "default": "id",
          "hints": ["advanced"]
        },
        "experimentId": {
          "type": "string",
          "title": "Experiment ID",
          "description": "Calculate ranking metrics using variants from experiment",
          "minLength": 1
        },
        "experimentObjectiveName": {
          "type": "string",
          "title": "Experiment metric name",
          "description": "Experiment objective name",
          "minLength": 1
        },
        "defaultProfile": {
          "type": "string",
          "title": "Default Query Profile",
          "description": "Default query profile to use if not specified in experiment variants"
        }
      }
    },
    "outputCollection": {
      "type": "string",
      "title": "Output collection",
      "description": "Output collection to save the ranking metrics to",
      "minLength": 1
    },
    "rankingPositionK": {
      "type": "integer",
      "title": "Ranking Position @K",
      "description": "Ranking position at K for metrics calculation",
      "default": 10,
      "hints": ["advanced"]
    },
    "metricsPerQuery": {
      "type": "boolean",
      "title": "Calculate metrics per query",
      "description": "Calculate ranking metrics per each query in ground truth set and save them to Solr collection",
      "default": true,
      "hints": ["advanced"]
    },
    "type": {
      "type": "string",
      "title": "Spark Job Type",
      "enum": ["ranking_metrics"],
      "default": "ranking_metrics",
      "hints": ["readonly"]
    }
  },
  "additionalProperties": true,
  "category": "Other",
  "categoryPriority": 1,
  "propertyGroups": [{
    "label": "Ground Truth Parameters",
    "properties": ["groundTruthConfig"]
  }, {
    "label": "Ranking Experiment Parameters",
    "properties": ["rankingExperimentConfig"]
  }]
};

export const SchemaParamFields = ({schema}) => {
  const sanitize = str => {
    if (typeof str !== "string") return str;
    return str.replace(/^"(.*)"$/s, "$1").replace(/\\/g, "").replace(/"/g, "'");
  };
  const formatDescription = str => {
    const s = sanitize(str);
    return (/[.!?]\)*$/).test(s) ? s : `${s}.`;
  };
  const {description, properties = {}, required: requiredProps = []} = schema;
  const visibleProps = useMemo(() => Object.entries(properties).filter(([, prop]) => !prop.hints?.includes("hidden")), [properties]);
  return <div>
      {description && <p>{formatDescription(description)}</p>}

      {visibleProps.map(([name, prop]) => {
    const isRequired = requiredProps.includes(name);
    const hasDefault = prop.default !== undefined;
    const rawDefault = prop.default;
    const isComplexDefault = hasDefault && (typeof rawDefault === "object" || typeof rawDefault === "string" && (rawDefault.length > 20 || rawDefault.includes('"')));
    const fieldProps = {
      key: name,
      body: prop.title || name,
      type: prop.type,
      ...prop.title && ({
        post: [<><span className="text-stone-400 dark:text-stone-500">API property: </span>{name}</>]
      }),
      ...isRequired && ({
        required: true
      }),
      ...!isComplexDefault && hasDefault ? {
        default: sanitize(String(rawDefault))
      } : {}
    };
    const isObject = prop.type === "object" && prop.properties;
    const isArrayOfObjects = prop.type === "array" && prop.items?.type === "object" && prop.items.properties;
    return <ParamField {...fieldProps}>
            {prop.description && <p>{formatDescription(prop.description)}</p>}

            {isComplexDefault && <div className="flex">
                <p>
                  <strong>Default:</strong>
                </p>
                <pre className="!my-0">
                  <code>
                    {JSON.stringify(rawDefault, null, 2)}
                  </code>
                </pre>
              </div>}

            {isArrayOfObjects && <div className="flex">
              <p>
                <strong>Object attributes:</strong>
              </p>
              <pre className="!my-0">
                <code>
                  {'{\n'}
                  {Object.entries(prop.items.properties).map(([iname, iprop]) => <>
                      {`  ${iname}`}
                      {prop.items?.required?.includes(iname) && <span style={{
      color: 'red'
    }}> required</span>}
                      {`: {\n    display name: ${sanitize(iprop.title || '')}\n    type: ${iprop.type}\n  }\n`}
                    </>)}
                  {'}'}
                </code>
              </pre>
              </div>}

            {isObject && <Expandable title="properties">
                <SchemaParamFields schema={{
      properties: prop.properties,
      required: prop.required
    }} />
              </Expandable>}
          </ParamField>;
  })}
    </div>;
};

export const LwTemplate = ({title = "Key questions to get you started", icon = "sparkles", cta = "Powered by Agent Studio", linkHref = "https://lucidworks.com/demo/?utm_source=docs&utm_medium=referral&utm_campaign=docs_cta_ai"}) => {
  const [isLoaded, setIsLoaded] = useState(false);
  useEffect(() => {
    const timer = setTimeout(() => {
      setIsLoaded(true);
    }, 500);
    return () => clearTimeout(timer);
  }, []);
  return <div className="lw-template-container">
      <Card title={title} icon={icon}>
        {isLoaded && <span dangerouslySetInnerHTML={{
    __html: `<lw-template id="a029c1a9-28be-427e-b0e1-5d918920246a"></lw-template
            >`
  }} />}
        <Link href={linkHref} className="agent-studio-link text-left text-gray-600 gap-2 dark:text-gray-400 text-sm font-medium flex flex-row items-center hover:text-primary dark:hover:text-primary-light group-hover:text-primary group-hover:dark:text-primary-light">Powered by Lucidworks Agent Studio</Link>
      </Card>
    </div>;
};

[localhost link]: http://localhost:3000/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/ranking-metrics

[mintlify link]: https://doc.lucidworks.com/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/ranking-metrics

[old doc.lw link]: https://doc.lucidworks.com/managed-fusion/5.9/ufm17z

Use this job to calculate relevance metrics by replaying [ground truth queries](/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/ground-truth) against catalog data using variants from an [experiment](/docs/lucidworks-search/06-metrics-and-analytics/experiments). Metrics include Normalized Discounted Cumulative Gain (nDCG) and others.

To create a Ranking Metrics job, sign in to Lucidworks Search and click **Collections > Jobs**. Then click **Add+** and in the Experiment Evaluation Jobs section, select **Ranking Metrics**. You can enter basic and advanced parameters to configure the job. If the field has a default value, it is populated when you click to add the job.

<LwTemplate />

## Basic parameters

<Note>
  To enter advanced parameters in the UI, click **Advanced**. Those parameters are described in [the advanced parameters section](#advanced-parameters).
</Note>

* **Spark job ID.** The unique ID for the Spark job that references this job in the API. This is the `id` field in the configuration file. Required field.
* **Output collection.** The Solr collection where the job output is stored. The job will write the output to this collection. This is the `outputCollection` field in the configuration file. Required field.
* **Ground Truth Parameters.** This section includes this parameter:

  * **Ground truth input collection.** The collection that stores the ground truth dataset this job accesses. This is the `inputCollection` field in the configuration file. Required field.
* **Ranking Experiment Parameters.** This section includes the following parameters:

  * **Ranking experiment input collection.** The collection that stores the experiment data this job accesses. This is the `rankingExperimentConfig` `inputCollection` field in the configuration file. Optional field.
  * **Experiment ID.** The identifier for the experiment that stores the variants this job uses to calculate ranking metrics. This is the `rankingExperimentConfig` `experimentId` field in the configuration file. Optional field.
  * **Experiment metric name.** The name of the purpose (objective) of the experiment this job accesses to calculate ranking metrics. This is the `rankingExperimentConfig` `experimentObjectiveName` field in the configuration file. Optional field.
  * **Default query profile.** The name of the query profile this job defaults to if the value is not specified in the experiment variants. This is the `rankingExperimentConfig` `defaultProfile` field in the configuration file. Optional field.

## Advanced parameters

If you click the **Advanced** toggle, the following optional fields are displayed in the UI.

* **Spark Settings.** This section lets you enter `parameter name:parameter value` options to use in this job. This is the `sparkConfig` field in the configuration file.
* **Ranking position @K.** The number of returned or recommended items that are ranked (based on the relevancy rating) that are used for metrics calculation. This is the `rankingPositionK` field in the configuration file.
* **Calculate metrics per query.** If this checkbox is selected (set to `true`), the job calculates the ranking metrics per query in the ground truth dataset, and saves the metrics data to the **Output collection** designated for this job. This is the `metricsPerQuery` field in the configuration file.
* **Ground Truth Parameters.** The advanced option adds these parameters:

  * **Filter queries.** The Solr filter queries this job applies against the ground truth collection to calculate ranking metrics. This is the `groundTruthConfig` `filterQueries` field in the configuration file.
  * **Query field.** The query field in the ground truth collection. This is the `groundTruthConfig` `queryField` field in the configuration file.
  * **Doc ID field.** This field contains the ranked document IDs in the collection. This is the `groundTruthConfig` `docIdField` field in the configuration file.
  * **Weight field.** This field contains the weight of the document as it relates to the query. This is the `groundTruthConfig` `weightField` field in the configuration file.
* **Ranking Experiment Parameters.** The advanced option adds these parameters:

  * **Query pipelines.** These are the query pipelines for the experiment that stores the variants this job uses to calculate ranking metrics. This is the `rankingExperimentConfig` `queryPipelines` field in the configuration file.
  * **Doc ID field.** This field contains the values (that match the ground truth data) this job uses to calculate ranking metrics. This is the `rankingExperimentConfig` `docIdField` field in the configuration file.

## Configuration properties

<SchemaParamFields schema={schema} />