> ## Documentation Index
> Fetch the complete documentation index at: https://doc.lucidworks.com/llms.txt
> Use this file to discover all available pages before exploring further.

# Phrase Extraction

> Job configuration specifications

export const schema = {
  "type": "object",
  "title": "Phrase Extraction",
  "description": "Use this job when you want to identify statistically significant phrases in your content.",
  "required": ["id", "trainingCollection", "fieldToVectorize", "dataFormat", "analyzerConfig", "type"],
  "properties": {
    "id": {
      "type": "string",
      "title": "Spark Job ID",
      "description": "The ID for this Spark job. Used in the API to reference this job. Allowed characters: a-z, A-Z, dash (-) and underscore (_). Maximum length: 63 characters.",
      "maxLength": 63,
      "pattern": "[a-zA-Z][_\\-a-zA-Z0-9]*[a-zA-Z0-9]?"
    },
    "sparkConfig": {
      "type": "array",
      "title": "Spark Settings",
      "description": "Spark configuration settings.",
      "hints": ["advanced"],
      "items": {
        "type": "object",
        "required": ["key"],
        "properties": {
          "key": {
            "type": "string",
            "title": "Parameter Name"
          },
          "value": {
            "type": "string",
            "title": "Parameter Value"
          }
        }
      }
    },
    "trainingCollection": {
      "type": "string",
      "title": "Training Collection",
      "description": "Solr Collection containing labeled training data",
      "minLength": 1
    },
    "fieldToVectorize": {
      "type": "string",
      "title": "Field to Vectorize",
      "description": "Solr field containing text training data. Data from multiple fields with different weights can be combined by specifying them as field1:weight1,field2:weight2 etc.",
      "minLength": 1
    },
    "dataFormat": {
      "type": "string",
      "title": "Data format",
      "description": "Spark-compatible format that contains training data (like 'solr', 'parquet', 'orc' etc)",
      "default": "solr",
      "minLength": 1
    },
    "trainingDataFrameConfigOptions": {
      "type": "object",
      "title": "Dataframe Config Options",
      "description": "Additional spark dataframe loading configuration options",
      "properties": {},
      "additionalProperties": {
        "type": "string"
      },
      "hints": ["advanced"]
    },
    "trainingDataFilterQuery": {
      "type": "string",
      "title": "Training data filter query",
      "description": "Solr query to use when loading training data if using Solr",
      "default": "*:*",
      "hints": ["advanced"]
    },
    "sparkSQL": {
      "type": "string",
      "title": "Spark SQL filter query",
      "description": "Use this field to create a Spark SQL query for filtering your input data. The input data will be registered as spark_input",
      "default": "SELECT * from spark_input",
      "hints": ["code/sql", "advanced"]
    },
    "trainingDataSamplingFraction": {
      "type": "number",
      "title": "Training data sampling fraction",
      "description": "Fraction of the training data to use",
      "default": 1,
      "hints": ["advanced"],
      "maximum": 1,
      "exclusiveMaximum": false
    },
    "randomSeed": {
      "type": "integer",
      "title": "Random seed",
      "description": "For any deterministic pseudorandom number generation",
      "default": 8180,
      "hints": ["advanced"]
    },
    "outputCollection": {
      "type": "string",
      "title": "Output Collection",
      "description": "Solr Collection to store extracted phrases; defaults to the query_rewrite_staging collection for the associated app."
    },
    "overwriteOutput": {
      "type": "boolean",
      "title": "Overwrite Output",
      "description": "Overwrite output collection",
      "default": true,
      "hints": ["hidden", "advanced"]
    },
    "dataOutputFormat": {
      "type": "string",
      "title": "Data output format",
      "description": "Spark-compatible output format (like 'solr', 'parquet', etc)",
      "default": "solr",
      "hints": ["advanced"],
      "minLength": 1
    },
    "sourceFields": {
      "type": "string",
      "title": "Fields to Load",
      "description": "Solr fields to load (comma-delimited). Leave empty to allow the job to select the required fields to load at runtime.",
      "hints": ["advanced"]
    },
    "partitionCols": {
      "type": "string",
      "title": "Partition fields",
      "description": "If writing to non-Solr sources, this field will accept a comma-delimited list of column names for partitioning the dataframe before writing to the external output ",
      "hints": ["advanced"]
    },
    "writeOptions": {
      "type": "array",
      "title": "Write Options",
      "description": "Options used when writing output to Solr or other sources",
      "hints": ["advanced"],
      "items": {
        "type": "object",
        "required": ["key"],
        "properties": {
          "key": {
            "type": "string",
            "title": "Parameter Name"
          },
          "value": {
            "type": "string",
            "title": "Parameter Value"
          }
        }
      }
    },
    "readOptions": {
      "type": "array",
      "title": "Read Options",
      "description": "Options used when reading input from Solr or other sources.",
      "hints": ["advanced"],
      "items": {
        "type": "object",
        "required": ["key"],
        "properties": {
          "key": {
            "type": "string",
            "title": "Parameter Name"
          },
          "value": {
            "type": "string",
            "title": "Parameter Value"
          }
        }
      }
    },
    "ngramSize": {
      "type": "integer",
      "title": "Ngram Size",
      "description": "The number of words in the ngram you want to consider for the sips.",
      "default": 3,
      "maximum": 5,
      "exclusiveMaximum": false,
      "minimum": 2,
      "exclusiveMinimum": false
    },
    "minmatch": {
      "type": "integer",
      "title": "Minimum Count",
      "description": "The number of times a phrase must exist to be considered. NOTE: if input is non signal data, please reduce the number to e.g. 5.",
      "default": 100,
      "minimum": 1,
      "exclusiveMinimum": false
    },
    "analyzerConfig": {
      "type": "string",
      "title": "Lucene Text Analyzer",
      "description": "The style of text analyzer you would like to use.",
      "default": "{ \"analyzers\": [{ \"name\": \"StdTokLowerStop\",\"charFilters\": [ { \"type\": \"htmlstrip\" } ],\"tokenizer\": { \"type\": \"standard\" },\"filters\": [{ \"type\": \"lowercase\" }] }],\"fields\": [{ \"regex\": \".+\", \"analyzer\": \"StdTokLowerStop\" } ]}",
      "hints": ["lengthy", "code/json"]
    },
    "attachPhrases": {
      "type": "boolean",
      "title": "Extract Key Phrases from Input Text",
      "description": "Checking this will cause the job to associate extracted phrases from each source doc. and write them back to the output collection. If input data is signals, it is suggested to turn this option off. Also, currently it is not allowed to check this option while attempting to write to a _query_rewrite_staging collection.",
      "default": false,
      "hints": ["advanced"]
    },
    "stopwordsList": {
      "type": "array",
      "title": "List of stopwords",
      "description": "Stopwords defined in Lucene analyzer config",
      "hints": ["readonly", "hidden"],
      "items": {
        "type": "string",
        "minLength": 1,
        "reference": "blob",
        "blobType": "file:spark"
      }
    },
    "minLikelihood": {
      "type": "number",
      "title": "Minimum Likelihood Score",
      "description": "Phrases below this threshold will not be written in the output of this job.",
      "hints": ["advanced"]
    },
    "enableAutoPublish": {
      "type": "boolean",
      "title": "Enable auto-publishing",
      "description": "If true, automatically publishes rewrites for rules. Default is false to allow for initial human-aided reviewing",
      "default": false,
      "hints": ["advanced"]
    },
    "sparkPartitions": {
      "type": "integer",
      "title": "Set minimum Spark partitions for input",
      "description": "Spark will re-partition the input to have this number of partitions. Increase for greater parallelism",
      "default": 200,
      "hints": ["advanced"]
    },
    "type": {
      "type": "string",
      "title": "Spark Job Type",
      "enum": ["sip"],
      "default": "sip",
      "hints": ["readonly"]
    }
  },
  "additionalProperties": true,
  "category": "Other",
  "categoryPriority": 1,
  "propertyGroups": [{
    "label": "Input/Output Parameters",
    "properties": ["trainingCollection", "outputCollection", "dataFormat", "trainingDataFilterQuery", "readOptions", "writeOptions", "trainingDataFrameConfigOptions", "trainingDataSamplingFraction", "randomSeed"]
  }, {
    "label": "Field Parameters",
    "properties": ["fieldToVectorize", "sourceFields"]
  }, {
    "label": "Model Tuning Parameters",
    "properties": ["minmatch", "ngramSize"]
  }, {
    "label": "Featurization Parameters",
    "properties": ["analyzerConfig"]
  }]
};

export const SchemaParamFields = ({schema}) => {
  const sanitize = str => {
    if (typeof str !== "string") return str;
    return str.replace(/^"(.*)"$/s, "$1").replace(/\\/g, "").replace(/"/g, "'");
  };
  const formatDescription = str => {
    const s = sanitize(str);
    return (/[.!?]\)*$/).test(s) ? s : `${s}.`;
  };
  const {description, properties = {}, required: requiredProps = []} = schema;
  const visibleProps = useMemo(() => Object.entries(properties).filter(([, prop]) => !prop.hints?.includes("hidden")), [properties]);
  return <div>
      {description && <p>{formatDescription(description)}</p>}

      {visibleProps.map(([name, prop]) => {
    const isRequired = requiredProps.includes(name);
    const hasDefault = prop.default !== undefined;
    const rawDefault = prop.default;
    const isComplexDefault = hasDefault && (typeof rawDefault === "object" || typeof rawDefault === "string" && (rawDefault.length > 20 || rawDefault.includes('"')));
    const fieldProps = {
      key: name,
      body: prop.title || name,
      type: prop.type,
      ...prop.title && ({
        post: [<><span className="text-stone-400 dark:text-stone-500">API property: </span>{name}</>]
      }),
      ...isRequired && ({
        required: true
      }),
      ...!isComplexDefault && hasDefault ? {
        default: sanitize(String(rawDefault))
      } : {}
    };
    const isObject = prop.type === "object" && prop.properties;
    const isArrayOfObjects = prop.type === "array" && prop.items?.type === "object" && prop.items.properties;
    return <ParamField {...fieldProps}>
            {prop.description && <p>{formatDescription(prop.description)}</p>}

            {isComplexDefault && <div className="flex">
                <p>
                  <strong>Default:</strong>
                </p>
                <pre className="!my-0">
                  <code>
                    {JSON.stringify(rawDefault, null, 2)}
                  </code>
                </pre>
              </div>}

            {isArrayOfObjects && <div className="flex">
              <p>
                <strong>Object attributes:</strong>
              </p>
              <pre className="!my-0">
                <code>
                  {'{\n'}
                  {Object.entries(prop.items.properties).map(([iname, iprop]) => <>
                      {`  ${iname}`}
                      {prop.items?.required?.includes(iname) && <span style={{
      color: 'red'
    }}> required</span>}
                      {`: {\n    display name: ${sanitize(iprop.title || '')}\n    type: ${iprop.type}\n  }\n`}
                    </>)}
                  {'}'}
                </code>
              </pre>
              </div>}

            {isObject && <Expandable title="properties">
                <SchemaParamFields schema={{
      properties: prop.properties,
      required: prop.required
    }} />
              </Expandable>}
          </ParamField>;
  })}
    </div>;
};

export const LwTemplate = ({title = "Key questions to get you started", icon = "sparkles", cta = "Powered by Agent Studio", linkHref = "https://lucidworks.com/demo/?utm_source=docs&utm_medium=referral&utm_campaign=docs_cta_ai"}) => {
  const [isLoaded, setIsLoaded] = useState(false);
  useEffect(() => {
    const timer = setTimeout(() => {
      setIsLoaded(true);
    }, 500);
    return () => clearTimeout(timer);
  }, []);
  return <div className="lw-template-container">
      <Card title={title} icon={icon}>
        {isLoaded && <span dangerouslySetInnerHTML={{
    __html: `<lw-template id="a029c1a9-28be-427e-b0e1-5d918920246a"></lw-template
            >`
  }} />}
        <Link href={linkHref} className="agent-studio-link text-left text-gray-600 gap-2 dark:text-gray-400 text-sm font-medium flex flex-row items-center hover:text-primary dark:hover:text-primary-light group-hover:text-primary group-hover:dark:text-primary-light">Powered by Lucidworks Agent Studio</Link>
      </Card>
    </div>;
};

[localhost link]: http://localhost:3000/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/phrase-extraction

[mintlify link]: https://doc.lucidworks.com/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/phrase-extraction

[old doc.lw link]: https://doc.lucidworks.com/managed-fusion/5.9/xzyz7w

Identify multi-word phrases in signals.

<Note>
  This job is deprecated in Lucidworks Search 5.9.15 and will be removed in a future release.
  Lucidworks recommends migrating to [Neural Hybrid Search](/docs/lucidworks-search/11-vector-search/overview), which achieves superior relevance compared to legacy machine learning methods.
</Note>

<Card title="Resolving Underperforming Queries" class="note-image" href="https://academy.lucidworks.com/resolving-underperforming-queries-with-query-rewriting-jobs" cta="Take this course on the LucidAcademy." icon="graduation-cap" iconType="duotone">
  The course for **Resolving Underperforming Queries** focuses on tips for tuning, running, and cleaning up Fusion’s query rewrite jobs.
</Card>

|                      |                                                                                       |
| -------------------- | ------------------------------------------------------------------------------------- |
| **Default job name** | `COLLECTION_NAME_phrase_extraction`                                                   |
| **Input**            | Raw signals (the `COLLECTION_NAME_signals` collection by default)                     |
| **Output**           | Extracted phrases (the `COLLECTION_NAME_query_rewrite_staging` collection by default) |

This job writes to the `COLLECTION_NAME_query_rewrite_staging` collection. It also uses reviewed documents from that collection to improve the accuracy of the job. You can review, edit, deploy, or delete output from this job using the [Query Rewriting](/docs/lucidworks-search/07-improve-your-queries/query-rewriting/overview).

Lucidworks Search ships with the [OpenNLP Maxent model](http://opennlp.sourceforge.net/models-1.5/en-pos-maxent.bin) already loaded in the [blob store](/docs/lucidworks-search/04-move-data-in/blob-storage/overview).

This job’s output, and output from the [Token and Phrase Spell Correction job](/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/token-phrase-spell-correction), can be used as input for the [Synonym Detection job](/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/synonym-detection).

<LwTemplate />

## Minimum configuration

For most use cases, the minimum configuration for this job consists of these fields:

* `id`/**Spark Job ID**
  Give this job an arbitrary ID string.
* `trainingCollection`/**Training Collection**
  Specify the input collection.
* `fieldToVectorize`/**Field to Vectorize**
  Specify the field in the input collection where phrases can be found.
* `outputCollection`/**Output Collection**
  Specify the collection in which the output documents should be indexed.

<Tip>
  When running this job over a content document collection, be sure to set `attachPhrases`/**Extract Key Phrases from Input Text** to "true". The default is "false", which works well when running the job over a signals collection.
</Tip>

## Output documents

By default, the job only outputs the phrases found from the original document. In each row of the phrases output, these fields are most useful:

* The phrase itself is in the `phrases_s` field, which can be used for faceting.
* The `likelihood_d` field gives the likelihood that the phrase is legitimate, from 0 to infinity.\
  Low-probability phrases are automatically trimmed from the results.
* When a phrase’s likelihood value is ambiguous, the `review` field is set to "true" to indicate that the phrase should be reviewed.
* A `phrase_count` field indicates the number of instances of the phrase in the input collection.

The complete list of output fields is shown below.

## Output fields

|                    |                                                                                  |
| ------------------ | -------------------------------------------------------------------------------- |
| `aggr_id_s`        | The name of the Phrase Extraction job that generated this document.              |
| `doc_type_s`       | This is always `key_phrases` for documents generated by a Phrase Extraction job. |
| `id`               | A unique ID for this document.                                                   |
| `input_collection` | The collection used for this job’s input.                                        |
| `likelihood_d`     | The likelihood that this `phrases_s` is a phrase, from 0 to infinity.            |
| `phrase_count`     | The number of occurrences of this phrase in the input collection.                |
| `phrases_s`        | The phrase detected by the job.                                                  |
| `review`           | "True" indicates that this may not be a valid phrase and should be reviewed.     |
| `score`            | This is always "1".                                                              |
| `timestamp`        | The date and time when the document was generated.                               |
| `word_num_i`       | The number of words in this phrase.                                              |
| `_version_`        | An internal Solr field used for partial updates.                                 |

If the `attachPhrases`/**Extract Key Phrases from Input Text** parameter is set to "true", then the job also outputs the original documents from the input collection with an appended field, `phrases_extracted_tt`, that lists the extracted phrases from this document.

The way to distinguish the phrases output from the original document output is by the field `doc_type_s`, with one of these values:

* `key_phrases` denotes phrases output.
* `original_doc_with_phrases` denotes the original documents.

## Configuration properties

<SchemaParamFields schema={schema} />