> ## Documentation Index
> Fetch the complete documentation index at: https://doc.lucidworks.com/llms.txt
> Use this file to discover all available pages before exploring further.

# Text

> Parser stage configuration specifications

export const schema = {
  "type": "object",
  "title": "Text",
  "description": "Parses plain text content with optional trimming and splitting. Character encoding can be specified or automatically detected",
  "required": ["charset", "ignoreBOM", "type"],
  "properties": {
    "id": {
      "type": "string",
      "title": "Parser ID",
      "default": "d0522cb8-92d3-47cb-97c3-b30e51376273"
    },
    "label": {
      "type": "string",
      "title": "Label",
      "description": "A label for this Parser Stage",
      "maxLength": 255
    },
    "enabled": {
      "type": "boolean",
      "title": "Enable this Parser Stage",
      "default": true
    },
    "mediaTypes": {
      "type": "array",
      "title": "Media Types to match",
      "description": "Documents with a media type on this list will be matched by this parser stage. See inheritMediaTypes / use default media types for more.",
      "items": {
        "type": "string",
        "pattern": "^[^\\/]+\\/[^\\/]+$",
        "format": "rfc2646"
      }
    },
    "inheritMediaTypes": {
      "type": "boolean",
      "title": "Match default media types in this Parser Stage",
      "description": "Each parser stage has a built-in list of media types it handles by default. If this setting is true, that list will be used along with any optional additional types provided in the mediaTypes list. If this setting is false, this stage will only be selected for media types in the mediaTypes list, and the mediaTypes list becomes a mandatory property which must have at least one valid media type.",
      "default": true
    },
    "ignoredMediaTypes": {
      "type": "array",
      "title": "Media Types to ignore",
      "description": "Documents with a media type on this list will be not be processed by this parser stage.",
      "items": {
        "type": "string",
        "pattern": "^[^\\/]+\\/[^\\/]+$",
        "format": "rfc2646"
      }
    },
    "pathPatterns": {
      "type": "array",
      "title": "File names to parse",
      "description": "Specify a file name or pattern that must be matched for this parser stage to run. Forward slashes (\"/\") are used to join names of files inside archives with the archive name.",
      "items": {
        "type": "object",
        "properties": {
          "syntax": {
            "type": "string",
            "title": "Pattern type",
            "description": "glob uses bash shell-style wildcards; regex uses Java (PCRE-style) regex",
            "enum": ["glob", "regex"],
            "default": "glob"
          },
          "pattern": {
            "type": "string",
            "title": "File name or pattern",
            "description": "e.g.: \"z.txt\" or \"*.md\" or \"/a/*/b/f.txt\" for glob; \"z.txt$\" or \".*\\.txt$\" or \"^/a/[^\\/]*/b/f.txt$\" for regex"
          }
        }
      }
    },
    "errorHandling": {
      "type": "string",
      "title": "Error Handling",
      "enum": ["ignore", "log", "fail", "mark"],
      "default": "mark"
    },
    "outputFieldPrefix": {
      "type": "string",
      "title": "Prefix parsed fields with",
      "description": "Fields extracted by this parser will be prefixed with this string. The remainder of the field name will be as detected in the stream",
      "maxLength": 20,
      "pattern": "^$|^[A-Za-z_][A-Za-z0-9_\\-\\.]+$"
    },
    "charset": {
      "type": "string",
      "title": "Character Set",
      "description": "Example: \"UTF-8\"",
      "default": "detect"
    },
    "ignoreBOM": {
      "type": "boolean",
      "title": "Ignore BOM",
      "description": "Ignore Byte-Order Mark (BOM) if present and always use the configured character set. When set to false a valid BOM character set overrides the configured default character set.",
      "default": false
    },
    "splitLines": {
      "type": "boolean",
      "title": "Split lines",
      "description": "Split text into lines to create multiple records, default false",
      "default": false
    },
    "skipHeaderLines": {
      "type": "integer",
      "title": "Skip header lines",
      "description": "Skip a number of header lines, default 0",
      "default": 0
    },
    "trimWhitespace": {
      "type": "boolean",
      "title": "Trim whitespace",
      "description": "Trim off leading and trailing whitespace from lines, default false",
      "default": false
    },
    "skipEmptyLines": {
      "type": "boolean",
      "title": "Skip empty lines",
      "description": "Skip any empty lines encountered, default false",
      "default": false
    },
    "outputField": {
      "type": "string",
      "title": "Output field",
      "description": "Name of the output field where text is stored, default 'body'",
      "default": "body",
      "minLength": 1
    },
    "maxLength": {
      "type": "integer",
      "title": "Maximum length",
      "description": "Maximum number of characters to allow for the body, -1 for unlimited, default 1MB",
      "default": 1048576,
      "maximum": 2147483647,
      "exclusiveMaximum": false,
      "minimum": 0,
      "exclusiveMinimum": false
    },
    "maxLineLength": {
      "type": "integer",
      "title": "Maximum line length",
      "description": "Maximum number of characters to allow for any single line, default 1MB",
      "default": 1048576,
      "maximum": 2147483647,
      "exclusiveMaximum": false,
      "minimum": 0,
      "exclusiveMinimum": false
    },
    "commentField": {
      "type": "string",
      "title": "Comment field",
      "description": "Name of the output field where comment is stored, default 'comment'",
      "default": "comment",
      "minLength": 1
    },
    "comment": {
      "type": "string",
      "title": "Comment character",
      "description": "Characters at start of line to indicate a comment, default # (hash)",
      "default": "#",
      "minLength": 1
    },
    "commentHandling": {
      "type": "string",
      "title": "Comment Handling",
      "description": "How to handle comments: include as-is, ignore (and remove from text), add as field (and remove from text), default include",
      "enum": ["ignore", "include", "as_field"],
      "default": "include"
    },
    "type": {
      "type": "string",
      "enum": ["text"],
      "default": "text"
    }
  },
  "additionalProperties": false,
  "category": "Other",
  "categoryPriority": 1,
  "unsafe": false
};

export const SchemaParamFields = ({schema}) => {
  const sanitize = str => {
    if (typeof str !== "string") return str;
    return str.replace(/^"(.*)"$/s, "$1").replace(/\\/g, "").replace(/"/g, "'");
  };
  const formatDescription = str => {
    const s = sanitize(str);
    return (/[.!?]\)*$/).test(s) ? s : `${s}.`;
  };
  const {description, properties = {}, required: requiredProps = []} = schema;
  const visibleProps = useMemo(() => Object.entries(properties).filter(([, prop]) => !prop.hints?.includes("hidden")), [properties]);
  return <div>
      {description && <p>{formatDescription(description)}</p>}

      {visibleProps.map(([name, prop]) => {
    const isRequired = requiredProps.includes(name);
    const hasDefault = prop.default !== undefined;
    const rawDefault = prop.default;
    const isComplexDefault = hasDefault && (typeof rawDefault === "object" || typeof rawDefault === "string" && (rawDefault.length > 20 || rawDefault.includes('"')));
    const fieldProps = {
      key: name,
      body: prop.title || name,
      type: prop.type,
      ...prop.title && ({
        post: [<><span className="text-stone-400 dark:text-stone-500">API property: </span>{name}</>]
      }),
      ...isRequired && ({
        required: true
      }),
      ...!isComplexDefault && hasDefault ? {
        default: sanitize(String(rawDefault))
      } : {}
    };
    const isObject = prop.type === "object" && prop.properties;
    const isArrayOfObjects = prop.type === "array" && prop.items?.type === "object" && prop.items.properties;
    return <ParamField {...fieldProps}>
            {prop.description && <p>{formatDescription(prop.description)}</p>}

            {isComplexDefault && <div className="flex">
                <p>
                  <strong>Default:</strong>
                </p>
                <pre className="!my-0">
                  <code>
                    {JSON.stringify(rawDefault, null, 2)}
                  </code>
                </pre>
              </div>}

            {isArrayOfObjects && <div className="flex">
              <p>
                <strong>Object attributes:</strong>
              </p>
              <pre className="!my-0">
                <code>
                  {'{\n'}
                  {Object.entries(prop.items.properties).map(([iname, iprop]) => <>
                      {`  ${iname}`}
                      {prop.items?.required?.includes(iname) && <span style={{
      color: 'red'
    }}> required</span>}
                      {`: {\n    display name: ${sanitize(iprop.title || '')}\n    type: ${iprop.type}\n  }\n`}
                    </>)}
                  {'}'}
                </code>
              </pre>
              </div>}

            {isObject && <Expandable title="properties">
                <SchemaParamFields schema={{
      properties: prop.properties,
      required: prop.required
    }} />
              </Expandable>}
          </ParamField>;
  })}
    </div>;
};

export const LwTemplate = ({title = "Key questions to get you started", icon = "sparkles", cta = "Powered by Agent Studio", linkHref = "https://lucidworks.com/demo/?utm_source=docs&utm_medium=referral&utm_campaign=docs_cta_ai"}) => {
  const [isLoaded, setIsLoaded] = useState(false);
  useEffect(() => {
    const timer = setTimeout(() => {
      setIsLoaded(true);
    }, 500);
    return () => clearTimeout(timer);
  }, []);
  return <div className="lw-template-container">
      <Card title={title} icon={icon}>
        {isLoaded && <span dangerouslySetInnerHTML={{
    __html: `<lw-template id="a029c1a9-28be-427e-b0e1-5d918920246a"></lw-template
            >`
  }} />}
        <Link href={linkHref} className="agent-studio-link text-left text-gray-600 gap-2 dark:text-gray-400 text-sm font-medium flex flex-row items-center hover:text-primary dark:hover:text-primary-light group-hover:text-primary group-hover:dark:text-primary-light">Powered by Lucidworks Agent Studio</Link>
      </Card>
    </div>;
};

[localhost link]: http://localhost:3000/docs/lucidworks-search/09-developer-documentation/config-specs/parsers/text-parser

[mintlify link]: https://doc.lucidworks.com/docs/lucidworks-search/09-developer-documentation/config-specs/parsers/text-parser

[old doc.lw link]: https://doc.lucidworks.com/managed-fusion/5.9/70dpnu

The Plain Text parser can split a text file by lines or consume it into a single document.

<LwTemplate />

Options for treatment of this filetype include:

* Plain Text Parser Fields
* Number of header rows to skip
* Split on line end or not
* Comment character
* Skip empty lines
* Charset

<Tip>
  When entering configuration values in the UI, use *unescaped* characters, such as `\t` for the tab character. When entering configuration values in the API, use *escaped* characters, such as `\\t` for the tab character.
</Tip>

<SchemaParamFields schema={schema} />