> ## Documentation Index
> Fetch the complete documentation index at: https://doc.lucidworks.com/llms.txt
> Use this file to discover all available pages before exploring further.

# Exclude Documents

> Index pipeline stage configuration specifications

export const schema = {
  "type": "object",
  "title": "Exclude Documents",
  "description": "This stage drops a document if all of the specified rules match; otherwise, passes the document through unchanged",
  "required": ["matchRules"],
  "properties": {
    "skip": {
      "type": "boolean",
      "title": "Skip This Stage",
      "description": "Set to true to skip this stage.",
      "default": false,
      "hints": ["advanced"]
    },
    "label": {
      "type": "string",
      "title": "Label",
      "description": "A unique label for this stage.",
      "hints": ["advanced"],
      "maxLength": 255
    },
    "condition": {
      "type": "string",
      "title": "Condition",
      "description": "Define a conditional script that must result in true or false. This can be used to determine if the stage should process or not.",
      "hints": ["code", "code/javascript", "advanced"]
    },
    "matchRules": {
      "type": "array",
      "title": "Fields and Patterns",
      "minItems": 1,
      "items": {
        "type": "object",
        "required": ["field", "pattern"],
        "properties": {
          "field": {
            "type": "string",
            "title": "Field",
            "description": "The name of the field to match"
          },
          "pattern": {
            "type": "string",
            "title": "Regex Pattern",
            "description": "Pattern to match the field value against. The value may be a regex pattern.",
            "format": "regex"
          }
        }
      }
    }
  },
  "category": "Document Filtering and Enrichment",
  "categoryPriority": 8,
  "unsafe": false
};

export const SchemaParamFields = ({schema}) => {
  const sanitize = str => {
    if (typeof str !== "string") return str;
    return str.replace(/^"(.*)"$/s, "$1").replace(/\\/g, "").replace(/"/g, "'");
  };
  const formatDescription = str => {
    const s = sanitize(str);
    return (/[.!?]\)*$/).test(s) ? s : `${s}.`;
  };
  const {description, properties = {}, required: requiredProps = []} = schema;
  const visibleProps = useMemo(() => Object.entries(properties).filter(([, prop]) => !prop.hints?.includes("hidden")), [properties]);
  return <div>
      {description && <p>{formatDescription(description)}</p>}

      {visibleProps.map(([name, prop]) => {
    const isRequired = requiredProps.includes(name);
    const hasDefault = prop.default !== undefined;
    const rawDefault = prop.default;
    const isComplexDefault = hasDefault && (typeof rawDefault === "object" || typeof rawDefault === "string" && (rawDefault.length > 20 || rawDefault.includes('"')));
    const fieldProps = {
      key: name,
      body: prop.title || name,
      type: prop.type,
      ...prop.title && ({
        post: [<><span className="text-stone-400 dark:text-stone-500">API property: </span>{name}</>]
      }),
      ...isRequired && ({
        required: true
      }),
      ...!isComplexDefault && hasDefault ? {
        default: sanitize(String(rawDefault))
      } : {}
    };
    const isObject = prop.type === "object" && prop.properties;
    const isArrayOfObjects = prop.type === "array" && prop.items?.type === "object" && prop.items.properties;
    return <ParamField {...fieldProps}>
            {prop.description && <p>{formatDescription(prop.description)}</p>}

            {isComplexDefault && <div className="flex">
                <p>
                  <strong>Default:</strong>
                </p>
                <pre className="!my-0">
                  <code>
                    {JSON.stringify(rawDefault, null, 2)}
                  </code>
                </pre>
              </div>}

            {isArrayOfObjects && <div className="flex">
              <p>
                <strong>Object attributes:</strong>
              </p>
              <pre className="!my-0">
                <code>
                  {'{\n'}
                  {Object.entries(prop.items.properties).map(([iname, iprop]) => <>
                      {`  ${iname}`}
                      {prop.items?.required?.includes(iname) && <span style={{
      color: 'red'
    }}> required</span>}
                      {`: {\n    display name: ${sanitize(iprop.title || '')}\n    type: ${iprop.type}\n  }\n`}
                    </>)}
                  {'}'}
                </code>
              </pre>
              </div>}

            {isObject && <Expandable title="properties">
                <SchemaParamFields schema={{
      properties: prop.properties,
      required: prop.required
    }} />
              </Expandable>}
          </ParamField>;
  })}
    </div>;
};

export const LwTemplate = ({title = "Key questions to get you started", icon = "sparkles", cta = "Powered by Agent Studio", linkHref = "https://lucidworks.com/demo/?utm_source=docs&utm_medium=referral&utm_campaign=docs_cta_ai"}) => {
  const [isLoaded, setIsLoaded] = useState(false);
  useEffect(() => {
    const timer = setTimeout(() => {
      setIsLoaded(true);
    }, 500);
    return () => clearTimeout(timer);
  }, []);
  return <div className="lw-template-container">
      <Card title={title} icon={icon}>
        {isLoaded && <span dangerouslySetInnerHTML={{
    __html: `<lw-template id="a029c1a9-28be-427e-b0e1-5d918920246a"></lw-template
            >`
  }} />}
        <Link href={linkHref} className="agent-studio-link text-left text-gray-600 gap-2 dark:text-gray-400 text-sm font-medium flex flex-row items-center hover:text-primary dark:hover:text-primary-light group-hover:text-primary group-hover:dark:text-primary-light">Powered by Lucidworks Agent Studio</Link>
      </Card>
    </div>;
};

[localhost link]: http://localhost:3000/docs/lucidworks-search/09-developer-documentation/config-specs/index-pipeline-stages/exclude-documents

[mintlify link]: https://doc.lucidworks.com/docs/lucidworks-search/09-developer-documentation/config-specs/index-pipeline-stages/exclude-documents

[old doc.lw link]: https://doc.lucidworks.com/managed-fusion/5.9/h9drz4

The Exclude Documents stage drops all documents that match all of the specified rules (Boolean AND).
If some field has multiple values then at least one value must match against the specified pattern.
No further processing is done on any matching documents, thus they will not be indexed into a Lucidworks Search collection.
All non-matching documents are passed to the next stage in the pipeline. Rules are defined using regular expression field matching.

<LwTemplate />

## Examples

**Give the "simple-exclude" pipeline a stage that excludes certain document types**

```bash wrap  theme={"dark"}
curl -u USERNAME:PASSWORD -X POST -H "Content-type: application/json" 'http://EXAMPLE_COMPANY.lucidworks.cloud/api/index-pipelines' -d '
{
  "id" : "simple-exclude",
  "stages" : [ {
    "type" : "exclude-doc",
    "matchRules" : [ {
        "field" : "document_type",
        "pattern" : "(xls|xlsx|xlst|doc|docx)"
    }]
  }]
}'
```

<Note>
  Replace `EXAMPLE_COMPANY` with the name provided by your Lucidworks representative.
</Note>

**Send a text document through the "simple-exclude" pipeline**

```bash wrap  theme={"dark"}
curl -u USERNAME:PASSWORD 'http://EXAMPLE_COMPANY.lucidworks.cloud/api/index-pipelines/simple-exclude/collections/logs/index?simulate=true&echo=true' -H 'Content-type: application/json' -d '
{
  "document_type": "txt"
}'
```

The response is document metadata, indicating the document passed the stage:

```json wrap  theme={"dark"}
[ {
  "id" : "93da43ff-4218-4f24-a690-23b530926104",
  "fields" : [ {
    "name" : "document_type",
    "value" : "txt",
    "metadata" : { },
    "annotations" : [ ]
  } ]
} ]
```

**Send an XLS document through the "simple-exclude" pipeline**

```bash wrap  theme={"dark"}
curl -u USERNAME:PASSWORD 'http://EXAMPLE_COMPANY.lucidworks.cloud/api/index-pipelines/simple-exclude/collections/logs/index?simulate=true&echo=true' -H 'Content-type: application/json' -d '
{
  "document_type": "xls"
}'
```

The empty response indicates the document was dropped:

```json wrap  theme={"dark"}
[ ]
```

## Configuration

<Tip>
  When entering configuration values in the UI, use *unescaped* characters, such as `\t` for the tab character. When entering configuration values in the API, use *escaped* characters, such as `\\t` for the tab character.
</Tip>

<SchemaParamFields schema={schema} />