> ## Documentation Index > Fetch the complete documentation index at: https://doc.lucidworks.com/llms.txt > Use this file to discover all available pages before exploring further. # AWS S3 V1 > The AWS S3 V1 Connector can access AWS S3 buckets in native format. export const schema = { "type": "object", "title": "S3", "description": "Connector to index content in AWS S3 buckets.", "required": ["id", "connector", "type", "pipeline", "properties"], "properties": { "category": { "type": "string", "title": "Category", "default": "Filesystem", "hints": ["hidden", "readonly"] }, "id": { "type": "string", "title": "Datasource ID", "description": "Unique name for this datasource.", "minLength": 1, "pattern": "^[a-zA-Z0-9_-]+$" }, "connector": { "type": "string", "title": "Connector Type", "description": "Connector Type.", "hints": ["hidden"], "minLength": 1 }, "type": { "type": "string", "title": "Datasource Type", "description": "Datasource type supported by the selected connector type.", "hints": ["hidden"], "minLength": 1 }, "pipeline": { "type": "string", "title": "Pipeline ID", "description": "Name of an existing index pipeline for processing documents.", "minLength": 1 }, "description": { "type": "string", "title": "Description", "description": "Optional description for this datasource." }, "type_description": { "type": "string", "title": "Type Description", "default": "Connector to index content in AWS S3 buckets.", "hints": ["hidden", "readonly"] }, "parserId": { "type": "string", "title": "Parser", "description": "Parser used when parsing raw content. For some connectors, a configuration to 'retry' parsing if an error occurs is available as an advanced setting" }, "properties": { "type": "object", "title": "Properties", "description": "Datasource configuration properties", "required": ["url"], "properties": { "username": { "type": "string", "title": "AWS Key", "description": "An AWS Access Key ID that can access the content." }, "password": { "type": "string", "title": "AWS Secret Key", "description": "The AWS Secret Key associated with the Access Key.", "hints": ["secret"] }, "collection": { "type": "string", "title": "Collection", "description": "Collection documents will be indexed to.", "hints": ["hidden"], "pattern": "^[a-zA-Z0-9_-]+$" }, "db": { "type": "object", "title": "Connector DB", "description": "Type and properties for a ConnectorDB implementation to use with this datasource.", "required": ["type"], "properties": { "type": { "type": "string", "title": "Implementation Class Name", "description": "Fully qualified class name of ConnectorDb implementation.", "default": "com.lucidworks.connectors.db.impl.MapDbConnectorDb", "minLength": 1 }, "inlinks": { "type": "boolean", "title": "Process Inlinks?", "description": "Keep track of incoming links. This negatively impacts performance and size of DB.", "default": false }, "aliases": { "type": "boolean", "title": "Process Aliases?", "description": "Keep track of original URI-s that resolved to the current URI. This negatively impacts performance and size of DB.", "default": false }, "inv_aliases": { "type": "boolean", "title": "Process Inverted Aliases?", "description": "Keep track of target URI-s that the current URI resolves to. This negatively impacts performance and size of DB.", "default": false } }, "hints": ["hidden"] }, "url": { "type": "string", "title": "S3 URL", "description": "A fully-qualified S3 URL, including bucket and sub-bucket paths, as required, e.g., 's3://{bucketName}/{path}'.", "minLength": 1, "pattern": ".*:.*" }, "max_docs": { "type": "integer", "title": "Max items", "description": "Maximum number of documents to fetch. The default (-1) means no limit.", "default": -1, "minimum": -1, "exclusiveMinimum": false }, "max_bytes": { "type": "integer", "title": "Maximum file size (bytes)", "description": "Maximum size (in bytes) of documents to fetch or -1 for unlimited file size.", "default": 10485760, "minimum": -1, "exclusiveMinimum": false }, "index_directories": { "type": "boolean", "title": "Index directories", "description": "Set to true to add directories to the index as documents. If set to false, directories will not be added to the index, but they will still be traversed for documents.", "default": false }, "max_threads": { "type": "integer", "title": "Fetch threads", "description": "The maximum number of threads to use for fetching data. Note: Each thread will create a new connection to the repository, which may make overall throughput faster, but this also requires more system resources, including CPU and memory.", "default": 1, "hints": ["advanced"] }, "add_failed_docs": { "type": "boolean", "title": "Add failed documents", "description": "Set to true to add documents even if they partially fail processing. Failed documents will be added with as much metadata as available, but may not include all expected fields.", "default": false, "hints": ["advanced"] }, "crawl_item_timeout": { "type": "integer", "title": "Fetch timeout", "description": "Time in milliseconds to fetch any individual document.", "default": 600000, "hints": ["advanced"], "minimum": 0, "exclusiveMinimum": true }, "maximum_connections": { "type": "integer", "title": "Maximum fetch connections", "description": "Maximum number of concurrent connections to the filesystem. A large number of documents could cause a large number of simultaneous connections to the repository and lead to errors or degraded performance. In some cases, reducing this number may help performance issues.", "default": 1000, "hints": ["advanced"] }, "initial_mapping": { "type": "object", "title": "Initial field mapping", "description": "Provides mapping of fields before documents are sent to an index pipeline.", "properties": { "skip": { "type": "boolean", "title": "Skip This Stage", "description": "Set to true to skip this stage.", "default": false, "hints": ["advanced"] }, "label": { "type": "string", "title": "Label", "description": "A unique label for this stage.", "hints": ["advanced"], "maxLength": 255 }, "condition": { "type": "string", "title": "Condition", "description": "Define a conditional script that must result in true or false. This can be used to determine if the stage should process or not.", "hints": ["code", "code/javascript", "advanced"] }, "reservedFieldsMappingAllowed": { "type": "boolean", "title": "Allow System Fields Mapping?", "default": false, "hints": ["advanced"] }, "retentionMappings": { "type": "array", "title": "Field Retention", "description": "Fields that should be kept or deleted", "default": [], "hints": ["advanced"], "items": { "type": "object", "required": ["field"], "properties": { "field": { "type": "string", "title": "Field", "description": "The name of the field to operate on.", "hints": ["advanced"] }, "operation": { "type": "string", "title": "Operation", "description": "The type of operation to perform: keep (default) or delete", "enum": ["keep", "delete"], "default": "keep", "hints": ["advanced"] } } } }, "updateMappings": { "type": "array", "title": "Field Value Updates", "description": "Values that should be added to or set on a field. When a value is added, any values previously on the field will be retained. When a value is set, any values previously on the field will be overwritten.", "default": [], "hints": ["advanced"], "items": { "type": "object", "required": ["field", "value"], "properties": { "field": { "type": "string", "title": "Field", "description": "The name of the field to operate on.", "hints": ["advanced"] }, "value": { "type": "string", "title": "Value", "description": "The value to add to or set on the field.", "hints": ["advanced"] }, "operation": { "type": "string", "title": "Operation", "description": "The type of operation to perform: add (default) or set.", "enum": ["add", "set"], "default": "add", "hints": ["advanced"] } } } }, "translationMappings": { "type": "array", "title": "Field Translations", "description": "Fields that should be moved or copied to another field. When a field is moved, the values from the source field are moved over to the target field and the source field is removed. When a field is copied, the values from the source field are copied over to the target field and the source field is retained.", "default": [{ "source": "fetch_time", "target": "fetch_time_dt", "operation": "move" }, { "source": "ds:description", "target": "description", "operation": "move" }], "hints": ["advanced"], "items": { "type": "object", "required": ["source", "target"], "properties": { "source": { "type": "string", "title": "Source Field", "description": "The name of the field to operate on.", "hints": ["advanced"] }, "target": { "type": "string", "title": "Target Field", "description": "The name of the target field.", "hints": ["advanced"] }, "operation": { "type": "string", "title": "Operation", "description": "The type of operation to perform: copy (default) or move.", "enum": ["copy", "move"], "default": "copy", "hints": ["advanced"] } } } }, "unmappedRule": { "type": "object", "title": "Unmapped Fields", "description": "Fields not mapped by the above rules. By default, any remaining fields will be kept on the document.", "properties": { "keep": { "type": "boolean", "title": "Keep", "description": "Keep all unmapped fields", "default": true, "hints": ["advanced"] }, "delete": { "type": "boolean", "title": "Delete", "description": "Delete all unmapped fields", "default": false, "hints": ["advanced"] }, "fieldToMoveValuesTo": { "type": "string", "title": "Move", "description": "Move all unmapped field values to this field", "hints": ["advanced"] }, "fieldToCopyValuesTo": { "type": "string", "title": "Copy", "description": "Copy all unmapped field values to this field", "hints": ["advanced"] }, "valueToAddToUnmappedFields": { "type": "string", "title": "Add", "description": "Add this value to all unmapped fields", "hints": ["advanced"] }, "valueToSetOnUnmappedFields": { "type": "string", "title": "Set", "description": "Set this value on all unmapped fields", "hints": ["advanced"] } }, "default": { "keep": true, "delete": false, "fieldToMoveValuesTo": "", "fieldToCopyValuesTo": "", "valueToAddToUnmappedFields": "", "valueToSetOnUnmappedFields": "" } } }, "category": "Field Transformation", "categoryPriority": 7, "hints": ["advanced"], "unsafe": false }, "crawl_depth": { "type": "integer", "title": "Max crawl depth", "description": "Number of levels in a directory or site tree to descend for documents.", "default": -1, "minimum": -1, "exclusiveMinimum": false }, "bounds": { "type": "string", "title": "Crawl bounds", "description": "Limits the crawl to a specific directory sub-tree, hostname or domain.", "enum": ["tree", "host", "domain", "none"], "default": "tree" }, "include_paths": { "type": "array", "title": "Inclusive regexes", "description": "Regular expressions for URI patterns to include. This will limit this datasource to only URIs that match the regular expression.", "items": { "type": "string" } }, "exclude_paths": { "type": "array", "title": "Exclusive regexes", "description": "Regular expressions for URI patterns to exclude. This will limit this datasource to only URIs that do not match the regular expression.", "items": { "type": "string" } }, "include_extensions": { "type": "array", "title": "Included file extensions", "description": "List the file extensions to be fetched. Note: Files with possible matching MIME types but non-matching file extensions will be skipped. Extensions should be listed without periods, using whitespace to separate items (e.g., 'pdf zip').", "items": { "type": "string" } }, "use_instance_creds": { "type": "boolean", "title": "Use AWS credentials form system", "description": "Use provider chain that use system properties rather than an AWS key. Can be used to provide AWS EC2 instance credentials (For fusion hosted in an EC2 instance, as its nodes already have an ec2 instance role assigned). Detailed information can be found AWS SDK documentation (https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html) You can specify another AWS region through \"SSE-KMS Encryption>AWS Region\" option. Other parameters can be set through helm chart or deployment modification.", "default": false, "hints": ["advanced"] }, "commit_on_finish": { "type": "boolean", "title": "Solr commit on finish", "description": "Set to true for a request to be sent to Solr after the last batch has been fetched to commit the documents to the index.", "default": true, "hints": ["advanced"] }, "verify_access": { "type": "boolean", "title": "Validate access", "description": "Set to true to require successful connection to the filesystem before saving this datasource.", "default": true, "hints": ["advanced"] }, "use_sigv4": { "type": "boolean", "title": "Use AWS Signature Version 4", "description": "Sets the name of the signature algorithm to use for signing requests to \"AWSS3V4SignerType\". Required to retrieve encrypted objects by SSE-KMS.", "default": false, "hints": ["advanced"] }, "aws_region": { "type": "string", "title": "AWS Region", "description": "Sets the region to be used by the client. This will be used to determine both the service endpoint (eg: https://sns.us-west-2.amazonaws.com) and signing region.", "enum": ["us-gov-west-1", "us-gov-east-1", "us-east-1", "us-east-2", "us-west-1", "us-west-2", "eu-west-1", "eu-west-2", "eu-west-3", "eu-central-1", "eu-north-1", "eu-south-1", "ap-east-1", "ap-south-1", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ap-northeast-2", "sa-east-1", "cn-north-1", "cn-northwest-1", "ca-central-1", "me-south-1", "af-south-1"], "default": "us-west-2", "hints": ["advanced"] }, "retryDelay": { "type": "integer", "title": "Retry Delay Factor", "description": "The initial retry time delay, in milliseconds.", "default": 1000, "minimum": 1000, "exclusiveMinimum": false }, "stopRetry": { "type": "integer", "title": "Max Retry Time", "description": "The maximum time to retry failed requests, in minutes.", "default": 5, "minimum": 1, "exclusiveMinimum": false }, "proxyHost": { "type": "string", "title": "Proxy Host", "description": "The optional proxy host the client will connect through", "hints": ["advanced"] }, "proxyPort": { "type": "integer", "title": "Proxy Port", "description": "The optional proxy port the client will connect through", "hints": ["advanced"] }, "proxyUsername": { "type": "string", "title": "Proxy Username", "description": "The optional proxy user name to use if connecting through a proxy", "hints": ["advanced"] }, "proxyPassword": { "type": "string", "title": "Proxy Password", "description": "The optional proxy password to use when connecting through a proxy", "hints": ["advanced"] }, "proxyHttps": { "type": "boolean", "title": "Use HTTPS protocol for proxy connection", "description": "Force the HTTPS protocol to use for connecting to the proxy.", "hints": ["advanced"] } }, "propertyGroups": [{ "label": "Crawl Performance", "properties": ["max_threads", "add_failed_docs", "crawl_item_timeout", "maximum_connections", "retryDelay", "stopRetry"] }, { "label": "SSE-KMS Encryption", "properties": ["use_sigv4", "aws_region"] }, { "label": "Limit Documents", "properties": ["max_docs", "max_bytes", "index_directories", "crawl_depth", "bounds", "include_paths", "exclude_paths", "include_extensions"] }, { "label": "Field Mapping", "properties": ["initial_mapping"] }] } }, "category": "Other", "categoryPriority": 1, "unsafe": false }; export const SchemaParamFields = ({schema}) => { const sanitize = str => { if (typeof str !== "string") return str; return str.replace(/^"(.*)"$/s, "$1").replace(/\\/g, "").replace(/"/g, "'"); }; const formatDescription = str => { const s = sanitize(str); return (/[.!?]\)*$/).test(s) ? s : `${s}.`; }; const {description, properties = {}, required: requiredProps = []} = schema; const visibleProps = useMemo(() => Object.entries(properties).filter(([, prop]) => !prop.hints?.includes("hidden")), [properties]); return
{description &&

{formatDescription(description)}

} {visibleProps.map(([name, prop]) => { const isRequired = requiredProps.includes(name); const hasDefault = prop.default !== undefined; const rawDefault = prop.default; const isComplexDefault = hasDefault && (typeof rawDefault === "object" || typeof rawDefault === "string" && (rawDefault.length > 20 || rawDefault.includes('"'))); const fieldProps = { key: name, body: prop.title || name, type: prop.type, ...prop.title && ({ post: [<>API property: {name}] }), ...isRequired && ({ required: true }), ...!isComplexDefault && hasDefault ? { default: sanitize(String(rawDefault)) } : {} }; const isObject = prop.type === "object" && prop.properties; const isArrayOfObjects = prop.type === "array" && prop.items?.type === "object" && prop.items.properties; return {prop.description &&

{formatDescription(prop.description)}

} {isComplexDefault &&

Default: 

                  
                    {JSON.stringify(rawDefault, null, 2)}
} {isArrayOfObjects &&

Object attributes: 

                
                  {'{\n'}
                  {Object.entries(prop.items.properties).map(([iname, iprop]) => <>
                      {`  ${iname}`}
                      {prop.items?.required?.includes(iname) &&  required}
                      {`: {\n    display name: ${sanitize(iprop.title || '')}\n    type: ${iprop.type}\n  }\n`}
                    )}
                  {'}'}
} {isObject && } ; })}
; }; export const LwTemplate = ({title = "Key questions to get you started", icon = "sparkles", cta = "Powered by Agent Studio", linkHref = "https://lucidworks.com/demo/?utm_source=docs&utm_medium=referral&utm_campaign=docs_cta_ai"}) => { const [isLoaded, setIsLoaded] = useState(false); useEffect(() => { const timer = setTimeout(() => { setIsLoaded(true); }, 500); return () => clearTimeout(timer); }, []); return
{isLoaded && ` }} />} Powered by Lucidworks Agent Studio
; }; [localhost link]: http://localhost:3000/docs/fusion-connectors/connectors/v1/s3 [mintlify link]: https://doc.lucidworks.com/docs/fusion-connectors/connectors/v1/s3 [old doc.lw link]: https://doc.lucidworks.com/fusion-connectors/52 **Compatible with Fusion version:** 4.0.0 through 5.3.5 Deprecation and removal notice This connector is deprecated as of Fusion 5.2 and is removed or expected to be removed as of Fusion 5.5. Use the AWS S3 Pro connector instead. For more information about deprecations and removals, including possible alternatives, see [Deprecations and Removals](/docs/fusion-connectors/deprecations-and-removals). The connector uses the S3 API to request data from S3. It calls the `listBucket` service, which lists all buckets owned by the user account supplied to the connector. When creating an S3 data source using the UI, Fusion automatically verifies that the user information supplied has access to the bucket defined in the URL property. If the bucket is not in the list returned by S3, data source creation may fail. At crawl time, if the bucket is not in the list returned by S3, the crawl fails. Permission errors when trying to create or crawl the data source may be caused by incorrect username or password, or they may be due to user account permissions. The user account must have List Bucket permissions for the account which owns the bucket that the crawler is trying to access. ## Configuration When entering configuration values in the UI, use *unescaped* characters, such as `\t` for the tab character. When entering configuration values in the API, use *escaped* characters, such as `\\t` for the tab character.