> ## Documentation Index
> Fetch the complete documentation index at: https://doc.lucidworks.com/llms.txt
> Use this file to discover all available pages before exploring further.
# Javascript V2
> The Javascript connector allows users to write ad-hoc document retrieval routines to fetch content from filesystems and websites.
export const schema = {
"type": "object",
"title": "JavaScript (deprecated)",
"description": "Connector for document routines written in JavaScript to fetch content from filesystems and websites.",
"required": ["id", "connector", "type", "pipeline", "properties"],
"properties": {
"id": {
"type": "string",
"title": "Datasource ID",
"description": "Unique name for this datasource.",
"minLength": 1,
"pattern": "^[a-zA-Z0-9_-]+$"
},
"connector": {
"type": "string",
"title": "Connector Type",
"description": "Connector Type.",
"hints": ["hidden"],
"minLength": 1
},
"type": {
"type": "string",
"title": "Datasource Type",
"description": "Datasource type supported by the selected connector type.",
"hints": ["hidden"],
"minLength": 1
},
"pipeline": {
"type": "string",
"title": "Pipeline ID",
"description": "Name of an existing index pipeline for processing documents.",
"minLength": 1
},
"description": {
"type": "string",
"title": "Description",
"description": "Optional description for this datasource."
},
"category": {
"type": "string",
"title": "Category",
"default": "Script",
"hints": ["hidden", "readonly"]
},
"type_description": {
"type": "string",
"title": "Type Description",
"default": "Connector for document routines written in JavaScript to fetch content from filesystems and websites.",
"hints": ["hidden", "readonly"]
},
"properties": {
"type": "object",
"title": "Properties",
"description": "Datasource configuration properties",
"required": ["f.script"],
"properties": {
"collection": {
"type": "string",
"title": "Collection",
"description": "Collection documents will be indexed to.",
"hints": ["hidden"],
"pattern": "^[a-zA-Z0-9_-]+$"
},
"db": {
"type": "object",
"title": "Connector DB",
"description": "Type and properties for a ConnectorDB implementation to use with this datasource.",
"required": ["type"],
"properties": {
"type": {
"type": "string",
"title": "Implementation Class Name",
"description": "Fully qualified class name of ConnectorDb implementation.",
"default": "com.lucidworks.connectors.db.impl.MapDbConnectorDb",
"minLength": 1
},
"inlinks": {
"type": "boolean",
"title": "Process Inlinks?",
"description": "Keep track of incoming links. This negatively impacts performance and size of DB.",
"default": false
},
"aliases": {
"type": "boolean",
"title": "Process Aliases?",
"description": "Keep track of original URI-s that resolved to the current URI. This negatively impacts performance and size of DB.",
"default": false
},
"inv_aliases": {
"type": "boolean",
"title": "Process Inverted Aliases?",
"description": "Keep track of target URI-s that the current URI resolves to. This negatively impacts performance and size of DB.",
"default": false
}
},
"hints": ["hidden"]
},
"startLinks": {
"type": "array",
"title": "Start Links",
"description": "One or more starting URIs for this datasource.",
"default": ["__js__"],
"items": {
"type": "string",
"minLength": 1
}
},
"dedupe": {
"type": "boolean",
"title": "Dedupe documents",
"description": "If true, documents will be deduplicated. Deduplication can be done based on an analysis of the content, on the content of a specific field, or by a JavaScript function. If neither a field nor a script are defined, content analysis will be used.",
"default": false,
"hints": ["advanced"]
},
"dedupeField": {
"type": "string",
"title": "Dedupe field",
"description": "Field to be used for dedupe. Define either a field or a dedupe script, otherwise the full raw content of each document will be used.",
"hints": ["advanced"]
},
"dedupeScript": {
"type": "string",
"title": "Dedupe script",
"description": "Custom javascript to dedupe documents. The script must define a 'genSignature(content){}' function, but can use any combination of document fields. The function must return a string.",
"hints": ["advanced", "code", "code/javascript"]
},
"dedupeSaveSignature": {
"type": "boolean",
"title": "Save dedupe signature",
"description": "If true,the signature used for dedupe will be stored in a 'dedupeSignature_s' field. Note this may cause errors about 'immense terms' in that field.",
"default": false,
"hints": ["advanced"]
},
"delete": {
"type": "boolean",
"title": "Delete dead URIs",
"description": "Set to true to remove documents from the index when they can no longer be accessed as unique documents.",
"default": true
},
"deleteErrorsAfter": {
"type": "integer",
"title": "Fetch failure allowance",
"description": "Number of fetch failures to tolerate before removing a document from the index. The default of -1 means no fetch failures will be removed.",
"default": -1
},
"fetchThreads": {
"type": "integer",
"title": "Fetch threads",
"description": "The number of threads to use during fetching. The default is 5.",
"default": 5
},
"emitThreads": {
"type": "integer",
"title": "Emit threads",
"description": "The number of threads used to send documents from the connector to the index pipeline. The default is 5.",
"default": 5
},
"chunkSize": {
"type": "integer",
"title": "Fetch batch size",
"description": "The number of items to batch for each round of fetching. A higher value can make crawling faster, but memory usage is also increased. The default is 1.",
"default": 1,
"hints": ["advanced"]
},
"fetchDelayMS": {
"type": "integer",
"title": "Fetch delay",
"description": "Number of milliseconds to wait between fetch requests. The default is 0. This property can be used to throttle a crawl if necessary.",
"default": 0,
"hints": ["advanced"]
},
"refreshAll": {
"type": "boolean",
"title": "Recrawl all items",
"description": "Set to true to always recrawl all items found in the crawldb.",
"default": true,
"hints": ["advanced"]
},
"refreshStartLinks": {
"type": "boolean",
"title": "Recrawl start links",
"description": "Set to true to recrawl items specified in the list of start links.",
"default": false,
"hints": ["advanced"]
},
"refreshErrors": {
"type": "boolean",
"title": "Recrawl errors",
"description": "Set to true to recrawl items that failed during the last crawl.",
"default": false,
"hints": ["advanced"]
},
"refreshOlderThan": {
"type": "integer",
"title": "Recrawl age",
"description": "Number of seconds to recrawl items whose last fetched date is longer ago than this value.",
"default": -1,
"hints": ["advanced"]
},
"refreshIDPrefixes": {
"type": "array",
"title": "Recrawl ID prefixes",
"description": "A prefix to recrawl all items whose IDs begin with this value.",
"hints": ["advanced"],
"items": {
"type": "string"
}
},
"refreshIDRegexes": {
"type": "array",
"title": "Recrawl ID regexes",
"description": "A regular expression to recrawl all items whose IDs match this pattern.",
"hints": ["advanced"],
"items": {
"type": "string"
}
},
"refreshScript": {
"type": "string",
"title": "Recrawl script",
"description": "A JavaScript function ('shouldRefresh()') to customize the items recrawled. ",
"hints": ["advanced", "code", "code/javascript"]
},
"forceRefresh": {
"type": "boolean",
"title": "Force recrawl",
"description": "Set to true to recrawl all items even if they have not changed since the last crawl.",
"default": false,
"hints": ["advanced"]
},
"forceRefreshClearSignatures": {
"type": "boolean",
"title": "Clear signatures",
"description": "If true, signatures will be cleared if force recrawl is enabled.",
"default": true,
"hints": ["advanced"]
},
"retryEmit": {
"type": "boolean",
"title": "Retry emits",
"description": "Set to true for emit batch failures to be retried on a document-by-document basis.",
"default": true,
"hints": ["advanced"]
},
"depth": {
"type": "integer",
"title": "Max crawl depth",
"description": "Number of levels in a directory or site tree to descend for documents.",
"default": -1
},
"maxItems": {
"type": "integer",
"title": "Max items",
"description": "Maximum number of documents to fetch. The default (-1) means no limit.",
"default": -1
},
"failFastOnStartLinkFailure": {
"type": "boolean",
"title": "Fail crawl if start links are invalid",
"description": "If true, when Fusion cannot connect to any of the provided start links, the crawl is stopped and an exception logged.",
"default": true,
"hints": ["advanced"]
},
"crawlDBType": {
"type": "string",
"title": "Crawl database type",
"description": "The type of crawl database to use, in-memory or on-disk.",
"enum": ["in-memory", "on-disk"],
"default": "on-disk",
"hints": ["advanced"]
},
"commitAfterItems": {
"type": "integer",
"title": "Commit After This Many Items",
"description": "Commit the crawlDB to disk after this many items have been received. A smaller number here will result in a slower crawl because of commits to disk being more frequent; conversely, a larger number here will cause a resumed job after a crash to need to recrawl more records.",
"default": 10000,
"hints": ["advanced"]
},
"initial_mapping": {
"type": "object",
"title": "Initial field mapping",
"description": "Provides mapping of fields before documents are sent to an index pipeline.",
"properties": {
"skip": {
"type": "boolean",
"title": "Skip This Stage",
"description": "Set to true to skip this stage.",
"default": false,
"hints": ["advanced"]
},
"label": {
"type": "string",
"title": "Label",
"description": "A unique label for this stage.",
"hints": ["advanced"],
"maxLength": 255
},
"condition": {
"type": "string",
"title": "Condition",
"description": "Define a conditional script that must result in true or false. This can be used to determine if the stage should process or not.",
"hints": ["code", "code/javascript", "advanced"]
},
"reservedFieldsMappingAllowed": {
"type": "boolean",
"title": "Allow System Fields Mapping?",
"default": false,
"hints": ["advanced"]
},
"mappings": {
"type": "array",
"title": "Field Mappings",
"description": "List of mapping rules",
"default": [{
"source": "charSet",
"target": "charSet_s",
"operation": "move"
}, {
"source": "fetchedDate",
"target": "fetchedDate_dt",
"operation": "move"
}, {
"source": "lastModified",
"target": "lastModified_dt",
"operation": "move"
}, {
"source": "signature",
"target": "dedupeSignature_s",
"operation": "move"
}, {
"source": "length",
"target": "length_l",
"operation": "move"
}, {
"source": "mimeType",
"target": "mimeType_s",
"operation": "move"
}, {
"source": "parent",
"target": "parent_s",
"operation": "move"
}, {
"source": "owner",
"target": "owner_s",
"operation": "move"
}, {
"source": "group",
"target": "group_s",
"operation": "move"
}],
"hints": ["advanced"],
"items": {
"type": "object",
"required": ["source"],
"properties": {
"source": {
"type": "string",
"title": "Source Field",
"description": "The name of the field to be mapped.",
"hints": ["advanced"]
},
"target": {
"type": "string",
"title": "Target Field",
"description": "The name of the field to be mapped to.",
"hints": ["advanced"]
},
"operation": {
"type": "string",
"title": "Operation",
"description": "The type of mapping to perform: move, copy, delete, add, set, or keep.",
"enum": ["copy", "move", "delete", "set", "add", "keep"],
"default": "copy",
"hints": ["advanced"]
}
}
}
},
"unmapped": {
"type": "object",
"title": "Unmapped Fields",
"description": "If fields do not match any of the field mapping rules, these rules will apply.",
"required": ["source"],
"properties": {
"source": {
"type": "string",
"title": "Source Field",
"description": "The name of the field to be mapped.",
"hints": ["advanced"]
},
"target": {
"type": "string",
"title": "Target Field",
"description": "The name of the field to be mapped to.",
"hints": ["advanced"]
},
"operation": {
"type": "string",
"title": "Operation",
"description": "The type of mapping to perform: move, copy, delete, add, set, or keep.",
"enum": ["copy", "move", "delete", "set", "add", "keep"],
"default": "copy",
"hints": ["advanced"]
}
},
"hints": ["advanced"]
}
},
"category": "Field Transformation",
"categoryPriority": 7,
"hints": ["advanced"],
"unsafe": false
},
"excludeExtensions": {
"type": "array",
"title": "Excluded file extensions",
"description": "File extensions that should not to be fetched. This will limit this datasource to all extensions except this list.",
"items": {
"type": "string"
}
},
"excludeRegexes": {
"type": "array",
"title": "Exclusive regexes",
"description": "Regular expressions for URI patterns to exclude. This will limit this datasource to only URIs that do not match the regular expression.",
"items": {
"type": "string"
}
},
"includeExtensions": {
"type": "array",
"title": "Included file extensions",
"description": "File extensions to be fetched. This will limit this datasource to only these file extensions.",
"items": {
"type": "string"
}
},
"includeRegexes": {
"type": "array",
"title": "Inclusive regexes",
"description": "Regular expressions for URI patterns to include. This will limit this datasource to only URIs that match the regular expression.",
"items": {
"type": "string"
}
},
"retainOutlinks": {
"type": "boolean",
"title": "Retain links in the crawldb",
"description": "Set to true for links found during fetching to be stored in the crawldb. This increases precision in certain recrawl scenarios, but requires more memory and disk space.",
"default": false,
"hints": ["advanced"]
},
"aliasExpiration": {
"type": "integer",
"title": "Alias expiration",
"description": "The number of crawls after which an alias will expire. The default is 1 crawl.",
"default": 1,
"hints": ["advanced"]
},
"restrictToTree": {
"type": "boolean",
"title": "Restrict crawl to start-link tree",
"description": "If true, only documents found in a tree below the start links will be fetched. By default, this means limiting the crawl to the domain of the start links. For example, if the start link is 'http://host.com/US' then only links to the 'host.com' domain will be followed. Further options are available for modifying this behavior.",
"default": false
},
"restrictToTreeAllowSubdomains": {
"type": "boolean",
"title": "Ignore sub-domains when restricting crawl",
"description": "Modifies the behavior of 'Restrict crawl to start-link tree' so that a link to any sub-domain of the start links is allowed. For example, if the start link is 'http://host.com', this option ensures that links to 'http://news.host.com' are also followed. This option requires 'Restrict to start-link tree' to be enabled to have any effect.",
"default": false
},
"restrictToTreeUseHostAndPath": {
"type": "boolean",
"title": "Restrict crawl to start-link path",
"description": "Modifies the behavior of 'Restrict crawl to start-link tree' to include the 'path' of the start link in the restriction logic. For example, if the start link is 'http://host.com/US', this option will limit all followed URLs to ones starting with the '/US/' path. This option requires 'Restrict to start-link tree' to be enabled to have any effect.",
"default": false
},
"restrictToTreeIgnoredHostPrefixes": {
"type": "array",
"title": "Restrict crawl host prefix exemptions",
"description": "Modifies the behavior of 'Restrict crawl to start-link tree' to ignore the configured list of prefixes when restricting the crawl. Commonly, 'www.' is ignored so links with the same domain are allowed, whether of the form 'http://host.com' or 'http://www.host.com'. This option requires 'Restrict to start-link tree' to be enabled to have any effect.",
"default": ["www."],
"items": {
"type": "string"
}
},
"f.script": {
"type": "string",
"title": "Script",
"description": "JavaScript program to fetch documents.",
"hints": ["code", "code/javascript"]
},
"rewriteLinkScript": {
"type": "string",
"title": "URI rewrite script",
"description": "A Javascript function 'rewriteLink(link) { }' to modify links to documents before they are fetched.",
"hints": ["advanced", "code", "code/javascript"]
},
"diagnosticMode": {
"type": "boolean",
"title": "Diagnostic mode",
"description": "Enable to print more detailed information to the logs about each request.",
"default": false,
"hints": ["advanced"]
}
},
"propertyGroups": [{
"label": "Link Discovery",
"properties": ["restrictToTree", "restrictToTreeAllowSubdomains", "restrictToTreeUseHostAndPath", "restrictToTreeIgnoredHostPrefixes"]
}, {
"label": "Limit Documents",
"properties": ["depth", "maxItems", "includeExtensions", "includeRegexes", "excludeExtensions", "excludeRegexes"]
}, {
"label": "Crawl Performance",
"properties": ["chunkSize", "fetchThreads", "fetchDelayMS", "emitThreads", "retryEmit", "failFastOnStartLinkFailure"]
}, {
"label": "Dedupe",
"properties": ["dedupe", "dedupeSaveSignature", "dedupeField", "dedupeScript"]
}, {
"label": "Recrawl Rules",
"properties": ["refreshAll", "refreshStartLinks", "refreshErrors", "refreshOlderThan", "refreshIDPrefixes", "refreshIDRegexes", "refreshScript", "forceRefresh", "forceRefreshClearSignatures", "delete", "deleteErrorsAfter"]
}, {
"label": "Crawl History",
"properties": ["retainOutlinks", "aliasExpiration", "crawlDBType", "commitAfterItems"]
}, {
"label": "Field Mapping",
"properties": ["initial_mapping"]
}]
}
},
"category": "Other",
"categoryPriority": 1,
"unsafe": false
};
export const SchemaParamFields = ({schema}) => {
const sanitize = str => {
if (typeof str !== "string") return str;
return str.replace(/^"(.*)"$/s, "$1").replace(/\\/g, "").replace(/"/g, "'");
};
const renderMd = str => {
const s = sanitize(str);
const text = (/[.!?]\)*$/).test(s) ? s : `${s}.`;
return text.split(/(\*\*[^*]+\*\*|_[^_]+_|`[^`]+`)/g).map((part, i) => {
if (part.startsWith("**")) return {part.slice(2, -2)};
if (part.startsWith("_")) return {part.slice(1, -1)};
if (part.startsWith("`")) return {part.slice(1, -1)};
return part;
});
};
const {description, properties = {}, required: requiredProps = []} = schema;
const visibleProps = useMemo(() => Object.entries(properties).filter(([, prop]) => !prop.hints?.includes("hidden")), [properties]);
const renderProp = ([name, prop]) => {
const isRequired = requiredProps.includes(name);
const hasDefault = prop.default !== undefined;
const rawDefault = prop.default;
const hints = prop.hints || [];
const isComplexDefault = hasDefault && (typeof rawDefault === "object" || typeof rawDefault === "string" && (rawDefault.length > 20 || rawDefault.includes('"')));
const postBadges = [];
if (prop.title) {
postBadges.push(<>API property: {name});
}
const constraints = [];
if (prop.minimum !== undefined && prop.maximum !== undefined) {
constraints.push(`Range: ${prop.minimum} – ${prop.maximum}`);
} else if (prop.minimum !== undefined) {
constraints.push(`Min: ${prop.minimum}`);
} else if (prop.maximum !== undefined) {
constraints.push(`Max: ${prop.maximum}`);
}
if (prop.minLength !== undefined && prop.maxLength !== undefined) {
constraints.push(`Length: ${prop.minLength} – ${prop.maxLength}`);
} else if (prop.minLength !== undefined) {
constraints.push(`Min length: ${prop.minLength}`);
} else if (prop.maxLength !== undefined) {
constraints.push(`Max length: ${prop.maxLength}`);
}
const fieldProps = {
key: name,
body: prop.title || name,
type: prop.type,
...postBadges.length > 0 && ({
post: postBadges
}),
...isRequired && ({
required: true
}),
...!isComplexDefault && hasDefault ? {
default: sanitize(String(rawDefault))
} : {}
};
const isObject = prop.type === "object" && prop.properties;
const isArrayOfObjects = prop.type === "array" && prop.items?.type === "object" && prop.items.properties;
return
{prop.description &&
{isLoaded && `
}} />}
Powered by Lucidworks Agent Studio
;
};
[localhost link]: http://localhost:3000/docs/fusion-connectors/connectors/javascript
[mintlify link]: https://doc.lucidworks.com/docs/fusion-connectors/connectors/javascript
[old doc.lw link]: https://doc.lucidworks.com/fusion-connectors/81
**Compatible with Fusion version:** 4.2.0 through 5.2.2
Deprecation and removal notice
This connector is deprecated as of August 24, 2020 and is removed or expected to be removed as of May 18, 2022. Use the index pipeline for fetching content.
For more information about deprecations and removals, including possible alternatives, see [Deprecations and Removals](/docs/fusion-connectors/deprecations-and-removals).
The Javascript connector executes a JavaScript program that is compiled by the JDK.
This program returns a content item which is handed off to the fetcher. The JavaScript program must be standard [ECMAScript](http://en.wikipedia.org/wiki/ECMAScript).
You can use any Java class available to the connectors JDK ClassLoader to manipulate that object within a function.
As in Java, to access Java classes by their simple names instead of their fully specified class names, e.g. to be able to write `String` instead of `java.lang.String`, these classes must be imported.
The java.lang package is not imported by default, because its classes would conflict with Object, Boolean, Math, and other built-in JavaScript objects.
To import a Java class, use the JavaImporter object and the `with` statement, which limits the scope of the imported Java packages and classes.
```java wrap theme={"dark"}
var imports = new JavaImporter(java.lang.String);
...
with (imports) {
var name = new String("foo"); ...
}
```
For global variables, you can reference these objects using the `Java.type` API extension.
See this tutorial for details: [http://winterbe.com/posts/2014/04/05/java8-nashorn-tutorial/](http://winterbe.com/posts/2014/04/05/java8-nashorn-tutorial/)
There is a known issue with `JavaImporter` that causes intermittent errors with patterns like `with (imports) { } *`. The preferred approach is `Java.type()`.
A pattern that works well is:
```java wrap theme={"dark"}
var SolrQuery = Java.type("org.apache.solr.client.solrj.SolrQuery");
var query = new SolrQuery();
```
An example of `Java.type()` usage:
```java wrap expandable theme={"dark"}
function (request, response, ctx, collection, solrServer, solrServerFactory) {
//Type imports
var String = Java.type("java.lang.String");
var ArrayList = Java.type("java.util.ArrayList");
var HashMap = Java.type("java.util.LinkedHashMap");
var HashSet = Java.type("java.util.LinkedHashSet");
var TreeSet = Java.type("java.util.TreeSet");
var Integer = Java.type("java.lang.Integer");
//get map out of response
var map = response.getInnerResponse().getUnderlyingObject();
var facets = map.get("facets");
var facetBoostValues = ctx.getProperty("boostValues.json.facet");
var facetBuryValues = ctx.getProperty("buryValues.json.facet");
var facetSuppressValues = ctx.getProperty("suppressValues.json.facet");
var facetMultiValue = ctx.getProperty("multivalueMap.json.facet");
if (facets != null) {
//Process JSON facets
if (facets.get("count") != null) facets.remove("count");
if (facets.get("products") != null) facets.remove("products");
// Set multivalue and number to show as defaults
var facetKeySet = facets.keySet();
if (facetKeySet != null) {
var facetIterator = facetKeySet.iterator();
while (facetIterator.hasNext()) {
var facet = facetIterator.next();
logger.info("==================");
logger.info("Processing facet: "+facet.toString());
facets.get(facet).put("multivalue", true);
var multiSelect = facetMultiValue.get(facet.toString());
if (null != multiSelect && multiSelect == 'single') {
facets.get(facet).put("multivalue", false);
logger.info("multi: False");
}
var buckets = facets.get(facet).get("buckets");
var boostValues = new ArrayList();
var boostBuckets = new ArrayList();
var boostValList = new ArrayList();
if (null != facetBoostValues) {
boostValues = facetBoostValues.get(facet.toString());
logger.info("BOOST: "+boostValues);
}
var buryValues = new ArrayList();
var buryBuckets = new ArrayList();
var buryValList = new ArrayList();
if (null != facetBuryValues) {
buryValues = facetBuryValues.get(facet.toString());
logger.info("BURY: "+buryValues);
}
var suppressValues = new ArrayList();
if (null != facetSuppressValues) {
suppressValues = facetSuppressValues.get(facet.toString());
logger.info("suppressValues: "+suppressValues);
}
var normalBuckets = new ArrayList();
for (var i = 0; i < buckets.size(); i++) {
var bucket = buckets[i];
var val = bucket.get("val");
if (null != boostValues && boostValues.indexOf(val) > -1) {
boostBuckets.add(bucket);
boostValList.add(val);
} else if (null != buryValues && buryValues.indexOf(val) > -1) {
buryBuckets.add(bucket);
buryValList.add(val);
} else if ((null == suppressValues) || (suppressValues.isEmpty()) || (suppressValues.indexOf(val) < 0)) {
normalBuckets.add(bucket);
}
}
if (null != boostValues) {
var finalBoostedBuckets = new ArrayList();
for (var i = 0; i < boostValues.size(); i++) {
var boostIndex = boostValList.indexOf(boostValues[i]);
if (boostIndex > -1) {
finalBoostedBuckets.add(boostBuckets.get(boostIndex));
}
}
if (finalBoostedBuckets.size() > 0) {
finalBoostedBuckets.addAll(normalBuckets);
facets.get(facet).put("buckets", finalBoostedBuckets);
}
}
if (null != buryValues) {
var finalBuriedBuckets = new ArrayList();
for (var i = 0; i < buryValues.size(); i++) {
var buryIndex = buryValList.indexOf(buryValues[i]);
if (buryIndex > -1) {
finalBuriedBuckets.add(buryBuckets.get(buryIndex));
}
}
if (finalBuriedBuckets.size() > 0) {
normalBuckets.addAll(finalBuriedBuckets);
facets.get(facet).put("buckets", normalBuckets);
}
}
}
}
}
}
```
## The JavaScript Program
The Javascript context provides the following variables:
| Variable | Type | Description |
| -------------- | ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `id` | java.lang.String | The ID of the object to fetch. This is almost always the URI of the datasource to connect to and fetch content. |
| `lastModified` | long | The time since the epoch from which the item was last touched. |
| `signature` | java.lang.String | An optional string meant to be used to compare versions of the ID being fetched, e.g. an ETag in a web-crawl. |
| `content` | crawler.common.MutableObject | A Content object that can be modified and returned, for fine grained control over the return. |
| `_fetcher` | Fetcher | The current Fetcher instance (usually type `JavascriptFetcher`), used to interact with the Fetcher, including getting a WebFetcher instance using `_fetcher.getWebFetcher()`. |
| `_context` | java.util.Map | A map used to store data to persist across calls to `fetch()`, e.g. an instance of WebFetcher obtained using `_fetcher.getWebFetcher()`. |
The program must return one of the following kinds of objects:
| Object | Description |
| ------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| String | A string object. This is converted to UTF-8 bytes and added as the raw content on a `common.crawler.Content` object and returned from the `fetch()` method. |
| byte \[] | A byte array. This array is set on a `common.crawler.Content` object and returned from the `fetch()` method. |
| `common.crawler.MutableContent` | If you want to have complete control over the return from `fetch()`, make changes to the content object provided in the Context and return it. Do not create a new object. |
| An array of Objects | The array is converted to Embedded Content. The Fetcher returns a parent Content object that has a "Container" discardMessage. The Embedded Content on that container is generated by calling `toString()` on the objects in the array. |
| A JavaScript Map | The map is converted to fields on the Content item returned. |
If the JavaScript script is implemented as a function, the return statement must return one of the above types. If the script is not function-based, the last line in the script must evaluate to one of these object types.
## Examples
### Return content as a java.lang.String
```java wrap theme={"dark"}
var str = new java.lang.String("Java");
str;
```
### Return content as a byte array
```java wrap theme={"dark"}
var bytes = new java.lang.String("Java");
bytes.getBytes('UTF-8');
```
### Return content as a JavaScript array
```java wrap theme={"dark"}
var strings = ["hi", "bye"];
strings;
```
### Return content as a JavaScript map
```java wrap theme={"dark"}
var map = {"hi": "bye", "bye": "hi", "number":1};
map;
```
### Leverage the Fetcher
```java wrap theme={"dark"}
var webFetcher = _context.get("webFetcher");
if (null == webFetcher) {
webFetcher = _fetcher.getWebFetcher();
// it is possible to pass config options to getWebFetcher() as a map as well, e.g.:
// _fetcher.getWebFetcher({"f.discardLinkURLQueries" : false });
_context.put("webFetcher", webFetcher);
}
var webContent = webFetcher.fetch(id, lastModified, signature);
var jsoupDoc = webContent.getDocument();
if (null !== jsoupDoc) {
// modify the Jsoup document or web-content as-needed here, adding new links, removing sections etc.
// ...
// ...
webContent.setRawContent(jsoupDoc.toString().getBytes("UTF-8"));
}
webContent;
```
## Configuration
When entering configuration values in the UI, use *unescaped* characters, such as `\t` for the tab character. When entering configuration values in the API, use *escaped* characters, such as `\\t` for the tab character.