FsCrawlerConfig

Configuration schema for Fs Crawler Config.

crawler/fsJavadoc: FsCrawlerConfig

Usage

YAML

changeDiscovery: CRAWLER_SCAN
cluster:
  adminBindAddress: string
  adminDisabled: false
  adminPort: 0
  clustered: false
  connector:
    class: HazelcastClusterConnector
    clusterName: string
    configurer:
      class: JdbcHazelcastConfigurer
    instanceName: string
    nodeExpiryTimeout: 0
    workerHeartbeatInterval: 0
committers: []
deferredShutdownDuration: 0
documentChecksummer:
  class: Md5DocumentChecksummer
  combineFieldsAndContent: false
  fieldMatcher:
    ignoreCase: false
    ignoreDiacritic: false
    matchEmpty: false
    method: BASIC
    negateMatches: false
    partial: false
    pattern: string
    replaceAll: false
    trim: false
  keep: false
  onSet: APPEND
  toField: string
documentDeduplicate: false
documentFetchSupport: DISABLED
documentFilters: []
eventListeners: []
fetchers: []
fetchersMaxRetries: 0
fetchersRetryDelay: 0
id: string
idleTimeout: 0
importer:
  handlers: []
  maxMemoryInstance: 0
  maxMemoryPool: 0
  responseProcessors:
    - {}
  tempDir: string
maxCrawlDuration: 0
maxDepth: 0
maxDocuments: 0
maxQueueBatchSize: 0
maxStreamCachePoolSize: 0
maxStreamCacheSize: 0
metadataChecksummer:
  class: LastModifiedMetadataChecksummer
  keep: false
  onSet: APPEND
  toField: string
metadataDeduplicate: false
metadataFetchSupport: DISABLED
metadataFilters: []
minProgressLoggingInterval: 0
numThreads: 0
orphansStrategy: PROCESS
postImportConsumers: []
preImportConsumers: []
referenceFilters: []
spoiledReferenceStrategizer:
  class: GenericSpoiledReferenceStrategizer
  fallbackStrategy: DELETE
  mappings: {}
startReferences:
  - string
startReferencesAsync: false
startReferencesFiles:
  - string
startReferencesProviders:
  - noop: string
stopOnExceptions:
  - string
workDir: string

JSON

{
  "changeDiscovery": "CRAWLER_SCAN",
  "cluster": {
    "adminBindAddress": "string",
    "adminDisabled": false,
    "adminPort": 0,
    "clustered": false,
    "connector": {
      "class": "HazelcastClusterConnector",
      "clusterName": "string",
      "configurer": {
        "class": "JdbcHazelcastConfigurer"
      },
      "instanceName": "string",
      "nodeExpiryTimeout": 0,
      "workerHeartbeatInterval": 0
    }
  },
  "committers": [],
  "deferredShutdownDuration": 0,
  "documentChecksummer": {
    "class": "Md5DocumentChecksummer",
    "combineFieldsAndContent": false,
    "fieldMatcher": {
      "ignoreCase": false,
      "ignoreDiacritic": false,
      "matchEmpty": false,
      "method": "BASIC",
      "negateMatches": false,
      "partial": false,
      "pattern": "string",
      "replaceAll": false,
      "trim": false
    },
    "keep": false,
    "onSet": "APPEND",
    "toField": "string"
  },
  "documentDeduplicate": false,
  "documentFetchSupport": "DISABLED",
  "documentFilters": [],
  "eventListeners": [],
  "fetchers": [],
  "fetchersMaxRetries": 0,
  "fetchersRetryDelay": 0,
  "id": "string",
  "idleTimeout": 0,
  "importer": {
    "handlers": [],
    "maxMemoryInstance": 0,
    "maxMemoryPool": 0,
    "responseProcessors": [
      {}
    ],
    "tempDir": "string"
  },
  "maxCrawlDuration": 0,
  "maxDepth": 0,
  "maxDocuments": 0,
  "maxQueueBatchSize": 0,
  "maxStreamCachePoolSize": 0,
  "maxStreamCacheSize": 0,
  "metadataChecksummer": {
    "class": "LastModifiedMetadataChecksummer",
    "keep": false,
    "onSet": "APPEND",
    "toField": "string"
  },
  "metadataDeduplicate": false,
  "metadataFetchSupport": "DISABLED",
  "metadataFilters": [],
  "minProgressLoggingInterval": 0,
  "numThreads": 0,
  "orphansStrategy": "PROCESS",
  "postImportConsumers": [],
  "preImportConsumers": [],
  "referenceFilters": [],
  "spoiledReferenceStrategizer": {
    "class": "GenericSpoiledReferenceStrategizer",
    "fallbackStrategy": "DELETE",
    "mappings": {}
  },
  "startReferences": [
    "string"
  ],
  "startReferencesAsync": false,
  "startReferencesFiles": [
    "string"
  ],
  "startReferencesProviders": [
    {
      "noop": "string"
    }
  ],
  "stopOnExceptions": [
    "string"
  ],
  "workDir": "string"
}

XML

<fsCrawlerConfig>
  <changeDiscovery>CRAWLER_SCAN</changeDiscovery>
  <cluster>
    <adminBindAddress>string</adminBindAddress>
    <adminDisabled>false</adminDisabled>
    <adminPort>0</adminPort>
    <clustered>false</clustered>
    <connector>
      <class>HazelcastClusterConnector</class>
      <clusterName>string</clusterName>
      <configurer/>
      <instanceName>string</instanceName>
      <nodeExpiryTimeout>0</nodeExpiryTimeout>
      <workerHeartbeatInterval>0</workerHeartbeatInterval>
    </connector>
  </cluster>
  <committers>
    <committer/>
  </committers>
  <deferredShutdownDuration>0</deferredShutdownDuration>
  <documentChecksummer>
    <class>Md5DocumentChecksummer</class>
    <combineFieldsAndContent>false</combineFieldsAndContent>
    <fieldMatcher/>
    <keep>false</keep>
    <onSet>APPEND</onSet>
    <toField>string</toField>
  </documentChecksummer>
  <documentDeduplicate>false</documentDeduplicate>
  <documentFetchSupport>DISABLED</documentFetchSupport>
  <documentFilters>
    <documentFilter/>
  </documentFilters>
  <eventListeners>
    <eventListener/>
  </eventListeners>
  <fetchers>
    <fetcher/>
  </fetchers>
  <fetchersMaxRetries>0</fetchersMaxRetries>
  <fetchersRetryDelay>0</fetchersRetryDelay>
  <id>string</id>
  <idleTimeout>0</idleTimeout>
  <importer>
    <handlers>
      <handler/>
    </handlers>
    <maxMemoryInstance>0</maxMemoryInstance>
    <maxMemoryPool>0</maxMemoryPool>
    <responseProcessors>
      <responseProcessor/>
    </responseProcessors>
    <tempDir>string</tempDir>
  </importer>
  <maxCrawlDuration>0</maxCrawlDuration>
  <maxDepth>0</maxDepth>
  <maxDocuments>0</maxDocuments>
  <maxQueueBatchSize>0</maxQueueBatchSize>
  <maxStreamCachePoolSize>0</maxStreamCachePoolSize>
  <maxStreamCacheSize>0</maxStreamCacheSize>
  <metadataChecksummer>
    <class>LastModifiedMetadataChecksummer</class>
    <keep>false</keep>
    <onSet>APPEND</onSet>
    <toField>string</toField>
  </metadataChecksummer>
  <metadataDeduplicate>false</metadataDeduplicate>
  <metadataFetchSupport>DISABLED</metadataFetchSupport>
  <metadataFilters>
    <metadataFilter/>
  </metadataFilters>
  <minProgressLoggingInterval>0</minProgressLoggingInterval>
  <numThreads>0</numThreads>
  <orphansStrategy>PROCESS</orphansStrategy>
  <postImportConsumers>
    <postImportConsumer/>
  </postImportConsumers>
  <preImportConsumers>
    <preImportConsumer/>
  </preImportConsumers>
  <referenceFilters>
    <referenceFilter/>
  </referenceFilters>
  <spoiledReferenceStrategizer>
    <class>GenericSpoiledReferenceStrategizer</class>
    <fallbackStrategy>DELETE</fallbackStrategy>
    <mappings/>
  </spoiledReferenceStrategizer>
  <startReferences>
    <ref>string</ref>
  </startReferences>
  <startReferencesAsync>false</startReferencesAsync>
  <startReferencesFiles>
    <file>string</file>
  </startReferencesFiles>
  <startReferencesProviders>
    <provider>
      <noop>string</noop>
    </provider>
  </startReferencesProviders>
  <stopOnExceptions>
    <stopOnException>string</stopOnException>
  </stopOnExceptions>
  <workDir>string</workDir>
</fsCrawlerConfig>

Properties

Property	Type	Required	Default
changeDiscovery	enum	No	CRAWLER_SCAN
cluster	ClusterConfig	No	ClusterConfig
committers	Committer[]	No	-
deferredShutdownDuration	integer	No	0
documentChecksummer	DocumentChecksummer	No	Md5DocumentChecksummer
documentDeduplicate	boolean	No	false
documentFetchSupport	enum	No	REQUIRED
documentFilters	DocumentFilter[]	No	-
eventListeners	EventListener[]	No	-
fetchers	Fetcher[]	No	[LocalFetcher]
fetchersMaxRetries	integer	No	0
fetchersRetryDelay	integer	No	-
id	string	Yes	-
idleTimeout	integer	No	5000
importer	ImporterConfig	No	ImporterConfig
maxCrawlDuration	integer	No	0
maxDepth	integer	No	-1
maxDocuments	integer	No	-1
maxQueueBatchSize	integer	No	50
maxStreamCachePoolSize	integer	No	100000000
maxStreamCacheSize	integer	No	1000000000
metadataChecksummer	MetadataChecksummer	No	-
metadataDeduplicate	boolean	No	false
metadataFetchSupport	enum	No	DISABLED
metadataFilters	MetadataFilter[]	No	-
minProgressLoggingInterval	integer	No	30000
numThreads	integer	No	2
orphansStrategy	enum	No	PROCESS
postImportConsumers	DocumentConsumer[]	No	-
preImportConsumers	DocumentConsumer[]	No	-
referenceFilters	ReferenceFilter[]	No	-
spoiledReferenceStrategizer	SpoiledReferenceStrategizer	No	GenericSpoiledReferenceStrategizer
startReferences	string[]	No	-
startReferencesAsync	boolean	No	false
startReferencesFiles	string[]	No	-
startReferencesProviders	ReferencesProvider[]	No	-
stopOnExceptions	string[]	No	-
workDir	string	No	-

Property Details

changeDiscovery

Allowed Values

• CRAWLER_SCAN
• SOURCE_DELTA

cluster

:::warning Missing Documentation Property cluster is missing documentation. Please update this section. :::

committers

:::warning Missing Documentation Property committers is missing documentation. Please update this section. :::

deferredShutdownDuration

:::warning Missing Documentation Property deferredShutdownDuration is missing documentation. Please update this section. :::

documentChecksummer

:::warning Missing Documentation Property documentChecksummer is missing documentation. Please update this section. :::

documentDeduplicate

:::warning Missing Documentation Property documentDeduplicate is missing documentation. Please update this section. :::

documentFetchSupport

Allowed Values

• DISABLED
• OPTIONAL
• REQUIRED

documentFilters

:::warning Missing Documentation Property documentFilters is missing documentation. Please update this section. :::

eventListeners

:::warning Missing Documentation Property eventListeners is missing documentation. Please update this section. :::

fetchers

:::warning Missing Documentation Property fetchers is missing documentation. Please update this section. :::

fetchersMaxRetries

:::warning Missing Documentation Property fetchersMaxRetries is missing documentation. Please update this section. :::

fetchersRetryDelay

:::warning Missing Documentation Property fetchersRetryDelay is missing documentation. Please update this section. :::

id

:::warning Missing Documentation Property id is missing documentation. Please update this section. :::

idleTimeout

:::warning Missing Documentation Property idleTimeout is missing documentation. Please update this section. :::

importer

:::warning Missing Documentation Property importer is missing documentation. Please update this section. :::

maxCrawlDuration

:::warning Missing Documentation Property maxCrawlDuration is missing documentation. Please update this section. :::

maxDepth

:::warning Missing Documentation Property maxDepth is missing documentation. Please update this section. :::

maxDocuments

:::warning Missing Documentation Property maxDocuments is missing documentation. Please update this section. :::

maxQueueBatchSize

:::warning Missing Documentation Property maxQueueBatchSize is missing documentation. Please update this section. :::

maxStreamCachePoolSize

:::warning Missing Documentation Property maxStreamCachePoolSize is missing documentation. Please update this section. :::

maxStreamCacheSize

:::warning Missing Documentation Property maxStreamCacheSize is missing documentation. Please update this section. :::

metadataChecksummer

:::warning Missing Documentation Property metadataChecksummer is missing documentation. Please update this section. :::

metadataDeduplicate

:::warning Missing Documentation Property metadataDeduplicate is missing documentation. Please update this section. :::

metadataFetchSupport

Allowed Values

• DISABLED
• OPTIONAL
• REQUIRED

metadataFilters

:::warning Missing Documentation Property metadataFilters is missing documentation. Please update this section. :::

minProgressLoggingInterval

:::warning Missing Documentation Property minProgressLoggingInterval is missing documentation. Please update this section. :::

numThreads

:::warning Missing Documentation Property numThreads is missing documentation. Please update this section. :::

orphansStrategy

Allowed Values

• PROCESS
• DELETE
• IGNORE

postImportConsumers

:::warning Missing Documentation Property postImportConsumers is missing documentation. Please update this section. :::

preImportConsumers

:::warning Missing Documentation Property preImportConsumers is missing documentation. Please update this section. :::

referenceFilters

:::warning Missing Documentation Property referenceFilters is missing documentation. Please update this section. :::

spoiledReferenceStrategizer

:::warning Missing Documentation Property spoiledReferenceStrategizer is missing documentation. Please update this section. :::

startReferences

:::warning Missing Documentation Property startReferences is missing documentation. Please update this section. :::

startReferencesAsync

:::warning Missing Documentation Property startReferencesAsync is missing documentation. Please update this section. :::

startReferencesFiles

:::warning Missing Documentation Property startReferencesFiles is missing documentation. Please update this section. :::

startReferencesProviders

:::warning Missing Documentation Property startReferencesProviders is missing documentation. Please update this section. :::

stopOnExceptions

:::warning Missing Documentation Property stopOnExceptions is missing documentation. Please update this section. :::

workDir

:::warning Missing Documentation Property workDir is missing documentation. Please update this section. :::