File System Crawler

Start here to configure a file system crawler. Use this skeleton, then drill down into concrete type pages for details.

Usage

YAML

changeDiscovery: CRAWLER_SCAN
cluster:
  adminDisabled: false
  adminPort: 0
  clustered: false
  connector:
    class: HazelcastClusterConnector
committers: []
deferredShutdownDuration: 0
documentChecksummer:
  class: Md5DocumentChecksummer
  combineFieldsAndContent: false
  fieldMatcher: {}
  keep: false
  onSet: APPEND
  toField: string
documentDeduplicate: false
documentFetchSupport: DISABLED
documentFilters: []
eventListeners: []
fetchers: []
fetchersMaxRetries: 0
fetchersRetryDelay: 0
id: string
idleTimeout: 0
importer:
  handlers: []
  maxMemoryInstance: 0
  maxMemoryPool: 0
  responseProcessors: []
  tempDir: string
maxCrawlDuration: 0
maxDepth: 0
maxDocuments: 0
maxQueueBatchSize: 0
maxStreamCachePoolSize: 0
maxStreamCacheSize: 0
metadataChecksummer:
  class: LastModifiedMetadataChecksummer
  keep: false
  onSet: APPEND
  toField: string
metadataDeduplicate: false
metadataFetchSupport: DISABLED
metadataFilters: []
minProgressLoggingInterval: 0
numThreads: 0
orphansStrategy: PROCESS
postImportConsumers: []
preImportConsumers: []
referenceFilters: []
spoiledReferenceStrategizer:
  class: GenericSpoiledReferenceStrategizer
  fallbackStrategy: DELETE
  mappings: {}
startReferences:
  - string
startReferencesAsync: false
startReferencesFiles:
  - string
startReferencesProviders:
  - {}
stopOnExceptions:
  - string
workDir: string

JSON

{
  "changeDiscovery": "CRAWLER_SCAN",
  "cluster": {
    "adminDisabled": false,
    "adminPort": 0,
    "clustered": false,
    "connector": {
      "class": "HazelcastClusterConnector"
    }
  },
  "committers": [],
  "deferredShutdownDuration": 0,
  "documentChecksummer": {
    "class": "Md5DocumentChecksummer",
    "combineFieldsAndContent": false,
    "fieldMatcher": {},
    "keep": false,
    "onSet": "APPEND",
    "toField": "string"
  },
  "documentDeduplicate": false,
  "documentFetchSupport": "DISABLED",
  "documentFilters": [],
  "eventListeners": [],
  "fetchers": [],
  "fetchersMaxRetries": 0,
  "fetchersRetryDelay": 0,
  "id": "string",
  "idleTimeout": 0,
  "importer": {
    "handlers": [],
    "maxMemoryInstance": 0,
    "maxMemoryPool": 0,
    "responseProcessors": [],
    "tempDir": "string"
  },
  "maxCrawlDuration": 0,
  "maxDepth": 0,
  "maxDocuments": 0,
  "maxQueueBatchSize": 0,
  "maxStreamCachePoolSize": 0,
  "maxStreamCacheSize": 0,
  "metadataChecksummer": {
    "class": "LastModifiedMetadataChecksummer",
    "keep": false,
    "onSet": "APPEND",
    "toField": "string"
  },
  "metadataDeduplicate": false,
  "metadataFetchSupport": "DISABLED",
  "metadataFilters": [],
  "minProgressLoggingInterval": 0,
  "numThreads": 0,
  "orphansStrategy": "PROCESS",
  "postImportConsumers": [],
  "preImportConsumers": [],
  "referenceFilters": [],
  "spoiledReferenceStrategizer": {
    "class": "GenericSpoiledReferenceStrategizer",
    "fallbackStrategy": "DELETE",
    "mappings": {}
  },
  "startReferences": [
    "string"
  ],
  "startReferencesAsync": false,
  "startReferencesFiles": [
    "string"
  ],
  "startReferencesProviders": [
    {}
  ],
  "stopOnExceptions": [
    "string"
  ],
  "workDir": "string"
}

XML

<crawler>
  <changeDiscovery>CRAWLER_SCAN</changeDiscovery>
  <cluster>
    <adminDisabled>false</adminDisabled>
    <adminPort>0</adminPort>
    <clustered>false</clustered>
    <connector>
      <class>HazelcastClusterConnector</class>
    </connector>
  </cluster>
  <committers>
    <committer/>
  </committers>
  <deferredShutdownDuration>0</deferredShutdownDuration>
  <documentChecksummer>
    <class>Md5DocumentChecksummer</class>
    <combineFieldsAndContent>false</combineFieldsAndContent>
    <fieldMatcher/>
    <keep>false</keep>
    <onSet>APPEND</onSet>
    <toField>string</toField>
  </documentChecksummer>
  <documentDeduplicate>false</documentDeduplicate>
  <documentFetchSupport>DISABLED</documentFetchSupport>
  <documentFilters>
    <documentFilter/>
  </documentFilters>
  <eventListeners>
    <eventListener/>
  </eventListeners>
  <fetchers>
    <fetcher/>
  </fetchers>
  <fetchersMaxRetries>0</fetchersMaxRetries>
  <fetchersRetryDelay>0</fetchersRetryDelay>
  <id>string</id>
  <idleTimeout>0</idleTimeout>
  <importer>
    <handlers>
      <handler/>
    </handlers>
    <maxMemoryInstance>0</maxMemoryInstance>
    <maxMemoryPool>0</maxMemoryPool>
    <responseProcessors>
      <responseProcessor/>
    </responseProcessors>
    <tempDir>string</tempDir>
  </importer>
  <maxCrawlDuration>0</maxCrawlDuration>
  <maxDepth>0</maxDepth>
  <maxDocuments>0</maxDocuments>
  <maxQueueBatchSize>0</maxQueueBatchSize>
  <maxStreamCachePoolSize>0</maxStreamCachePoolSize>
  <maxStreamCacheSize>0</maxStreamCacheSize>
  <metadataChecksummer>
    <class>LastModifiedMetadataChecksummer</class>
    <keep>false</keep>
    <onSet>APPEND</onSet>
    <toField>string</toField>
  </metadataChecksummer>
  <metadataDeduplicate>false</metadataDeduplicate>
  <metadataFetchSupport>DISABLED</metadataFetchSupport>
  <metadataFilters>
    <metadataFilter/>
  </metadataFilters>
  <minProgressLoggingInterval>0</minProgressLoggingInterval>
  <numThreads>0</numThreads>
  <orphansStrategy>PROCESS</orphansStrategy>
  <postImportConsumers>
    <postImportConsumer/>
  </postImportConsumers>
  <preImportConsumers>
    <preImportConsumer/>
  </preImportConsumers>
  <referenceFilters>
    <referenceFilter/>
  </referenceFilters>
  <spoiledReferenceStrategizer>
    <class>GenericSpoiledReferenceStrategizer</class>
    <fallbackStrategy>DELETE</fallbackStrategy>
    <mappings/>
  </spoiledReferenceStrategizer>
  <startReferences>
    <ref>string</ref>
  </startReferences>
  <startReferencesAsync>false</startReferencesAsync>
  <startReferencesFiles>
    <file>string</file>
  </startReferencesFiles>
  <startReferencesProviders>
    <provider/>
  </startReferencesProviders>
  <stopOnExceptions>
    <stopOnException>string</stopOnException>
  </stopOnExceptions>
  <workDir>string</workDir>
</crawler>

Key Types

• CrawlerConfig
• Fetcher
• DocumentFilter
• ReferenceFilter