Web Crawler

Start here to configure a web crawler. Use this skeleton, then drill down into concrete type pages for details.

Usage

YAML

canonicalLinkDetector:
  class: GenericCanonicalLinkDetector
  contentTypes: []
changeDiscovery: CRAWLER_SCAN
cluster:
  adminDisabled: false
  adminPort: 0
  clustered: false
  connector:
    class: HazelcastClusterConnector
committers: []
deferredShutdownDuration: 0
delayResolver:
  class: GenericDelayResolver
  defaultDelay: 0
  ignoreRobotsCrawlDelay: false
  schedules: []
  scope: CRAWLER
documentChecksummer:
  class: Md5DocumentChecksummer
  combineFieldsAndContent: false
  fieldMatcher: {}
  keep: false
  onSet: APPEND
  toField: string
documentDeduplicate: false
documentFetchSupport: DISABLED
documentFilters: []
eventListeners: []
fetchers: []
fetchersMaxRetries: 0
fetchersRetryDelay: 0
id: string
idleTimeout: 0
importer:
  handlers: []
  maxMemoryInstance: 0
  maxMemoryPool: 0
  responseProcessors: []
  tempDir: string
keepReferencedLinks:
  - INSCOPE
linkExtractors: []
maxCrawlDuration: 0
maxDepth: 0
maxDocuments: 0
maxQueueBatchSize: 0
maxStreamCachePoolSize: 0
maxStreamCacheSize: 0
metadataChecksummer:
  class: LastModifiedMetadataChecksummer
  keep: false
  onSet: APPEND
  toField: string
metadataDeduplicate: false
metadataFetchSupport: DISABLED
metadataFilters: []
minProgressLoggingInterval: 0
numThreads: 0
orphansStrategy: PROCESS
postImportConsumers: []
postImportLinks:
  ignoreCase: false
  ignoreDiacritic: false
  matchEmpty: false
  method: BASIC
  negateMatches: false
  partial: false
  pattern: string
  replaceAll: false
  trim: false
postImportLinksKeep: false
preImportConsumers: []
recrawlableResolver:
  class: GenericRecrawlableResolver
  minFrequencies: []
  sitemapSupport: FIRST
referenceFilters: []
robotsMetaProvider:
  class: StandardRobotsMetaProvider
  headersPrefix: string
robotsTxtProvider:
  class: StandardRobotsTxtProvider
  noop: string
sitemapLocator:
  class: GenericSitemapLocator
  paths: []
  robotsTxtSitemapDisabled: false
sitemapResolver:
  class: GenericSitemapResolver
  lenient: false
spoiledReferenceStrategizer:
  class: GenericSpoiledReferenceStrategizer
  fallbackStrategy: DELETE
  mappings: {}
startReferences:
  - string
startReferencesAsync: false
startReferencesFiles:
  - string
startReferencesProviders:
  - {}
startReferencesSitemaps:
  - string
stopOnExceptions:
  - string
urlNormalizers: []
urlScopeResolver:
  class: GenericUrlScopeResolver
  includeSubdomains: false
  stayOnDomain: false
  stayOnPort: false
  stayOnProtocol: false
  stayOnSitemap: false
workDir: string

JSON

{
  "canonicalLinkDetector": {
    "class": "GenericCanonicalLinkDetector",
    "contentTypes": []
  },
  "changeDiscovery": "CRAWLER_SCAN",
  "cluster": {
    "adminDisabled": false,
    "adminPort": 0,
    "clustered": false,
    "connector": {
      "class": "HazelcastClusterConnector"
    }
  },
  "committers": [],
  "deferredShutdownDuration": 0,
  "delayResolver": {
    "class": "GenericDelayResolver",
    "defaultDelay": 0,
    "ignoreRobotsCrawlDelay": false,
    "schedules": [],
    "scope": "CRAWLER"
  },
  "documentChecksummer": {
    "class": "Md5DocumentChecksummer",
    "combineFieldsAndContent": false,
    "fieldMatcher": {},
    "keep": false,
    "onSet": "APPEND",
    "toField": "string"
  },
  "documentDeduplicate": false,
  "documentFetchSupport": "DISABLED",
  "documentFilters": [],
  "eventListeners": [],
  "fetchers": [],
  "fetchersMaxRetries": 0,
  "fetchersRetryDelay": 0,
  "id": "string",
  "idleTimeout": 0,
  "importer": {
    "handlers": [],
    "maxMemoryInstance": 0,
    "maxMemoryPool": 0,
    "responseProcessors": [],
    "tempDir": "string"
  },
  "keepReferencedLinks": [
    "INSCOPE"
  ],
  "linkExtractors": [],
  "maxCrawlDuration": 0,
  "maxDepth": 0,
  "maxDocuments": 0,
  "maxQueueBatchSize": 0,
  "maxStreamCachePoolSize": 0,
  "maxStreamCacheSize": 0,
  "metadataChecksummer": {
    "class": "LastModifiedMetadataChecksummer",
    "keep": false,
    "onSet": "APPEND",
    "toField": "string"
  },
  "metadataDeduplicate": false,
  "metadataFetchSupport": "DISABLED",
  "metadataFilters": [],
  "minProgressLoggingInterval": 0,
  "numThreads": 0,
  "orphansStrategy": "PROCESS",
  "postImportConsumers": [],
  "postImportLinks": {
    "ignoreCase": false,
    "ignoreDiacritic": false,
    "matchEmpty": false,
    "method": "BASIC",
    "negateMatches": false,
    "partial": false,
    "pattern": "string",
    "replaceAll": false,
    "trim": false
  },
  "postImportLinksKeep": false,
  "preImportConsumers": [],
  "recrawlableResolver": {
    "class": "GenericRecrawlableResolver",
    "minFrequencies": [],
    "sitemapSupport": "FIRST"
  },
  "referenceFilters": [],
  "robotsMetaProvider": {
    "class": "StandardRobotsMetaProvider",
    "headersPrefix": "string"
  },
  "robotsTxtProvider": {
    "class": "StandardRobotsTxtProvider",
    "noop": "string"
  },
  "sitemapLocator": {
    "class": "GenericSitemapLocator",
    "paths": [],
    "robotsTxtSitemapDisabled": false
  },
  "sitemapResolver": {
    "class": "GenericSitemapResolver",
    "lenient": false
  },
  "spoiledReferenceStrategizer": {
    "class": "GenericSpoiledReferenceStrategizer",
    "fallbackStrategy": "DELETE",
    "mappings": {}
  },
  "startReferences": [
    "string"
  ],
  "startReferencesAsync": false,
  "startReferencesFiles": [
    "string"
  ],
  "startReferencesProviders": [
    {}
  ],
  "startReferencesSitemaps": [
    "string"
  ],
  "stopOnExceptions": [
    "string"
  ],
  "urlNormalizers": [],
  "urlScopeResolver": {
    "class": "GenericUrlScopeResolver",
    "includeSubdomains": false,
    "stayOnDomain": false,
    "stayOnPort": false,
    "stayOnProtocol": false,
    "stayOnSitemap": false
  },
  "workDir": "string"
}

XML

<crawler>
  <canonicalLinkDetector>
    <class>GenericCanonicalLinkDetector</class>
  </canonicalLinkDetector>
  <changeDiscovery>CRAWLER_SCAN</changeDiscovery>
  <cluster>
    <adminDisabled>false</adminDisabled>
    <adminPort>0</adminPort>
    <clustered>false</clustered>
    <connector>
      <class>HazelcastClusterConnector</class>
    </connector>
  </cluster>
  <committers>
    <committer/>
  </committers>
  <deferredShutdownDuration>0</deferredShutdownDuration>
  <delayResolver>
    <class>GenericDelayResolver</class>
    <defaultDelay>0</defaultDelay>
    <ignoreRobotsCrawlDelay>false</ignoreRobotsCrawlDelay>
    <scope>CRAWLER</scope>
  </delayResolver>
  <documentChecksummer>
    <class>Md5DocumentChecksummer</class>
    <combineFieldsAndContent>false</combineFieldsAndContent>
    <fieldMatcher/>
    <keep>false</keep>
    <onSet>APPEND</onSet>
    <toField>string</toField>
  </documentChecksummer>
  <documentDeduplicate>false</documentDeduplicate>
  <documentFetchSupport>DISABLED</documentFetchSupport>
  <documentFilters>
    <documentFilter/>
  </documentFilters>
  <eventListeners>
    <eventListener/>
  </eventListeners>
  <fetchers>
    <fetcher/>
  </fetchers>
  <fetchersMaxRetries>0</fetchersMaxRetries>
  <fetchersRetryDelay>0</fetchersRetryDelay>
  <id>string</id>
  <idleTimeout>0</idleTimeout>
  <importer>
    <handlers>
      <handler/>
    </handlers>
    <maxMemoryInstance>0</maxMemoryInstance>
    <maxMemoryPool>0</maxMemoryPool>
    <responseProcessors>
      <responseProcessor/>
    </responseProcessors>
    <tempDir>string</tempDir>
  </importer>
  <keepReferencedLinks>
    <keepReferencedLink>INSCOPE</keepReferencedLink>
  </keepReferencedLinks>
  <linkExtractors>
    <linkExtractor/>
  </linkExtractors>
  <maxCrawlDuration>0</maxCrawlDuration>
  <maxDepth>0</maxDepth>
  <maxDocuments>0</maxDocuments>
  <maxQueueBatchSize>0</maxQueueBatchSize>
  <maxStreamCachePoolSize>0</maxStreamCachePoolSize>
  <maxStreamCacheSize>0</maxStreamCacheSize>
  <metadataChecksummer>
    <class>LastModifiedMetadataChecksummer</class>
    <keep>false</keep>
    <onSet>APPEND</onSet>
    <toField>string</toField>
  </metadataChecksummer>
  <metadataDeduplicate>false</metadataDeduplicate>
  <metadataFetchSupport>DISABLED</metadataFetchSupport>
  <metadataFilters>
    <metadataFilter/>
  </metadataFilters>
  <minProgressLoggingInterval>0</minProgressLoggingInterval>
  <numThreads>0</numThreads>
  <orphansStrategy>PROCESS</orphansStrategy>
  <postImportConsumers>
    <postImportConsumer/>
  </postImportConsumers>
  <postImportLinks>
    <ignoreCase>false</ignoreCase>
    <ignoreDiacritic>false</ignoreDiacritic>
    <matchEmpty>false</matchEmpty>
    <method>BASIC</method>
    <negateMatches>false</negateMatches>
    <partial>false</partial>
    <pattern>string</pattern>
    <replaceAll>false</replaceAll>
    <trim>false</trim>
  </postImportLinks>
  <postImportLinksKeep>false</postImportLinksKeep>
  <preImportConsumers>
    <preImportConsumer/>
  </preImportConsumers>
  <recrawlableResolver>
    <class>GenericRecrawlableResolver</class>
    <sitemapSupport>FIRST</sitemapSupport>
  </recrawlableResolver>
  <referenceFilters>
    <referenceFilter/>
  </referenceFilters>
  <robotsMetaProvider>
    <class>StandardRobotsMetaProvider</class>
    <headersPrefix>string</headersPrefix>
  </robotsMetaProvider>
  <robotsTxtProvider>
    <class>StandardRobotsTxtProvider</class>
    <noop>string</noop>
  </robotsTxtProvider>
  <sitemapLocator>
    <class>GenericSitemapLocator</class>
    <robotsTxtSitemapDisabled>false</robotsTxtSitemapDisabled>
  </sitemapLocator>
  <sitemapResolver>
    <class>GenericSitemapResolver</class>
    <lenient>false</lenient>
  </sitemapResolver>
  <spoiledReferenceStrategizer>
    <class>GenericSpoiledReferenceStrategizer</class>
    <fallbackStrategy>DELETE</fallbackStrategy>
    <mappings/>
  </spoiledReferenceStrategizer>
  <startReferences>
    <ref>string</ref>
  </startReferences>
  <startReferencesAsync>false</startReferencesAsync>
  <startReferencesFiles>
    <file>string</file>
  </startReferencesFiles>
  <startReferencesProviders>
    <provider/>
  </startReferencesProviders>
  <startReferencesSitemaps>
    <sitemap>string</sitemap>
  </startReferencesSitemaps>
  <stopOnExceptions>
    <stopOnException>string</stopOnException>
  </stopOnExceptions>
  <urlNormalizers>
    <urlNormalizer/>
  </urlNormalizers>
  <urlScopeResolver>
    <class>GenericUrlScopeResolver</class>
    <includeSubdomains>false</includeSubdomains>
    <stayOnDomain>false</stayOnDomain>
    <stayOnPort>false</stayOnPort>
    <stayOnProtocol>false</stayOnProtocol>
    <stayOnSitemap>false</stayOnSitemap>
  </urlScopeResolver>
  <workDir>string</workDir>
</crawler>

Key Types

• WebCrawlerConfig
• CrawlerConfig
• Fetcher
• DocumentFilter