ILSP Focused Crawler

<pre><code class="xml"><?xml version="1.0" encoding="UTF-8"?>

<configuration>

	<agent>

		<email>yourmail@mail.com</email>

		<web_address>www.youraddress.com</web_address>

	</agent>

	<classifier>

		<min_content_terms>

			<value>2</value>

			<description>Minimum number of terms that must exist in clean

				content of each web page in order to be stored.</description>

		</min_content_terms>

		<min_unique_content_terms>

			<value>2</value>

			<description>Minimum unique terms that must exist in clean content</description>

		</min_unique_content_terms>

		<max_depth>

			<value>10</value>

			<description>Maximum depth to crawl before abandoning a specific path. Depth

			is increased every time a link is extracted from a non-relevant web page.</description>

		</max_depth>

	</classifier>

	<fetcher>

		<fetch_buffer_size>

			<description>Max number of urls to fetch per run</description>

			<value>512</value>

		</fetch_buffer_size>

		<socket_timeout>

			<value>10000</value>

			<description>Socket timeout in milliseconds(per URL)</description>

		</socket_timeout>

		<connection_timeout>

			<value>10000</value>

			<description>Connection timeout in milliseconds(per URL)</description>

		</connection_timeout>

		<max_retry_count>

			<value>2</value>

			<description>Max number of attempts to fetch a Web page before giving up</description>

		</max_retry_count>

		<min_response_rate>

			<value>0</value>

			<description>Min bytes-per-seconds for fetching a web page</description>

		</min_response_rate>

		<valid_mime_types>

			<mime_type value="text/html" />

			<mime_type value="text/plain" />

			<mime_type value="application/xhtml+xml" />

			<description>Accepted mime types</description>

		</valid_mime_types>

		<crawl_delay>

			<value>1500</value>

			<description>delay in milliseconds between requests</description>

		</crawl_delay>

		<max_content_size>

			<value>531072</value>

			<description>Max content size (bytes) for downloading a web page</description>

		</max_content_size>

		<max_requests_per_run>

			<value>512</value>

			<description>Max fetch set size per run (Sets are made by URLs from the same host)</description>

		</max_requests_per_run>

		<max_requests_per_host_per_run>

			<value>512</value>

			<description>Max URLs from a specific host per run</description>

		</max_requests_per_host_per_run>

		<max_connections_per_host>

			<value>32</value>

			<description>Max number of fetching threads for each host</description>

		</max_connections_per_host>		

		<max_fetched_per_host>

			<value>500000</value>

			<description>Max web pages to fetch per host</description>

		</max_fetched_per_host>

		<max_redirects>

			<value>5</value>

			<descriptions>Max number of redirects</descriptions>

		</max_redirects>

		<request_timeout>

			<value>600000</value>

			<description>Max time to wait for Fetcher to get all URLs in a run</description>

		</request_timeout>

	</fetcher>	

</configuration>

</code></pre>