ucdavis-config.xml

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE xml>
<!--
   Copyright 2010-2017 Norconex Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

   	http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
-->

<httpcollector id="UCDAVIS HTTP Collector">
 <!-- Variables: Optionally define variables in this configuration file
   	using the "set" directive, or by using a file of the same name
   	but with the filterExtension ".variables" or ".properties".  Refer
   	to site documentation to find out what each filterExtension does.
   	Finally, once can pass an optional properties file when starting the
   	crawler.  The following is good practice to reference frequently
   	used classes in a shorter way.
   	-->
   	<!-- URL to index -->
   	#set($site_index1    = "https://health.ucdavis.edu")
   	<!-- URL to index second site -->
   	#set($site_index2    = "https://hr.ucdavis.edu")
   	<!-- Collection created in solar where to store documents -->
   	#set($collection_solr    ="http://hspawebsrch01:8983/solr/internet")
	<!-- Collection created in solar where to store documents -->
   	#set($collection_solr2    ="http://hspawebsrch01:8983/solr/intranet")
<!-- Deep of the crawler -->
	   #set($site_deep	="30")





  <!-- Decide where to store generated files. -->
  <progressDir>./internet-output/minimum/progress</progressDir>
  <logsDir>./internet-output/minimum/logs</logsDir>

  <crawlers>
    <crawler id="Norconex Minimum Test Page">

      <!-- Requires at least one start URL (or urlsFile). 
           Optionally limit crawling to same protocol/domain/port as 
           start URLs. -->
      <startURLs stayOnDomain="true" stayOnPort="true" stayOnProtocol="false">
        <url>$site_index1</url>
		<url>$site_index2</url>
		
      </startURLs>
      <numThreads>30</numThreads>
	  <!-- Optional URL normalization feature. The class must implement
         com.norconex.collector.http.url.IURLNormalizer, 
         like the following class does. This is in order to avoid duplicates pages
      -->
	  <urlNormalizer class="com.norconex.collector.http.url.impl.GenericURLNormalizer">
            <normalizations>lowerCaseSchemeHost, upperCaseEscapeSequence, decodeUnreservedCharacters, removeDefaultPort, removeFragment, removeDotSegments, addTrailingSlash, removeDuplicateSlashes, removeSessionIds, upperCaseEscapeSequence</normalizations> 
			<replacements>
				<replace>
					<match>&amp;view=print</match>
					<replacement>&amp;view=shtml</replacement>
				</replace>
			</replacements>
      </urlNormalizer>
	  <!-- Optionally filter URL BEFORE any download. Classes must implement 
         com.norconex.collector.core.filter.IReferenceFilter, 
         like the following examples.
         -->
    <referenceFilters>
        <!-- Exclude images, CSS and JS files -->
      <filter class="com.norconex.collector.core.filter.impl.ExtensionReferenceFilter" onMatch="exclude" caseSensitive="false">jpg,jpeg,gif,png,ico,css,js,svg</filter>
      <!-- Exclude dynamic pages containing a query (?) -->
      <filter class="com.norconex.collector.core.filter.impl.RegexReferenceFilter" onMatch="exclude" caseSensitive="true">.+\?.*</filter>
	  <!-- Exclude pages on explicit request -->
      <filter class="com.norconex.collector.core.filter.impl.RegexReferenceFilter" onMatch="exclude" caseSensitive="true">
	  .*health.ucdavis.edu\/welcome\/features\/2015-2016\/12\/20151230_Snoopy-visit.html</filter>
	  <filter class="com.norconex.collector.core.filter.impl.RegexReferenceFilter" onMatch="exclude" caseSensitive="true">
	  .*health.ucdavis.edu\/welcome\/features\/2013-2014\/06\/20140626_July4_safety.html</filter>
    </referenceFilters>
	  
      <!-- === Recommendations: ============================================ -->

      <!-- Specify a crawler default directory where to generate files. -->
      <workDir>./internet-output/minimum</workDir>

      <!-- Put a maximum depth to avoid infinite crawling (e.g. calendars). -->
      <maxDepth>$site_deep</maxDepth>

      <!-- We know we don't want to crawl the entire site, so ignore sitemap. -->
      <sitemapResolverFactory ignore="true" />
	  
	  <!-- Fetch a URL HTTP Headers.  Classes must implement
         com.norconex.collector.http.fetch.IHttpMetadataFetcher.  
         The following is a simple implementation.
         -->
      <!--metadataFetcher class="com.norconex.collector.http.fetch.impl.GenericMetadataFetcher">
		<validStatusCodes>200</validStatusCodes>
	  </metadataFetcher-->

      <!-- Optionally filter AFTER download of HTTP headers.  Classes must 
         implement com.norconex.collector.core.filter.IMetadataFilter.  
         -->
      <metadataFilters>
		<!-- Do not index content-type of CSS or JavaScript -->
		<filter class="com.norconex.collector.core.filter.impl.RegexMetadataFilter" onMatch="exclude" caseSensitive="false" field="Content-Type">.*css.*|.*javascript.*|.*svg.*|.*zip.*|.*ZIP.*</filter>
	  </metadataFilters>        

      <!-- Be as nice as you can to sites you crawl. -->
      <delay default="800" />
      
      <!-- Document importing -->
      <importer>
	  
	<documentFetcher detectContentType="true" detectCharset="true"/>
	<preParseHandlers>
                <!-- If your target repository does not support arbitrary fields,
           make sure you only keep the fields you need. -->
		   <transformer class="com.norconex.importer.handler.transformer.impl.StripBetweenTransformer" inclusive="false">
		   <restrictTo field="document.contentType">text/html</restrictTo>
             <!-- new tagg added -->
			 <stripBetween>
              <start><![CDATA[<headerremove>]]></start>
              <end><![CDATA[</headerremove>]]></end>
            </stripBetween>
			 <stripBetween>
              <start><![CDATA[<div class="navtop">]]></start>
              <end><![CDATA[<div class="container" id="header">]]></end>
            </stripBetween>
			<stripBetween>
              <start><![CDATA[<div class="navskip">]]></start>
              <end><![CDATA[<!-- end nav -->]]></end>
            </stripBetween>
			<stripBetween>
              <start><![CDATA[<div class="navskip">]]></start>
              <end><![CDATA[<!-- End main search -->]]></end>
            </stripBetween>
			  <stripBetween>
              <start><![CDATA[<div class="navskip">]]></start>
              <end><![CDATA[<div id="breadcrumb">]]></end>
            </stripBetween>
			<stripBetween>
              <start><![CDATA[<!------------------ Start primary menu navigation for Top Nav sites ------------------>]]></start>
              <end><![CDATA[<!------------------ Start main content ------------------>]]></end>
            </stripBetween>
          </transformer> 
		         <tagger class="com.norconex.importer.handler.tagger.impl.DocumentLengthTagger" field="document.size.preparse" >
				<restrictTo field="document.contentType">text/html</restrictTo>
				</tagger>
                <tagger class="com.norconex.importer.handler.tagger.impl.RenameTagger">
				    <rename fromField="document.contentEncoding" toField="content_encoding" overwrite="true"/>
                    <rename fromField="document.contentType" toField="content_type" overwrite="true"/>
                </tagger> 
				<tagger class="com.norconex.importer.handler.tagger.impl.RenameTagger">
				    <restrictTo field="document.contentType">text/html</restrictTo>
				    <rename fromField="text" toField="text_content" overwrite="true"/>
                </tagger> 
                <tagger class="com.norconex.importer.handler.tagger.impl.DebugTagger" logLevel="DEBUG"/>
		
      
    </preParseHandlers>
	  
	  
    <postParseHandlers>
		<!--transformer class="com.norconex.importer.handler.transformer.impl.ReduceConsecutivesTransformer" caseSensitive="true">
          <reduce>\s</reduce>
          <reduce>\n</reduce>
          <reduce>\r</reduce>
          <reduce>\t</reduce>
          <reduce>\n\r</reduce>
          <reduce>\r\n</reduce>
          <reduce>\s\n</reduce>
          <reduce>\s\r</reduce>
          <reduce>\s\r\n</reduce>
          <reduce>\s\n\r</reduce>
        </transformer-->
		
		<tagger class="com.norconex.importer.handler.tagger.impl.DocumentLengthTagger" field="document.size.postparse" />
        <!-- Reject small documents (<100 Bytes)-->
                <filter class="com.norconex.importer.handler.filter.impl.NumericMetadataFilter" onMatch="exclude" field="document.size.postparse" >
                <condition operator="lt" number="100" />
                </filter>
		
		<!-- Rename fields -->
        <tagger class="com.norconex.importer.handler.tagger.impl.RenameTagger">
           <rename fromField="document.size.postparse" toField="document_length" overwrite="true" />
        </tagger>
		
		<tagger class="com.norconex.importer.handler.tagger.impl.DeleteTagger">
            <fields>document.size.postparse</fields>
        </tagger>
		
		<tagger class="com.norconex.importer.handler.tagger.impl.DateFormatTagger"
          fromField="last-modified"
          toField="lastmodified" 
          toFormat="yyyyMMdd" >
           <fromFormat>yyyyMMdd</fromFormat>
        </tagger>
  
        <tagger class="com.norconex.importer.handler.tagger.impl.DateFormatTagger"
          fromField="last-modified"
          toField="solr_date" 
          toFormat="yyyy-MM-dd" >
           <fromFormat>yyyyMMdd</fromFormat>
        </tagger>
  
  
				<tagger class="com.norconex.importer.handler.tagger.impl.ReplaceTagger">
          <replace fromField="title" replaceAll="true" regex="true" caseSensitive="false">
              <fromValue>[ \t\r\n]{2,}</fromValue>
              <toValue xml:space="preserve"> </toValue>
          </replace>
      </tagger>
	   
	    <tagger class="com.norconex.importer.handler.tagger.impl.ScriptTagger">
			<script><![CDATA[
				var objectnoexist=null;
				objectnoexist=metadata.getString('title');
				var str = "";
				if(typeof(objectnoexist)=='undefined' || objectnoexist == null ) {
				str=metadata.getString('document.reference');
				
				var n = str.lastIndexOf("/");
				var l = str.lastIndexOf(".");
				var ti = str.substring(n+1,l);
				metadata.setString('title',ti); 
				} 
				var cleanC = content;
				cleanC = cleanC + metadata.getString('title');
				metadata.setString('text_clean',cleanC.replace(/[\n\r\t]/g,''));
				var source = metadata.getString('document.reference');
				var find = "/";
				var lv1 = "NA";
				var lv2 = "NA";
				var result = [];
				for (i = 0; i < source.length; ++i) {
					if (source.substring(i, i + find.length) == find) {
						result.push(i);
					}
				}
				var le = result.length;
				var biaslevel =0;
		
		if( le > 2 ){
		if(le >  3 ){
			lv1=source.substring(result[2]+1,result[3]);
			biaslevel =1;
			if(source.substring(result[3]+1,result[4]).indexOf(".") !== -1 || result[4] == null ){
				biaslevel =1;				
			}else {
                lv2=source.substring(result[3]+1,result[4]);
				biaslevel =2;
			}
		}else {
			if(source.substring(result[2]+1).indexOf(".") !== -1 || result[3] == null){
				biaslevel=0;
				
			}else { 
			lv1=source.substring(result[2]+1);
			biaslevel=0;
			
			}
		}	  
	}
    metadata.setString('biaslevel',biaslevel);
	metadata.setString('level_1',lv1);
	metadata.setString('level_2',lv2);


				
			]]></script>
		</tagger>
		  
      
          <!-- If your target repository does not support arbitrary fields,
               make sure you only keep the fields you need. -->
          <tagger class="com.norconex.importer.handler.tagger.impl.KeepOnlyTagger">
            <fields>title,keywords,description,document.reference,text_content,content_encoding,content_type,copyright,author,document_length,lastmodified,text_clean,level_1,level_2,solr_date,biaslevel</fields>
          </tagger>
		  
        </postParseHandlers>
		
      </importer> 
      
      <!-- Decide what to do with your files by specifying a Committer. -->
	  <committer class="com.norconex.committer.core.impl.MultiCommitter">
		
		<committer class="com.norconex.committer.core.impl.FileSystemCommitter">
			<directory>./internet-output/minimum/crawledFiles</directory>
        </committer>
		<committer class="com.norconex.committer.solr.SolrCommitter">
			<solrURL>$collection_solr</solrURL>
			<sourceReferenceField keep="false">document.reference</sourceReferenceField>
			<targetReferenceField>id</targetReferenceField>
			<targetContentField>text_content</targetContentField>
			<commitBatchSize>100</commitBatchSize>
			<queueDir>./internet-output/optional/queue</queueDir>
			<queueSize>100</queueSize>
			<maxRetries>2</maxRetries>
			<maxRetryWait>5000</maxRetryWait>
		</committer>
		<committer class="com.norconex.committer.solr.SolrCommitter">
			<solrURL>$collection_solr2</solrURL>
			<sourceReferenceField keep="false">document.reference</sourceReferenceField>
			<targetReferenceField>id</targetReferenceField>
			<targetContentField>text_content</targetContentField>
			<commitBatchSize>100</commitBatchSize>
			<queueDir>./internet-output/optional2/queue</queueDir>
			<queueSize>100</queueSize>
			<maxRetries>2</maxRetries>
			<maxRetryWait>5000</maxRetryWait>
		</committer>
	  </committer>
    </crawler>
  </crawlers>

</httpcollector>