CODEWITHSUNDEEP
mysqljdbcsolr
CuReSoft
CuReSoft

Reputation: 3

DIH Mysql to Solr import problems

I have trouble Indexing Documents from Mysql to Solr.

My Config:
data-config.xml

<dataConfig>
<dataSource type="JdbcDataSource"
driver="com.mysql.jdbc.Driver"
url="jdbc:mysql://xxx?characterEncoding=utf8"
user="xxx"
password="xxx"/>

<document name="articledata">
    <entity name="outer"
transformer="HTMLStripTransformer"
query="SELECT
id,kundenid,LOWER(title) as title,LOWER(content) as content,
DATE_FORMAT(cr,'%Y-%m-%dT%H:%i:%sZ') as cr,
lang
FROM articledata
WHERE 
DATE(cr) BETWEEN DATE(DATE_SUB(now(),INTERVAL 3 DAY)) AND DATE(now())
AND content IS NOT NULL
ORDER BY DATE(cr) DESC">



      <field column="id" name="id" />
         <field column="kundenid" name="kundenid" />
            <field column="title" name="title" />
            <field column="content" name="content" stripHTML="true" />
         <field column="cr" name="cr" />
         <field column="lang" name="lang" />
        </entity>
    </document>
</dataConfig>

schema.xml

    <?xml version="1.0" ?>
    <schema name="articledata core zero" version="1.1">
      <types>
    <fieldtype name="int"  class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0"/>   
    <fieldtype name="string"  class="solr.StrField" sortMissingLast="true" omitNorms="true"/>   
    <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/>
    <fieldType name="dt" class="solr.TrieDateField" precisionStep="0" positionIncrementGap="0" />

    <fieldType name="textgen" class="solr.TextField" positionIncrementGap="100">
    <analyzer type="index">
        <tokenizer class="solr.WhitespaceTokenizerFactory"/>
        <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
        <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="0"/>
        <filter class="solr.LowerCaseFilterFactory"/>

  </analyzer>
  <analyzer type="query">
    <tokenizer class="solr.WhitespaceTokenizerFactory"/>
    <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
    <filter class="solr.StopFilterFactory"
            ignoreCase="true"
            words="stopwords.txt"
            enablePositionIncrements="true"
            />
    <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="0"/>
    <filter class="solr.LowerCaseFilterFactory"/>
  </analyzer>
</fieldType>

  </types>

 <fields>   
  <field name="id"        type="int"    indexed="true"   stored="true"   required="true"/>
  <field name="kundenid"  type="int"    indexed="true"   stored="true"   required="true"/>
  <field name="title"     type="string"   indexed="true"   stored="true"   /> 
  <field name="content"   type="textgen"    indexed="true"  stored="true"  />
  <field name="cr"       type="dt"   indexed="true"  stored="true" /> 
  <field name="lang"      type="string"    indexed="true"  stored="true" /> 
  <field name="_version_" type="long"      indexed="true"  stored="true"/>
 </fields>

 <uniqueKey>id</uniqueKey>
 <defaultSearchField>content</defaultSearchField>

 <solrQueryParser defaultOperator="AND"/>

</schema>

With this configuration i get Documents like this: 

"docs": [
      {
        "content": "[B@7f017c71",
        "id": 20785923,
        "cr": "2014-07-24T08:01:58Z",
        "title": "general motors entdeckt neue mängel bei hunderttausenden wagen - news - alle aktuellen news - dpa-afx - general motors dl-,01 - onvista",
        "kundenid": 1,
        "_version_": 1474502436614832000
      },

The title gets indexed properly

The content shows up as bullshit chars and is not searchable.

Any ideas how i can fix that?

Thanks in advance.

Upvotes: 0

Views: 141

Answers (1)

Jayesh Bhoyar
Jayesh Bhoyar

Reputation: 727

I suspect that your content field in DB must be text/BLOB and not varchar (as title must be varchar). Hence you are able to index title correctly and content is not getting indexed correctly.

If you are having a BLOB of data or text data in DB then it would possibly be useful to use a field type that has the right set of tokenizers, analyzers and filters.

For example, adding a StandardTokenizerFactory keeps tokens to a meaningful value set.

An example of the fieldtype definition:

<fieldType name="text_general" class="solr.TextField" positionIncrementGap="100" omitNorms="true">
  <analyzer type="index">
    <tokenizer class="solr.StandardTokenizerFactory" />
    <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
    <filter class="solr.LowerCaseFilterFactory" />
  </analyzer>
</fieldtype>

If the problem still persist then following information will help you in investigating this issue:

1) Can check what values you get from MYSQL when you run query: SELECT id,kundenid,LOWER(title) as title,LOWER(content) as content, DATE_FORMAT(cr,'%Y-%m-%dT%H:%i:%sZ') as cr,lang FROM articledata WHERE DATE(cr) BETWEEN DATE(DATE_SUB(now(),INTERVAL 3 DAY)) AND DATE(now()) AND content IS NOT NULL ORDER BY DATE(cr) DESC"

2) Try to change textgen to string.

3) Try Removing stripHTML="true" from content

Hope this will help you in resolving your issue or at least help you in investigating further.

Upvotes: 1

Related Questions