Lucene分析器的实现。
Lucene分词器Tokenizer，它的继承子类的实现。
Tokenizer类的继承关系，如图所示：
ChineseTokenizer类实现中文分词
中文分词在Lucene中的处理很简单，就是单个字分。它的实现类为ChineseTokenizer，在包org.apache.lucene.analysis.cn中，源代码如下：
package org.apache.lucene.analysis.cn;
import java.io.Reader;import org.apache.lucene.analysis.*;
public final class ChineseTokenizer extends Tokenizer {
public ChineseTokenizer(Reader in) { input = in; }
private int offset = 0, bufferIndex=0, dataLen=0; private final static int MAX_WORD_LEN = 255; private final static int IO_BUFFER_SIZE = 1024; private final char[] buffer = new char[MAX_WORD_LEN]; private final char[] ioBuffer = new char[IO_BUFFER_SIZE];
private int length; private int start;
private final void push(char c) { // 对待分词的文本进行预处理，输入到缓冲区buffer中
if (length == 0) start = offset-1; // 根据词条长度，设置起始位置索引 buffer[length++] = Character.toLowerCase(c); // 预处理：将中文Unicode码转化成小写
}
private final Token flush() { // 根据缓冲区预处理后的文本，构造词条
if (length>0) { return new Token(new String(buffer, 0, length), start, start+length); } else return null; }
public final Token next() throws java.io.IOException { // 返回下一个词条
length = 0; start = offset;
while (true) {
final char c; offset++;
if (bufferIndex >= dataLen) { // 当缓冲区没有溢出 dataLen = input.read(ioBuffer); bufferIndex = 0; }
if (dataLen == -1) return flush(); else c = ioBuffer[bufferIndex++];
switch(Character.getType(c)) {
case Character.DECIMAL_DIGIT_NUMBER: case Character.LOWERCASE_LETTER: case Character.UPPERCASE_LETTER: push(c); if (length == MAX_WORD_LEN) return flush(); break;
case Character.OTHER_LETTER: if (length>0) { bufferIndex--; offset--; return flush(); } push(c); return flush();
default: if (length>0) return flush(); break; } }
}}
这里，还提及到一个CJKTokenizer分词类，它处理分词的时候，比ChineseTokenizer分词处理要好一点，但是也存在弊病，源代码给了一个例子，如下：
如果一个中文词汇C1C2C3C4被索引，使用ChineseTokenizer分词，返回的词条(Token)为：C1，C2，C3，C4；使用CJKTokenizer进行分词，则返回的词条(Token)为：C1C2，C2C3，C3C4。
问题在于：当检索关键字为C1，C1C2，C1C3，C4C2，C1C2C3……的时候，ChineseTokenizer可以对其实现分词，而CJKTokenizer就不能实现分词了。
CJKTokenizer类实现中文分词
CJKTokenizer类的源代码如下所示：
package org.apache.lucene.analysis.cjk;
import org.apache.lucene.analysis.Token;import org.apache.lucene.analysis.Tokenizer;
import java.io.Reader;
public final class CJKTokenizer extends Tokenizer { /** Max word length */ private static final int MAX_WORD_LEN = 255;
/** buffer size: */ private static final int IO_BUFFER_SIZE = 256;
/** word offset, used to imply which character(in ) is parsed */ private int offset = 0;
/** the index used only for ioBuffer */ private int bufferIndex = 0;
/** data length */ private int dataLen = 0;
/** * 字符缓冲区，存储那些经过处理后返回的词条 */ private final char[] buffer = new char[MAX_WORD_LEN];
/** * I/O buffer, used to store the content of the input(one of the <br> * members of Tokenizer) */ private final char[] ioBuffer = new char[IO_BUFFER_SIZE];
/** word type: single=>ASCII double=>non-ASCII word=>default */ private String tokenType = "word";
private boolean preIsTokened = false;
public CJKTokenizer(Reader in) { input = in; }
public final Token next() throws java.io.IOException { int length = 0;
/** the position used to create Token */ int start = offset;
while (true) { /** current charactor */ char c;
/** unicode block of current charactor for detail */ Character.UnicodeBlock ub;
offset++;
if (bufferIndex >= dataLen) { dataLen = input.read(ioBuffer); bufferIndex = 0; }
if (dataLen == -1) { if (length > 0) { if (preIsTokened == true) { length = 0; preIsTokened = false; }
break; } else { return null; } } else { //get current character c = ioBuffer[bufferIndex++];
//get the UnicodeBlock of the current character ub = Character.UnicodeBlock.of(c); }
//if the current character is ASCII or Extend ASCII if ((ub == Character.UnicodeBlock.BASIC_LATIN) || (ub == Character.UnicodeBlock.HALFWIDTH_AND_FULLWIDTH_FORMS) ) { if (ub == Character.UnicodeBlock.HALFWIDTH_AND_FULLWIDTH_FORMS) { /** convert HALFWIDTH_AND_FULLWIDTH_FORMS to BASIC_LATIN */ int i = (int) c; i = i - 65248; c = (char) i; }
// if the current character is a letter or "_" "+" "#" if (Character.isLetterOrDigit(c) || ((c == '_') || (c == '+') || (c == '#')) ) { if (length == 0) { // "javaC1C2C3C4linux" <br> // ^--: the current character begin to token the ASCII // letter start = offset - 1; } else if (tokenType == "double") { offset--; bufferIndex--; tokenType = "single";
if (preIsTokened == true) { // there is only one non-ASCII has been stored length = 0; preIsTokened = false;
break; } else { break; } }
// store the LowerCase(c) in the buffer buffer[length++] = Character.toLowerCase(c); tokenType = "single";
// break the procedure if buffer overflowed! if (length == MAX_WORD_LEN) { break; } } else if (length > 0) { if (preIsTokened == true) { length = 0; preIsTokened = false; } else { break; } } } else { // non-ASCII letter, eg."C1C2C3C4" if (Character.isLetter(c)) { if (length == 0) { start = offset - 1; buffer[length++] = c; tokenType = "double"; } else { if (tokenType == "single") { offset--; bufferIndex--;
//return the previous ASCII characters break; } else { buffer[length++] = c; tokenType = "double";
if (length == 2) { offset--; bufferIndex--; preIsTokened = true;
break; } } } } else if (length > 0) { if (preIsTokened == true) { // empty the buffer length = 0; preIsTokened = false; } else { break; } } } }
return new Token(new String(buffer, 0, length), start, start + length, tokenType ); }}
