Spark 中文分词

Spark 中文分词

一、导入需要的分词包

import org.ansj.domain.Termimport org.ansj.recognition.impl.StopRecognitionimport org.ansj.splitWord.analysis.ToAnalysis

二、停用词过滤

def filter(stopWords: Array[String]): StopRecognition = {    // add stop words     val filter = new StopRecognition    filter.insertStopNatures("w") // filter punctuation    filter.insertStopNatures("m") // filter m pattern    filter.insertStopNatures("null") // filter null    filter.insertStopNatures("
") // filter 
    filter.insertStopRegexes("^[a-zA-Z]{1,}") //filter English alphabet    filter.insertStopRegexes("^[0-9]+") //filter number    filter.insertStopRegexes("[^a-zA-Z0-9\\u4e00-\\u9fa5]+")    filter.insertStopRegexes("\t")    for (x <- stopWords) {      filter.insertStopWords(x)    }    filter  }

三、分词

def getWords(text: String, filter: StopRecognition): ArrayBuffer[String] = {    val words = new mutable.ArrayBuffer[String]()    val terms: java.util.List[Term] =  ToAnalysis.parse(text).recognition(filter).getTerms    for (i <- 0 until terms.size()) {        val word = terms.get(i).getName        if (word.length >= MIN_WORD_LENGTH) {            words += word        }    }    words}