RMM分詞演算法類

複製程式碼

RMM分詞演算法類

//RMM分詞演算法
class SplitWord{
 var $TagDic = Array();
 var $RankDic = Array();
 var $ SourceStr = '';
 var $ResultStr = '';
 var $SplitChar = ' '; //分隔符號
 var $SplitLen = 4; //保留字長
 var $MaxLen = 7; //字典最大中文字，這裡的數值為位元組數組的最大索引
 var $MinLen = 3; //最小中文字，這裡的數值為位元組數組的最大索引

 function SplitWord(){
 $this->__construct();
 }

 function __construct(){
 //高階分詞，預先載入字典以提分詞高速度
 $dicfile = dirname(__FILE__)."/ppldic.csv";
 $fp = fopen($dicfile,'r'); //讀取字庫中的字
 while($line = fgets ($fp,256)){
 $ws = explode(' ',$line); //將字庫中的字分割
 $this->TagDic[$ws[0]] = $ws[1];
 $this->RankDic[strlen($ws[0])][$ws[0]] = $ws[2];
 }
 fclose($fp) ; //關閉詞庫檔案
 }

 //析放資源
 function Clear(){
 @fclose($this->QuickDic);
 }

 //設定來源字串
 function SetSource($str){
 $this->SourceStr = $this->UpdateStr($str);
 $this->ResultStr = "";
 }

 //檢查字串是否不存在中文
 function NotGBK($str)
 {
 if($str=="") return "";
 if( ord($str[0])>0x80 ) return false;
 else return true;
 }

 //RMM分詞演算法
 function SplitRMM($str="" ){
 if($str!="") $this->SetSource($str);
 if($this->SourceStr=="") return "";
 $this-> SourceStr = $this->UpdateStr($this->SourceStr);
 $spwords = explode(" ",$this->SourceStr);
 $spLen = count($spwords);
 $spc = $this->SplitChar;
 for($i=($spLen-1);$i>=0;$i--){
 if($spwords[$i]=="") continue;
 if($this->NotGBK($spwords[$i])){
 if(preg_match("/[^0-9. -]/",$spwords[$i]))
 { $this->ResultStr = $spwords[$i].$spc.$this->ResultStr; }
 else
 {
 $nextword = "";
 @$nextword = substr($this->ResultStr,0,strpos($this->ResultStr,""));
 }
 }
 else
 {
 $c = $spwords[$ i][0].$spwords[$i][1];
 $n = hexdec(bin2hex($c));
 if(strlen($spwords[$i]) SplitLen)
 {
 }
 else
 {
 $this->ResultStr = $this->RunRMM($spwords[$i]).$spc.$this->ResultStr ;
 }
 }
 }
 return $this->ResultStr;
 }
 //對全中文字串進行逆向匹配方式分解
 function RunRMM($str ){
 $spc = $this->SplitChar;
 $spLen = strlen($str);
 $rsStr = "";
 $okWord = "";
 $tmpWord = "";
 $WordArray = Array();
 //逆向字典匹配
 for($i=($spLen-1);$i>=0;){
 //當i達到最小可能字的時候
 if($iMinLen){
 if($i==1){
 $WordArray[] = substr($str,0, 2);
 }else
 {
 $w = substr($str,0,$this->MinLen 1);
 if($this->IsWord($w)){
 $WordArray[] = $w;
 }else{
 $WordArray[] = substr($str,2,2);
 $WordArray[] = substr($str,0,2 );
 }
 }
 $i = -1; break;
 }
 //分析在最小字以上時的情況
 if($i>=$this- >MaxLen) $maxPos = $this->MaxLen;
 else $maxPos = $i;
 $isMatch = false;
 for($j=$maxPos;$j>=0;$j= $j-2){
 $w = substr($str,$i-$j,$j 1);
 if($this->IsWord($w)){
 $WordArray[ ] = $w;
 $i = $i-$j-1;
 $isMatch = true;
 break;
 }
 }
 }
 $rsStr = $this->otherword($WordArray);
 return $rsStr;
 }

function otherword($WordArray){
 $wlen = count($WordArray)-1; //計算陣列的元素個數
 $rsStr = ""; //初始化變數
 $spc = $this->SplitChar;
 for($i=$wlen;$i>=0;$i --)
 {
 $rsStr .= $spc.$WordArray[$i]."、"; //將陣列為頓號拆分
 }
 //傳回本段分詞結果
 $rsStr = preg_replace("/^".$spc."/","、",$rsStr);
 return $rsStr;
 }

 //判斷字典裡是否存在某個字
 function IsWord($okWord){
 $slen = strlen($okWord);
 if($slen > $this-> MaxLen) return false;
 else return isset($this->RankDic[$slen][$okWord]);
 }

 //整理字串（對標點符號，中英文混排等初步處理）
 function UpdateStr($str){
 $spc = $this->SplitChar;
 $slen = strlen($str);
 if($slen==0) return '';
 $okstr = '';
 $prechar = 0; // 0-空白1-英文2-中文3-符號
 for($i=0;$i if(ord($str[$i])  //英文的空白符號
 if(ord($str[$i])  if($prechar!=0&&$str[$i]!="r"&&$str[$i]!="n") $okstr .= $spc;
 $prechar=0;
 continue;
 }else if(preg_match("/[^0-9a-zA-Z@.%#:\&_-]/",$str[$i])){
 if( $prechar==0){ $okstr .= $str[$i]; $prechar=3;}
 else{ $okstr .= $spc.$str[$i]; $prechar=3;}
 }else{
 if($prechar==2||$prechar==3)
 { $okstr .= $spc.$str[$i]; $prechar=1;}
 else
 {
 if(preg_match("/@#%:/",$str[$i])){ $okstr .= $str[$i]; $prechar=3; }
 else { $okstr .= $str[$i]; $prechar=1; }
 }
 }
 }
 else{
 //如果上一個字元為非中文和非空格，則加一個空格
 if($prechar!=0 && $prechar!=2) $okstr .= $spc;
 //如果中文字元
 if(isset($str[$i 1])){
 $c = $str[$i].$str[$i 1];

 $n = hexdec(bin2hex($c));
 if($ n 0xAA40){
 if($prechar!=0) $okstr .= $spc.$c;
 else $okstr .= $c;
 $prechar = 3;
 }
 else{
 $okstr .= $c;
 $prechar = 2;
 }
 $i ;
 }
 }
 }
 return $okstr;
 }
}

// 呼叫
$split=new SplitWord();

echo $split->SplitRMM("php搜尋技術");

// 注意ppldic.csv 字典的格式是詞語空格編號n