DokuWiki-2009-12-25c その1

ことのおこり

これがそろそろうざったいのでDokuWikiをバージョンアップしようかなと思い立つ。

現在使用中はdokuwiki-2009-02-14b。あいだにdokuwiki-2009-12-02ってのがあったらしい。すんなりいけばいいな

indexer.php（2009-02-14mecab対応）

2009-02-14のときに手を入れた部分（02-14bは数ファイルのみの更新だった）

--- indexer.php.org 2009-02-14 21:13:24.000000000 +0900 +++ indexer.php.mod 2009-04-23 16:17:13.000000000 +0900 @@ -45,6 +45,8 @@ ']?'); define('IDX_ASIAN', '(?:'.IDX_ASIAN1.'|'.IDX_ASIAN2.'|'.IDX_ASIAN3.')'); +define('PRE_TOKENIZER', '/usr/bin/mecab -O wakati'); + /** * Measure the length of a string. * Differs from strlen in handling of asian characters. @@ -52,11 +54,16 @@ * @author Tom N Harris <tnharris@whoopdedo.org> */ function wordlen($w){ - $l = strlen($w); + //$l = strlen($w); + $l = utf8_strlen($w); + + /* // If left alone, all chinese "words" will get put into w3.idx // So the "length" of a "word" is faked if(preg_match('/'.IDX_ASIAN2.'/u',$w)) $l += ord($w) - 0xE1; // Lead bytes from 0xE2-0xEF + */ + return $l; } @@ -220,6 +227,28 @@ list($page,$body) = $data; + if(function_exists(proc_open) && defined('PRE_TOKENIZER')) { + $dspec = array( + 0 => array("pipe", "r"), + 1 => array("pipe", "w"), + 2 => array("file", "/dev/null", "w") + ); + $process = proc_open(PRE_TOKENIZER, $dspec, $pipes); + if(is_resource($process)) { + stream_set_blocking($pipes[0], FALSE); + stream_set_blocking($pipes[1], FALSE); + fwrite($pipes[0], $body . "\n"); + fclose($pipes[0]); + + $body = ''; + while(!feof($pipes[1])) { + $body .= fgets($pipes[1], 32768); + } + fclose($pipes[1]); + proc_close($process); + } + } + $body = strtr($body, "\r\n\t", ' '); $tokens = explode(' ', $body); $tokens = array_count_values($tokens); // count the frequency of each token @@ -489,7 +518,8 @@ $wild |= 2; $wlen -= 1; } - if ($wlen < IDX_MINWORDLENGTH && $wild == 0 && !is_numeric($xword)) continue; + //if ($wlen < IDX_MINWORDLENGTH && $wild == 0 && !is_numeric($xword)) continue; + if (preg_match('/[^0-9A-Za-z]/u', $string) && $wlen < IDX_MINWORDLENGTH && $wild == 0 && !is_numeric($xword)) continue; if(!isset($tokens[$xword])){ $tokenlength[$wlen][] = $xword; } @@ -628,12 +658,36 @@ */ function idx_tokenizer($string,&$stopwords,$wc=false){ $words = array(); + + if(function_exists(proc_open) && defined('PRE_TOKENIZER')) { + $dspec = array( + 0 => array("pipe", "r"), + 1 => array("pipe", "w"), + 2 => array("file", "/dev/null", "w") + ); + $process = proc_open(PRE_TOKENIZER, $dspec, $pipes); + if(is_resource($process)) { + stream_set_blocking($pipes[0], FALSE); + stream_set_blocking($pipes[1], FALSE); + fwrite($pipes[0], $string . "\n"); + fclose($pipes[0]); + $string = ''; + while(!feof($pipes[1])) { + $string .= fgets($pipes[1], 32768); + } + fclose($pipes[1]); + proc_close($process); + } + } + $wc = ($wc) ? '' : $wc = '\*'; if(preg_match('/[^0-9A-Za-z]/u', $string)){ + /* // handle asian chars as single words (may fail on older PHP version) $asia = @preg_replace('/('.IDX_ASIAN.')/u',' \1 ',$string); if(!is_null($asia)) $string = $asia; //recover from regexp failure + */ $arr = explode(' ', utf8_stripspecials($string,' ','\._\-:'.$wc)); foreach ($arr as $w) {

Debianでmecabをaptでインストールすると，/usr/bin/mecabにインストールされるけど，ほかの環境の人はここを変えないと。

つづく