字符串 - PHP 敏感词违法关键字检测 算法方案
- WBOY원래의
- 2016-06-06 20:34:132031검색
已有6000条关键字，分3批次。
一批为替换 replace，一批为遇到需要审核 censor，最后一批为遇到就禁止发布banned。
设计数据表如下：
<code>mysql> desc tbl_censor; +-------------+----------------------+------+-----+---------+----------------+ | Field | Type | Null | Key | Default | Extra | +-------------+----------------------+------+-----+---------+----------------+ | id | smallint(6) unsigned | NO | PRI | NULL | auto_increment | | censortype | smallint(6) | NO | | 1 | | | find | varchar(120) | NO | UNI | | | | replacement | varchar(255) | NO | | | | | extra | varchar(255) | NO | | | | | uptime | int(11) | YES | | NULL | | | enable | int(1) | NO | | 1 | | +-------------+----------------------+------+-----+---------+----------------+ 7 rows in set (0.01 sec) </code>
由于有6000多关键字，使用 foreach 的 strstr?还是preg_match ?
追求效率，每小时提交量为10万多文章。
刚刚写的一种：
<code>php</code><code>namespace app\helpers;
use app\models\other\Censor;
use app\models\other\CensorLog;
class CensorHelper
{
public $id;
public $data;
public $match_banned;
public $match_censor;
public function __construct($id = 'censor')
{
$this->id = $id;
$this->match_banned = [];
$this->match_censor = [];
$this->data = $this->getData();
}
/**
* @description 获取正则表达式
* @return array|mixed
*/
public function getData()
{
$data = \Yii::$app->cache->get($this->id);
if (empty($data)) {
$words = Censor::find()
->where(['enable' => 1])
->andWhere([' != ', 'replacement', ''])
->orderBy(['replacement' => SORT_ASC, 'find' => SORT_DESC])
->asArray()
->all();
$censor = [];
$banned = [];
$replace = [];
foreach ($words as $row) {
switch ($row['replacement']) {
case '{censor}':
$censor[] = $row['find'];
break;
case '{banned}':
$banned[] = $row['find'];
break;
default:
$replace['from'][] = $row['replacement'];
$replace['to'][] = $row['find'];
break;
}
}
if ($censor || $banned) {
$data = [
'censor' => $this->generateRegularExpression($censor),
'banned' => $this->generateRegularExpression($banned),
'replace' => $replace,
];
\Yii::$app->cache->set($this->id, $data);
}
}
return $data;
}
/**
* @describe 生成正则表达式
* @param array $words
* @return string
*/
public function generateRegularExpression(array $words)
{
$regular = implode('|', array_map('preg_quote', $words));
return "/$regular/i";
}
public function check($string)
{
$this->banned($string);
$this->censor($string);
}
public function censor($string)
{
if (!empty($this->data['censor']) && preg_match($this->data['censor'], $string, $matches)) {
$this->match_censor = array_merge($this->match_censor, $matches[0]);
}
}
public function banned($string)
{
if (!empty($this->data['banned']) && preg_match($this->data['banned'], $string, $matches)) {
$this->match_banned = array_merge($this->match_banned, $matches[0]);
}
}
//重新加载
public function flush()
{
\Yii::$app->cache->delete($this->id);
$this->getData();
}
/**
* @describe 替换
* @param $string
* @return mixed
*/
public function replace($string)
{
return !empty($this->data['replace']) ? str_replace($this->data['replace']['from'], $this->data['replace']['to'], $string) : $string;
}
/**
* @return string
*/
public function getLevel()
{
if (!empty($this->match_banned)) {
return 'banned';
} else if (!empty($this->match_censor)) {
return 'censor';
} else {
return 'pass';
}
}
/**
* @describe 添加记录
* @param $tableId
* @param $dataId
*/
public function addLog($tableId, $dataId)
{
$log = new CensorLog();
$log->datatb = $tableId;
$log->dataid = $dataId;
$log->matchcensor = implode(',', $this->match_censor);
$log->matchbanned = implode(',', $this->match_banned);
$log->addtime = time();
if (!\Yii::$app->user->isGuest) {
$log->uid = \Yii::$app->user->getId();
$log->uname = \Yii::$app->user->getUname();
}
$log->ip = IpHelper::getIP();
$log->iploc = IpHelper::getLocation($log->ip);
$log->save();
}
}
</code>
回复内容：
已有6000条关键字，分3批次。
一批为替换 replace，一批为遇到需要审核 censor，最后一批为遇到就禁止发布banned。
设计数据表如下：
<code>mysql> desc tbl_censor; +-------------+----------------------+------+-----+---------+----------------+ | Field | Type | Null | Key | Default | Extra | +-------------+----------------------+------+-----+---------+----------------+ | id | smallint(6) unsigned | NO | PRI | NULL | auto_increment | | censortype | smallint(6) | NO | | 1 | | | find | varchar(120) | NO | UNI | | | | replacement | varchar(255) | NO | | | | | extra | varchar(255) | NO | | | | | uptime | int(11) | YES | | NULL | | | enable | int(1) | NO | | 1 | | +-------------+----------------------+------+-----+---------+----------------+ 7 rows in set (0.01 sec) </code>
由于有6000多关键字，使用 foreach 的 strstr?还是preg_match ?
追求效率，每小时提交量为10万多文章。
刚刚写的一种：
<code>php</code><code>namespace app\helpers;
use app\models\other\Censor;
use app\models\other\CensorLog;
class CensorHelper
{
public $id;
public $data;
public $match_banned;
public $match_censor;
public function __construct($id = 'censor')
{
$this->id = $id;
$this->match_banned = [];
$this->match_censor = [];
$this->data = $this->getData();
}
/**
* @description 获取正则表达式
* @return array|mixed
*/
public function getData()
{
$data = \Yii::$app->cache->get($this->id);
if (empty($data)) {
$words = Censor::find()
->where(['enable' => 1])
->andWhere([' != ', 'replacement', ''])
->orderBy(['replacement' => SORT_ASC, 'find' => SORT_DESC])
->asArray()
->all();
$censor = [];
$banned = [];
$replace = [];
foreach ($words as $row) {
switch ($row['replacement']) {
case '{censor}':
$censor[] = $row['find'];
break;
case '{banned}':
$banned[] = $row['find'];
break;
default:
$replace['from'][] = $row['replacement'];
$replace['to'][] = $row['find'];
break;
}
}
if ($censor || $banned) {
$data = [
'censor' => $this->generateRegularExpression($censor),
'banned' => $this->generateRegularExpression($banned),
'replace' => $replace,
];
\Yii::$app->cache->set($this->id, $data);
}
}
return $data;
}
/**
* @describe 生成正则表达式
* @param array $words
* @return string
*/
public function generateRegularExpression(array $words)
{
$regular = implode('|', array_map('preg_quote', $words));
return "/$regular/i";
}
public function check($string)
{
$this->banned($string);
$this->censor($string);
}
public function censor($string)
{
if (!empty($this->data['censor']) && preg_match($this->data['censor'], $string, $matches)) {
$this->match_censor = array_merge($this->match_censor, $matches[0]);
}
}
public function banned($string)
{
if (!empty($this->data['banned']) && preg_match($this->data['banned'], $string, $matches)) {
$this->match_banned = array_merge($this->match_banned, $matches[0]);
}
}
//重新加载
public function flush()
{
\Yii::$app->cache->delete($this->id);
$this->getData();
}
/**
* @describe 替换
* @param $string
* @return mixed
*/
public function replace($string)
{
return !empty($this->data['replace']) ? str_replace($this->data['replace']['from'], $this->data['replace']['to'], $string) : $string;
}
/**
* @return string
*/
public function getLevel()
{
if (!empty($this->match_banned)) {
return 'banned';
} else if (!empty($this->match_censor)) {
return 'censor';
} else {
return 'pass';
}
}
/**
* @describe 添加记录
* @param $tableId
* @param $dataId
*/
public function addLog($tableId, $dataId)
{
$log = new CensorLog();
$log->datatb = $tableId;
$log->dataid = $dataId;
$log->matchcensor = implode(',', $this->match_censor);
$log->matchbanned = implode(',', $this->match_banned);
$log->addtime = time();
if (!\Yii::$app->user->isGuest) {
$log->uid = \Yii::$app->user->getId();
$log->uname = \Yii::$app->user->getUname();
}
$log->ip = IpHelper::getIP();
$log->iploc = IpHelper::getLocation($log->ip);
$log->save();
}
}
</code>
trie 树算法最适合。
PHP 关键词过滤扩展，该扩展依赖于 libdatrie（Trie 算法的 C++ 实现）。
你这个敏感词匹配，不需要用到正则，只用简单的匹配或者替换就行了。
关键字分成三类存memcached。
然后对文章进行匹配，应该从最严厉的banned来匹配，接着是要censor的关键字，最后才是可以replace的敏感词。
1 遇到就禁止发布 => str_pos
2 遇到需要审核 => str_pos
3 替换 => str_replace
성명：
본 글의 내용은 네티즌들의 자발적인 기여로 작성되었으며, 저작권은 원저작자에게 있습니다. 본 사이트는 이에 상응하는 법적 책임을 지지 않습니다. 표절이나 침해가 의심되는 콘텐츠를 발견한 경우 admin@php.cn으로 문의하세요.
이전 기사：php soap对象数组다음 기사：wordpress图片存在七牛云了 我让让图片赋值成300x300的