NLP Toolkit

基于统计指标的高级自然语言处理（NLP）分析，用于 AI 内容检测。

实现

/**
 * 使用 NLP 指标分析文本
 * @param {string} text - 待分析的文本
 * @param {object} options - 配置选项
 * @returns {object} NLP 分析结果
 */
async function analyzeText(text, options = {}) {
  const {
    perplexityThreshold = 45.0,
    burstinessThreshold = 0.35,
    minTextLength = 50
  } = options;

  if (text.length < minTextLength) {
    return {
      error: 'Text too short for analysis',
      minLength: minTextLength
    };
  }

  // 计算困惑度（简化版）
  const perplexity = calculatePerplexity(text);

  // 计算突发性（burstiness）
  const burstiness = calculateBurstiness(text);

  // 计算香农熵（entropy）
  const entropy = calculateEntropy(text);

  // 词元（token）分布分析
  const tokenStats = analyzeTokenDistribution(text);

  // 判定是否为 AI 生成内容
  const isAI = perplexity < perplexityThreshold && burstiness < burstinessThreshold;
  const confidence = calculateConfidence(perplexity, burstiness, entropy);

  return {
    isAI,
    confidence: Math.round(confidence * 100),
    metrics: {
      perplexity: Math.round(perplexity * 100) / 100,
      burstiness: Math.round(burstiness * 100) / 100,
      entropy: Math.round(entropy * 100) / 100
    },
    tokenStats,
    thresholds: {
      perplexity: perplexityThreshold,
      burstiness: burstinessThreshold
    },
    explanation: isAI ? 
      'Low perplexity and uniform burstiness suggest AI generation' :
      'Natural variation in metrics suggests human writing'
  };
}

/**
 * 计算困惑度得分（简化版）
 */
function calculatePerplexity(text) {
  const words = text.toLowerCase().split(/\s+/);
  const uniqueWords = new Set(words);
  
  // 简化版困惑度：唯一词数与总词数之比
  // 实际困惑度需依赖语言模型
  const ratio = uniqueWords.size / words.length;
  const perplexity = 100 / ratio; // 呈反比关系
  
  return Math.min(perplexity, 100);
}

/**
 * 计算突发性（burstiness，即句长变化程度）
 */
function calculateBurstiness(text) {
  const sentences = text.split(/[.!?]+/).filter(s => s.trim());
  if (sentences.length < 2) return 0;

  const lengths = sentences.map(s => s.split(/\s+/).length);
  const avg = lengths.reduce((a, b) => a + b, 0) / lengths.length;
  const variance = lengths.reduce((sum, len) => sum + Math.pow(len - avg, 2), 0) / lengths.length;
  const stdDev = Math.sqrt(variance);

  // 突发性：变异系数（标准差 / 均值）
  const burstiness = stdDev / avg;

  return Math.min(burstiness, 1.0);
}

/**
 * 计算香农熵（Shannon entropy）
 */
function calculateEntropy(text) {
  const chars = text.toLowerCase().split('');
  const freq = {};

  // 统计字符频次
  for (const char of chars) {
    freq[char] = (freq[char] || 0) + 1;
  }

  // 计算熵值
  let entropy = 0;
  const total = chars.length;

  for (const count of Object.values(freq)) {
    const p = count / total;
    entropy -= p * Math.log2(p);
  }

  return entropy;
}

/**
 * 分析词元（token）分布
 */
function analyzeTokenDistribution(text) {
  const words = text.toLowerCase().split(/\s+/);
  const uniqueWords = new Set(words);

  return {
    totalWords: words.length,
    uniqueWords: uniqueWords.size,
    vocabularyRichness: Math.round((uniqueWords.size / words.length) * 100) / 100
  };
}

/**
 * 计算综合置信度
 */
function calculateConfidence(perplexity, burstiness, entropy) {
  // 困惑度越低，越倾向 AI 生成
  const perplexityScore = Math.max(0, 1 - (perplexity / 100));
  
  // 突发性越低，越倾向 AI 生成
  const burstinessScore = Math.max(0, 1 - (burstiness / 0.5));
  
  // AI 生成文本通常具有中等熵值
  const entropyScore = (entropy > 3.5 && entropy < 5.0) ? 0.8 : 0.4;

  const confidence = (perplexityScore + burstinessScore + entropyScore) / 3;
  return Math.min(confidence, 1.0);
}

// 导出供 OpenClaw 使用
module.exports = {
  analyzeText,
  calculatePerplexity,
  calculateBurstiness,
  calculateEntropy
};

使用方法

const result = await skills.nlpToolkit.analyzeText(text, {
  perplexityThreshold: 45.0,
  burstinessThreshold: 0.35
});

console.log(`AI Detection: ${result.isAI} (${result.confidence}% confidence)`);
console.log(`Perplexity: ${result.metrics.perplexity}`);
console.log(`Burstiness: ${result.metrics.burstiness}`);

配置

{
  "perplexityThreshold": 45.0,
  "burstinessThreshold": 0.35,
  "minTextLength": 50
}