使用 AC自动机实现大数据量下的敏感词匹配与替换
·
1 效果图

2. 完整方法示例
// 例如这是敏感词
export const sensitiveWords = [
...new Set([ // 使用 Set 去重
...decryptBase64(advertisementSensitiveWordsBase64).split(','),
...decryptBase64(pornographicSensitiveWordsBase64).split(','),
...decryptBase64(illegalSensitiveWordsBase64).split(','),
...decryptBase64(urlSensitiveWordsBase64).split(','),
...decryptBase64(politicsSensitiveWordsBase64).split(',')
])
];
// -------------------- O(m+n) Aho-Corasick 自动机实现 --------------------
/**
* AC 自动机节点结构
*/
function createNode () {
return {
next: Object.create(null), // 字符 -> 子节点
fail: null, // 失配指针
out: [] // 命中的模式串列表
};
}
/**
* 规范化用户传入的扩展敏感词(逗号分隔或数组)
*/
function normalizeExtraWords (extra) {
if (!extra) return [];
if (Array.isArray(extra)) return extra.filter(Boolean);
if (typeof extra === 'string') return extra.split(',').map(x => x.trim()).filter(Boolean);
return [];
}
/**
* 构建 AC 自动机
* 时间复杂度 O(m),m 为词库总长度
*/
function buildAutomaton (words) {
const root = createNode();
// 1) 构建 Trie
for (const word of words) {
if (!word) continue;
let node = root;
for (let i = 0; i < word.length; i++) {
const ch = word[i];
if (!node.next[ch]) node.next[ch] = createNode();
node = node.next[ch];
}
node.out.push(word);
}
// 2) 构建失败指针(BFS)
const queue = [];
// 根第一层的 fail 指向根
for (const ch in root.next) {
const child = root.next[ch];
child.fail = root;
queue.push(child);
}
root.fail = root;
while (queue.length) {
const current = queue.shift();
for (const ch in current.next) {
const child = current.next[ch];
let fail = current.fail;
while (fail && !fail.next[ch] && fail !== root) fail = fail.fail;
child.fail = (fail && fail.next[ch]) ? fail.next[ch] : root;
// 合并输出
if (child.fail && child.fail.out && child.fail.out.length) {
child.out = child.out.concat(child.fail.out);
}
queue.push(child);
}
}
return root;
}
/**
* 在文本上运行 AC 自动机
* 返回所有命中区间与词
* 时间复杂度 O(n),n 为文本长度
*/
function runAutomaton (text, root) {
const matches = [];
let node = root;
for (let i = 0; i < text.length; i++) {
const ch = text[i];
while (node && !node.next[ch] && node !== root) node = node.fail;
node = node.next[ch] || root;
if (node.out && node.out.length) {
for (const w of node.out) {
const start = i - w.length + 1;
if (start >= 0) matches.push({ start, end: i + 1, word: w });
}
}
}
return matches;
}
/**
* 根据命中区间进行掩码替换
*/
function maskWithMatches (text, matches) {
if (!matches.length) return text;
// 合并区间,避免重叠覆盖问题
matches.sort((a, b) => a.start - b.start || a.end - b.end);
const merged = [];
for (const m of matches) {
if (!merged.length || m.start > merged[merged.length - 1].end) {
merged.push({ start: m.start, end: m.end });
} else {
merged[merged.length - 1].end = Math.max(merged[merged.length - 1].end, m.end);
}
}
let res = '';
let prev = 0;
for (const seg of merged) {
if (prev < seg.start) res += text.slice(prev, seg.start);
res += '*'.repeat(seg.end - seg.start);
prev = seg.end;
}
if (prev < text.length) res += text.slice(prev);
return res;
}
// 基础词库 AC(五类内置)
const baseWords = [...new Set(sensitiveWords.filter(Boolean))];
const baseAutomaton = buildAutomaton(baseWords);
// 缓存:不同扩展词集合的自动机
const automatonCache = new Map();
automatonCache.set('__base__', baseAutomaton);
function getAutomatonWithExtra (extra) {
const extraWords = normalizeExtraWords(extra);
if (!extraWords.length) return baseAutomaton;
const key = '__base__|' + extraWords.slice().sort().join('|');
if (automatonCache.has(key)) return automatonCache.get(key);
const combined = [...new Set([...baseWords, ...extraWords])];
const ac = buildAutomaton(combined);
automatonCache.set(key, ac);
return ac;
}
/**
* 检查文本中是否包含敏感词
* extraWords: 逗号分隔字符串或数组(用户词库)
* 返回:{ isSensitive: boolean, matchedWord?: string, matchedWords?: string[] }
*/
export function containsSensitiveWordsAC (text, extraWords) {
if (!text) return { isSensitive: false };
const ac = getAutomatonWithExtra(extraWords);
const matches = runAutomaton(text, ac);
if (matches.length) {
return {
isSensitive: true,
matchedWord: matches[0].word,
matchedWords: Array.from(new Set(matches.map(m => m.word)))
};
}
return { isSensitive: false };
}
/**
* 替换文本中的所有敏感词为 *
* extraWords: 逗号分隔字符串或数组(用户词库)
*/
export function replaceSensitiveWords (text, extraWords) {
if (!text) return text;
const ac = getAutomatonWithExtra(extraWords);
const matches = runAutomaton(text, ac);
return maskWithMatches(text, matches);
}
3 使用示例
<template>
<div class="container">
<el-button type="primary" round @click="submitForm">提交</el-button>
</div>
</template>
<script setup name="setup">
import { ref } from 'vue'
import { replaceSensitiveWords, containsSensitiveWordsAC } from '@/sensitiveWords/index'
const submitForm = () => {
const start = performance.now()
if (containsSensitiveWordsAC('兼职').isSensitive) {
const duration = performance.now() - start
console.log(replaceSensitiveWords('兼职'));
console.log(`敏感词检测耗时AC: ${duration.toFixed(2)}ms`)
}
}
</script>
<style scoped lang="less"></style>
更多推荐



所有评论(0)