文字敏感词审核实现（本地实现）

一、创建SNS服务

1.使用golang的gin作为HTTP框架监听指定端口，使用post方式进行请求

请求/回应格式为：

type CheckWordReq struct { Content string } type CheckWordRsp struct { Status pb.StatusCode }

2.gin框架方法注册路由

func (server *SnsServer) initHttpEngine() error { gin.SetMode(gin.ReleaseMode) engine := gin.New() // 第一个参数是通过的日志处理，第二个是panic engine.Use(log.Gin, gin.RecoveryWithWriter(common.GinRecoverWriter{})) group := engine.Group("/sns") group.POST("check_word", server.OnCheckWord) server.httpEngine = engine return nil }

二、SNS本地审核实现

1、本地SNS检查器的结构

//检查器实体 type WordCheckerLocal struct { trie *Trie omitRune map[rune]struct{} // 忽略字符串中的这些干扰字符 } //对Node进行管理，管理整棵树的生命周期 type Trie struct { root *Node } //代表单个文字 type Node struct { IsEnd bool //是否是检查树的终点|确认和前面字符是敏感词了 IsAllLetter bool //检查期间是否全是字母 Children map[rune]*Node //后续字符 }

2、检查方法

所有检查器实现检查方法

// HasBadWord 检查是否包含敏感词 true 包含 func (checker *WordCheckerLocal) HasBadWord(s string) pb.StatusCode { if checker.trie.Has(s, checker.omitRune) { return pb.StatusCode_StatusCode_BAD_WORD } return pb.StatusCode_StatusCode_OK } // Has 检查是否包含敏感词，返回是否找到和匹配的词 func (trie *Trie) Has(s string, omitRune map[rune]struct{}) bool { node := trie.root end := utf8.RuneCountInString(s) - 1 rs := []rune(strings.ToLower(s)) start := 0 // 记录当前匹配路径 var currentMatch []rune for i := 0; i < len(rs); i++ { r := rs[i] c, ok := node.Children[r] if !ok { if _, ok := omitRune[r]; ok { currentMatch = append(currentMatch, r) continue } node = trie.root currentMatch = []rune{} i = start start++ continue } if c.IsEnd { currentMatch = append(currentMatch, r) //log.Debug("check end", log.String("r", string(r)), log.Bool("isAllLetter", c.IsAllLetter)) if !c.IsAllLetter { log.Debug("find bad word", log.String("match word", string(currentMatch)), log.String("word", s)) return true } // 单词需要完全匹配 避免hello匹配到hell if i == end || !unicode.IsLetter(rs[i+1]) || unicode.Is(unicode.Han, rs[i+1]) { log.Debug("find bad word", log.String("word", string(currentMatch)), log.String("word", s)) return true } if _, ok := c.Children[rs[i+1]]; ok { node = c continue } node = trie.root currentMatch = []rune{} i = start start++ continue } currentMatch = append(currentMatch, r) node = c } return false }

3、提供外层方法供gin路由调用

func (server *SnsServer) OnCheckWord(c *gin.Context) { bodyBytes, err := io.ReadAll(c.Request.Body) rsp := vo.CheckWordRsp{ Status: pb.StatusCode_StatusCode_OK, } if err != nil { log.Warn("read", log.Err(err)) rsp.Status = pb.StatusCode_StatusCode_FAIL goto END } { var req vo.CheckWordReq err = util.JsonUnmarshal(bodyBytes, &req) if err != nil { log.Warn("unmarshal", log.Err(err)) rsp.Status = pb.StatusCode_StatusCode_FAIL goto END } log.Debug("check", log.String("Content", req.Content)) if status := server.wordChecker.HasBadWord(req.Content); status != pb.StatusCode_StatusCode_OK { log.Warn("wordChecker status not ok", log.Any("status", status), log.String("Content", req.Content)) rsp.Status = status goto END } } END: c.JSON(http.StatusOK, rsp) }

三、本地敏感词词库文件