ai写作辅助检测

2026-07-15 10:55:08 +08:00
parent c80ff9b99f
commit a46df66028
4 changed files with 695 additions and 0 deletions
--- a/application/common/ManuscriptTextCleanService.php
+++ b/application/common/ManuscriptTextCleanService.php
@@ -0,0 +1,121 @@
+<?php
+
+namespace app\common;
+
+/**
+ * 稿件文本简单清洗（检测前预处理，不改变接口返回的原始正文）
+ */
+class ManuscriptTextCleanService
+{
+    /**
+     * 检测前文本清洗：解码、去噪、去引用标记、统一空白
+     */
+    public static function clean(string $text): string
+    {
+        $text = trim($text);
+        if ($text === '') {
+            return '';
+        }
+
+        $text = html_entity_decode($text, ENT_QUOTES | ENT_HTML5, 'UTF-8');
+        $text = self::normalizeEncoding($text);
+        $text = self::normalizeQuotesAndDashes($text);
+        $text = self::stripInvisibleChars($text);
+        $text = self::stripUrlsAndEmails($text);
+        $text = self::stripCitationMarkers($text);
+        $text = self::stripFigureTableTags($text);
+        $text = self::stripFootnoteMarkers($text);
+        $text = self::stripSearchStrategyNoise($text);
+        $text = preg_replace('/\s+/u', ' ', $text);
+        $text = trim((string) preg_replace('/\s+([,.;:!?])/u', '$1', $text));
+
+        return trim($text);
+    }
+
+    /**
+     * @param array<string,string> $sections
+     * @return array<string,string>
+     */
+    public static function cleanSections(array $sections): array
+    {
+        $cleaned = [];
+        foreach ($sections as $key => $text) {
+            $cleaned[$key] = self::clean((string) $text);
+        }
+        return $cleaned;
+    }
+
+    private static function normalizeEncoding(string $text): string
+    {
+        if (!mb_check_encoding($text, 'UTF-8')) {
+            $text = mb_convert_encoding($text, 'UTF-8', 'GBK,GB2312,GB18030,ISO-8859-1');
+        }
+        return $text;
+    }
+
+    private static function normalizeQuotesAndDashes(string $text): string
+    {
+        return strtr($text, [
+            '‘' => "'",
+            '’' => "'",
+            '“' => '"',
+            '”' => '"',
+            '„' => '"',
+            '‟' => '"',
+            '—' => '-',
+            '–' => '-',
+            '‑' => '-',
+            '　' => ' ',
+        ]);
+    }
+
+    private static function stripInvisibleChars(string $text): string
+    {
+        $text = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/u', '', $text);
+        $text = preg_replace('/[\x{200B}-\x{200F}\x{202A}-\x{202E}\x{2060}\x{FEFF}]/u', '', $text);
+        $text = str_replace([chr(0xC2) . chr(0xA0), '&nbsp;'], ' ', $text);
+        return $text;
+    }
+
+    private static function stripUrlsAndEmails(string $text): string
+    {
+        $text = preg_replace('/https?:\/\/\S+/iu', ' ', $text);
+        $text = preg_replace('/\b[\w.+-]+@[\w.-]+\.[a-z]{2,}\b/iu', ' ', $text);
+        return $text;
+    }
+
+    private static function stripCitationMarkers(string $text): string
+    {
+        // [1] [1,2] [1-3] [1, 2-4]
+        $text = preg_replace('/\[\s*\d+(?:\s*[,，\-–—]\s*\d+)*\s*\]/u', ' ', $text);
+        // 上标残留数字（如 word 提取后的 ^1^）
+        $text = preg_replace('/\b\^\d+\^/u', ' ', $text);
+        return $text;
+    }
+
+    private static function stripFigureTableTags(string $text): string
+    {
+        $text = preg_replace('/\((?:Fig(?:ure)?|Table|Tab\.)\s*[A-Za-z0-9.-]+\)/iu', ' ', $text);
+        $text = preg_replace('/\b(?:Fig(?:ure)?|Table|Tab\.)\s*[A-Za-z0-9.-]+\b/iu', ' ', $text);
+        return $text;
+    }
+
+    private static function stripFootnoteMarkers(string $text): string
+    {
+        $text = preg_replace('/[*#†‡§]+\d*/u', ' ', $text);
+        return $text;
+    }
+
+    /**
+     * 去除文献检索式、数据库字段等 Methods 常见残留
+     */
+    private static function stripSearchStrategyNoise(string $text): string
+    {
+        $text = preg_replace('/\btitle\s*[\/\-]\s*abstract\b/iu', ' ', $text);
+        $text = preg_replace('/\babstract\s*[\/\-]\s*title\b/iu', ' ', $text);
+        $text = preg_replace('/\bti\s*[\-\/]?\s*ab\b/iu', ' ', $text);
+        $text = preg_replace('/\bmesh\s+terms?\b/iu', ' ', $text);
+        $text = preg_replace('/\b(search\s+terms?|search\s+strategy)\b/iu', ' ', $text);
+        return $text;
+    }
+}