122 lines
4.0 KiB
PHP
122 lines
4.0 KiB
PHP
<?php
|
||
|
||
namespace app\common;
|
||
|
||
/**
|
||
* 稿件文本简单清洗（检测前预处理，不改变接口返回的原始正文）
|
||
*/
|
||
class ManuscriptTextCleanService
|
||
{
|
||
/**
|
||
* 检测前文本清洗：解码、去噪、去引用标记、统一空白
|
||
*/
|
||
public static function clean(string $text): string
|
||
{
|
||
$text = trim($text);
|
||
if ($text === '') {
|
||
return '';
|
||
}
|
||
|
||
$text = html_entity_decode($text, ENT_QUOTES | ENT_HTML5, 'UTF-8');
|
||
$text = self::normalizeEncoding($text);
|
||
$text = self::normalizeQuotesAndDashes($text);
|
||
$text = self::stripInvisibleChars($text);
|
||
$text = self::stripUrlsAndEmails($text);
|
||
$text = self::stripCitationMarkers($text);
|
||
$text = self::stripFigureTableTags($text);
|
||
$text = self::stripFootnoteMarkers($text);
|
||
$text = self::stripSearchStrategyNoise($text);
|
||
$text = preg_replace('/\s+/u', ' ', $text);
|
||
$text = trim((string) preg_replace('/\s+([,.;:!?])/u', '$1', $text));
|
||
|
||
return trim($text);
|
||
}
|
||
|
||
/**
|
||
* @param array<string,string> $sections
|
||
* @return array<string,string>
|
||
*/
|
||
public static function cleanSections(array $sections): array
|
||
{
|
||
$cleaned = [];
|
||
foreach ($sections as $key => $text) {
|
||
$cleaned[$key] = self::clean((string) $text);
|
||
}
|
||
return $cleaned;
|
||
}
|
||
|
||
private static function normalizeEncoding(string $text): string
|
||
{
|
||
if (!mb_check_encoding($text, 'UTF-8')) {
|
||
$text = mb_convert_encoding($text, 'UTF-8', 'GBK,GB2312,GB18030,ISO-8859-1');
|
||
}
|
||
return $text;
|
||
}
|
||
|
||
private static function normalizeQuotesAndDashes(string $text): string
|
||
{
|
||
return strtr($text, [
|
||
'‘' => "'",
|
||
'’' => "'",
|
||
'“' => '"',
|
||
'”' => '"',
|
||
'„' => '"',
|
||
'‟' => '"',
|
||
'—' => '-',
|
||
'–' => '-',
|
||
'‑' => '-',
|
||
' ' => ' ',
|
||
]);
|
||
}
|
||
|
||
private static function stripInvisibleChars(string $text): string
|
||
{
|
||
$text = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/u', '', $text);
|
||
$text = preg_replace('/[\x{200B}-\x{200F}\x{202A}-\x{202E}\x{2060}\x{FEFF}]/u', '', $text);
|
||
$text = str_replace([chr(0xC2) . chr(0xA0), ' '], ' ', $text);
|
||
return $text;
|
||
}
|
||
|
||
private static function stripUrlsAndEmails(string $text): string
|
||
{
|
||
$text = preg_replace('/https?:\/\/\S+/iu', ' ', $text);
|
||
$text = preg_replace('/\b[\w.+-]+@[\w.-]+\.[a-z]{2,}\b/iu', ' ', $text);
|
||
return $text;
|
||
}
|
||
|
||
private static function stripCitationMarkers(string $text): string
|
||
{
|
||
// [1] [1,2] [1-3] [1, 2-4]
|
||
$text = preg_replace('/\[\s*\d+(?:\s*[,，\-–—]\s*\d+)*\s*\]/u', ' ', $text);
|
||
// 上标残留数字（如 word 提取后的 ^1^）
|
||
$text = preg_replace('/\b\^\d+\^/u', ' ', $text);
|
||
return $text;
|
||
}
|
||
|
||
private static function stripFigureTableTags(string $text): string
|
||
{
|
||
$text = preg_replace('/\((?:Fig(?:ure)?|Table|Tab\.)\s*[A-Za-z0-9.-]+\)/iu', ' ', $text);
|
||
$text = preg_replace('/\b(?:Fig(?:ure)?|Table|Tab\.)\s*[A-Za-z0-9.-]+\b/iu', ' ', $text);
|
||
return $text;
|
||
}
|
||
|
||
private static function stripFootnoteMarkers(string $text): string
|
||
{
|
||
$text = preg_replace('/[*#†‡§]+\d*/u', ' ', $text);
|
||
return $text;
|
||
}
|
||
|
||
/**
|
||
* 去除文献检索式、数据库字段等 Methods 常见残留
|
||
*/
|
||
private static function stripSearchStrategyNoise(string $text): string
|
||
{
|
||
$text = preg_replace('/\btitle\s*[\/\-]\s*abstract\b/iu', ' ', $text);
|
||
$text = preg_replace('/\babstract\s*[\/\-]\s*title\b/iu', ' ', $text);
|
||
$text = preg_replace('/\bti\s*[\-\/]?\s*ab\b/iu', ' ', $text);
|
||
$text = preg_replace('/\bmesh\s+terms?\b/iu', ' ', $text);
|
||
$text = preg_replace('/\b(search\s+terms?|search\s+strategy)\b/iu', ' ', $text);
|
||
return $text;
|
||
}
|
||
}
|