From a1e151048c1705c2f9a32562d288f87c8918c784 Mon Sep 17 00:00:00 2001 From: wangjinlei <751475802@qq.com> Date: Fri, 7 Aug 2026 16:04:59 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8F=82=E8=80=83=E6=96=87=E7=8C=AE=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F--book?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- application/common/BookCitationParser.php | 405 ++++++++++++++++++ application/common/BookMetadataService.php | 66 +-- .../common/ReferenceDispatchService.php | 96 +++-- 3 files changed, 475 insertions(+), 92 deletions(-) create mode 100644 application/common/BookCitationParser.php diff --git a/application/common/BookCitationParser.php b/application/common/BookCitationParser.php new file mode 100644 index 00000000..2b96482b --- /dev/null +++ b/application/common/BookCitationParser.php @@ -0,0 +1,405 @@ +useLlm = isset($config['use_llm']) ? (bool)$config['use_llm'] : true; + } + + /** + * @return array author/title/publisher/place/year/edition/pages/isbn/container + */ + public function parse($content) + { + $out = [ + 'author' => '', + 'title' => '', + 'publisher' => '', + 'place' => '', + 'year' => '', + 'edition' => '', + 'pages' => '', + 'isbn' => '', + 'container' => '', + ]; + + $clean = $this->normalize($content); + if ($clean === '') { + return $out; + } + + $out['isbn'] = $this->matchIsbn($clean); + $clean = $this->stripTail($clean); + + // 章节引用："章节作者. 章节名. In: 编者. 书名. 版次. 地点: 出版社; 年. 页码" + // 出版信息属于 In: 之后的那本书，必须分开解析，否则会把编者当成书名 + if (preg_match('/^(.*?)\bIn\s*:\s*(.+)$/isu', $clean, $m) && trim($m[1]) !== '' && trim($m[2]) !== '') { + $chapter = trim($m[1]); + $book = trim($m[2]); + + $this->parseSegments($this->takeAuthor($chapter, $out['author']), $out); + + $bookOut = $out; + $bookOut['title'] = ''; + $editors = ''; + $this->parseSegments($this->takeAuthor($book, $editors), $bookOut); + + $out['container'] = $bookOut['title']; + foreach (['publisher', 'place', 'year', 'edition', 'pages'] as $field) { + if ($out[$field] === '') { + $out[$field] = $bookOut[$field]; + } + } + if ($out['author'] === '') { + $out['author'] = $editors; + } + } else { + $this->parseSegments($this->takeAuthor($clean, $out['author']), $out); + } + + if ($this->needsLlm($out)) { + $out = $this->refineByLlm($clean, $out); + } + + return $out; + } + + /** + * 去掉条目序号："1. " / "[1] " / "(1) " / "1) " / "1、" + */ + public function stripIndex($content) + { + return preg_replace('/^\s*(?:\[\d{1,4}\]|\(\d{1,4}\)|\d{1,4}\s*[\.\)\]、,])\s*/u', '', (string)$content); + } + + private function normalize($content) + { + $content = trim((string)$content); + $content = str_replace( + ['：', '；', '，', '（', '）', '［', '］', '　', '．'], + [':', ';', ',', '(', ')', '[', ']', ' ', '.'], + $content + ); + $content = preg_replace('/\s+/u', ' ', $content); + $content = $this->stripIndex($content); + // 中文句号统一成英文句点，便于按同一套规则切分 + $content = str_replace('。', '. ', $content); + + return trim(preg_replace('/\s+/u', ' ', $content)); + } + + /** + * 去掉尾部的获取途径（Available at / Accessed / URL / DOI），它们会干扰分段 + */ + private function stripTail($text) + { + $text = preg_replace('/\s*(?:Available\s*(?:at|from)|Retrieved\s+from|Accessed\s+(?:on\s+)?)\s*:?.*$/iu', '', $text); + $text = preg_replace('/\s*https?:\/\/\S+/iu', '', $text); + $text = preg_replace('/\s*\bdoi\s*:\s*\S+/iu', '', $text); + $text = preg_replace('/\s*\bISBN(?:-1[03])?\s*:?\s*[\d\-\sXx]{10,20}\.?/iu', '', $text); + + return trim($text); + } + + private function matchIsbn($text) + { + if (!preg_match('/\bISBN(?:-1[03])?[:\s]*([0-9][0-9\-\s]{8,20}[0-9Xx])/i', $text, $m)) { + return ''; + } + + return preg_replace('/[^0-9Xx]/', '', $m[1]); + } + + // ------------------------------------------------------------------ + // 作者 + // ------------------------------------------------------------------ + + /** + * 从头部切出作者，返回剩余部分 + */ + private function takeAuthor($text, &$author) + { + $author = ''; + + // 温哥华格式：Surname AB, Surname CD, et al.（可带 editor/eds 标记） + $name = '\p{Lu}[\p{L}\'\x{2019}\-]+(?:\s+\p{Lu}[\p{L}\'\x{2019}\-]+)*\s+\p{Lu}{1,4}'; + $pattern = '/^((?:' . $name . ')(?:\s*,\s*(?:' . $name . '))*' + . '(?:\s*,?\s*et\s+al\.?)?)(?:\s*,\s*(?:eds?|editors?))?\s*[\.,]\s+/u'; + if (preg_match($pattern, $text, $m)) { + $author = trim($m[1], " ,."); + return trim(substr($text, strlen($m[0]))); + } + + // 机构作者 + $org = '/^([\p{Lu}][\p{L}\s&\-,\.]{4,90}?(?:Organization|Organisation|Association|Society|Institute' + . '|Institutes|Ministry|Committee|Council|Administration|Agency|Department|Bureau|Foundation' + . '|Academy|Commission|Centers?|WHO|CDC|NIH|FDA|NICE|UNICEF|OECD))\.\s+/u'; + if (preg_match($org, $text, $m)) { + $author = trim($m[1], " ,."); + return trim(substr($text, strlen($m[0]))); + } + + // 中文作者：张三, 李四, 等. + $cn = '/^([\x{4e00}-\x{9fff}·]{2,10}(?:\s*[,、;]\s*[\x{4e00}-\x{9fff}·]{2,10})*(?:\s*,?\s*等)?)\s*[\.,]\s*/u'; + if (preg_match($cn, $text, $m)) { + $author = trim($m[1], " ,."); + return trim(substr($text, strlen($m[0]))); + } + + // 兜底：第一个句点之前 + $parts = preg_split('/\.\s+/u', $text, 2); + if (is_array($parts) && count($parts) === 2 && mb_strlen($parts[0], 'UTF-8') <= 120) { + $author = trim($parts[0], " ,."); + return trim($parts[1]); + } + + return $text; + } + + // ------------------------------------------------------------------ + // 分段与归类 + // ------------------------------------------------------------------ + + private function parseSegments($text, array &$out) + { + foreach ($this->splitSegments($text) as $seg) { + $seg = trim($seg, " .,;"); + if ($seg === '') { + continue; + } + + // "6th ed. New York: Garland Science; 2015" 里版次和出版信息粘在一段， + // 摘掉版次再往下判，否则出版地会变成 "6th ed. New York" + $edition = $this->matchEdition($seg); + if ($edition !== '') { + if ($out['edition'] === '') { + $out['edition'] = $edition; + } + $seg = trim($this->stripEdition($seg), " .,;"); + if ($seg === '') { + continue; + } + } + + if ($out['pages'] === '' && preg_match('/^(?:p{1,2}\.?|pages?)\s*([\divxlcIVXLC][\divxlcIVXLC\-\x{2013}]*)$/iu', $seg, $m)) { + $out['pages'] = $m[1]; + continue; + } + + if ($this->takePublication($seg, $out)) { + continue; + } + + if ($out['title'] === '') { + $out['title'] = $this->cleanTitle($seg); + } + } + + if ($out['year'] === '' && preg_match_all('/\b(?:19|20)\d{2}\b/', $text, $ym)) { + $out['year'] = end($ym[0]); + } + } + + /** + * 缩写后的句点不是分段边界（ed. / vol. / p. / Inc. / 姓名缩写） + * + * @return string[] + */ + private function splitSegments($text) + { + $abbr = 'ed|eds|edn|vol|vols|no|nos|pp|p|st|mt|dr|prof|inc|ltd|co|jr|sr|al|rev|suppl|fig|ch|pt|approx'; + $masked = preg_replace('/\b(' . $abbr . ')\.(?=\s)/iu', '$1' . "\x01", $text); + // "Kumar V. Abbas AK." 这类姓名缩写之间的点 + $masked = preg_replace('/\b(\p{Lu})\.(?=\s*\p{Lu}\b)/u', '$1' . "\x01", $masked); + + $segments = preg_split('/\.\s+|\.$/u', $masked); + $out = []; + foreach ((array)$segments as $seg) { + $out[] = str_replace("\x01", '.', $seg); + } + + return $out; + } + + private function matchEdition($seg) + { + if (preg_match('/\b(\d{1,2})\s*(?:st|nd|rd|th|d)?\s*(?:ed\.?|edn\.?|edition)\b/iu', $seg, $m)) { + return $m[1]; + } + if (preg_match('/\b(first|second|third|fourth|fifth|sixth|seventh|eighth|ninth|tenth)\s+(?:ed\.?|edn\.?|edition)\b/iu', $seg, $m)) { + return strtolower($m[1]); + } + if (preg_match('/(?:第\s*)?(\d{1,2})\s*版/u', $seg, $m)) { + return $m[1]; + } + + return ''; + } + + private function stripEdition($seg) + { + $seg = preg_replace('/\b\d{1,2}\s*(?:st|nd|rd|th|d)?\s*(?:ed\.?|edn\.?|edition)\b[\.,;]?/iu', '', $seg); + $seg = preg_replace('/\b(?:first|second|third|fourth|fifth|sixth|seventh|eighth|ninth|tenth)\s+(?:ed\.?|edn\.?|edition)\b[\.,;]?/iu', '', $seg); + $seg = preg_replace('/(?:第\s*)?\d{1,2}\s*版/u', '', $seg); + + return $seg; + } + + /** + * 出版信息段：Place: Publisher; Year / Publisher, Year / 北京: 人民卫生出版社, 2018 + */ + private function takePublication($seg, array &$out) + { + $hasYear = preg_match('/\b((?:19|20)\d{2})\b/', $seg, $ym); + $hasPlace = preg_match('/^([^:]{2,60}):\s*(.+)$/u', $seg, $pm); + if (!$hasYear && !$hasPlace) { + return false; + } + + // "Depression in adults: treatment and management" 是带副标题的书名，不是"地点: 出版社" + if ($hasPlace && !$hasYear && ($out['title'] === '' || !$this->looksLikePublisher($pm[2]))) { + return false; + } + + // 标题里出现年份（如 "Global tuberculosis report 2020"）不算出版信息段 + if ($hasYear && !$hasPlace && $out['title'] === '' && !preg_match('/^\W*(?:19|20)\d{2}\W*$/', $seg)) { + $withoutYear = trim(preg_replace('/\b(?:19|20)\d{2}\b/', '', $seg), " .,;"); + if ($withoutYear !== '' && !$this->looksLikePublisher($withoutYear)) { + return false; + } + } + + if ($hasYear && $out['year'] === '') { + $out['year'] = $ym[1]; + } + + $body = $hasPlace ? trim($pm[2]) : $seg; + if ($hasPlace && $out['place'] === '') { + $out['place'] = trim($pm[1], " .,;"); + } + $body = trim(preg_replace('/\b(?:19|20)\d{2}\b/', '', $body), " .,;:"); + if ($body !== '' && $out['publisher'] === '') { + $out['publisher'] = $body; + } + + return true; + } + + private function looksLikePublisher($text) + { + return (bool)preg_match( + '/(Press|Publish\w*|Books?|Elsevier|Springer|Wiley|Saunders|Mosby|Lippincott|Williams|Wilkins' + . '|McGraw|Academic|University|Univ\b|出版社|书局|WHO|Organization)/iu', + $text + ); + } + + private function cleanTitle($title) + { + $title = trim((string)$title); + // GB/T 7714 的文献类型标识：[M] [M/OL] [C] 等 + $title = preg_replace('/\s*\[[A-Z]{1,2}(?:\/[A-Z]{1,2})?\]\s*/u', ' ', $title); + $title = preg_replace('/\s+/u', ' ', $title); + + return trim($title, " .,;:"); + } + + // ------------------------------------------------------------------ + // 大模型兜底 + // ------------------------------------------------------------------ + + private function needsLlm(array $out) + { + if (!$this->useLlm) { + return false; + } + if ($out['title'] === '' || $out['author'] === '') { + return true; + } + // 分段错位时标题里会留下一串人名 + return $this->looksLikeAuthorList($out['title']); + } + + private function looksLikeAuthorList($text) + { + $text = trim((string)$text); + if ($text === '') { + return false; + } + $name = '\p{Lu}[\p{L}\'\-]+\s+\p{Lu}{1,4}'; + + return (bool)preg_match('/^(?:' . $name . ')(?:\s*,\s*(?:' . $name . '))*(?:\s*,?\s*et\s+al\.?)?$/u', $text); + } + + private function refineByLlm($content, array $out) + { + try { + $llm = new LLMService(); + $system = 'You extract bibliographic fields from a single book (or book chapter) reference string. ' + . 'Return ONLY a JSON object with these keys: ' + . '{"author":"","title":"","publisher":"","place":"","year":"","edition":"","pages":""}. ' + . 'Rules: author = the author or editor list exactly as written, without a trailing period; ' + . 'title = the title of the cited work only, without edition, publisher, place or year; ' + . 'for a chapter reference, title = the chapter title; ' + . 'publisher = publisher name only, no place; year = 4-digit publication year; ' + . 'edition = the edition number as digits only (e.g. "6"), empty if not stated; ' + . 'pages = page range if stated. Use an empty string for anything not stated. ' + . 'Do not translate, do not invent, do not add explanation.'; + $user = "Reference:\n" . mb_substr($content, 0, 1500); + + $reply = $llm->requestChat([ + ['role' => 'system', 'content' => $system], + ['role' => 'user', 'content' => $user], + ], 0); + if ($reply === null || $reply === '') { + return $out; + } + + $parsed = $llm->parseJsonResponse($reply); + if (!is_array($parsed)) { + return $out; + } + + foreach (['author', 'title', 'publisher', 'place', 'year', 'edition', 'pages'] as $field) { + $value = trim((string)($parsed[$field] ?? '')); + if ($value === '') { + continue; + } + // 模型不得改写原文里没有的内容 + if ($field === 'year' && !preg_match('/^(?:19|20)\d{2}$/', $value)) { + continue; + } + if ($field === 'edition') { + $value = preg_match('/\d{1,2}/', $value, $m) ? $m[0] : ''; + if ($value === '') { + continue; + } + } + if ($field === 'title' && $this->looksLikeAuthorList($value)) { + continue; + } + $out[$field] = $field === 'title' ? $this->cleanTitle($value) : $value; + } + } catch (\Throwable $e) { + \think\Log::write('book citation llm parse failed: ' . $e->getMessage(), 'error'); + } + + return $out; + } +} diff --git a/application/common/BookMetadataService.php b/application/common/BookMetadataService.php index 58562565..09cc13a8 100644 --- a/application/common/BookMetadataService.php +++ b/application/common/BookMetadataService.php @@ -120,29 +120,23 @@ class BookMetadataService return $hints; } - // 章节引用（In: 编者. 书名. ...）反查的是书，不是章节 - $bookPart = $content; - if (preg_match('/\bIn:\s*(.+)$/i', $content, $m)) { - $bookPart = trim($m[1]); - $hints['title'] = ''; + // 反查用规则解析即可，不为了取个 ISBN 再多花一次大模型调用 + $parsed = (new BookCitationParser(['use_llm' => false]))->parse($content); + + // 章节引用反查的是书，不是章节 + if ($parsed['container'] !== '') { + $hints['title'] = $parsed['container']; + } elseif ($hints['title'] === '') { + $hints['title'] = $parsed['title']; } - $hints['edition'] = $this->parseEdition($content); - - if ($hints['year'] === '' && preg_match_all('/\b(?:19|20)\d{2}\b/', $content, $ym)) { - $hints['year'] = end($ym[0]); + foreach (['author', 'publisher', 'edition'] as $field) { + if ($hints[$field] === '' && $parsed[$field] !== '') { + $hints[$field] = $parsed[$field]; + } } - - // 地点: 出版社; 年份 - if ($hints['publisher'] === '' && preg_match('/:\s*([^;:]{2,60}?)\s*[;,]\s*(?:19|20)\d{2}/u', $bookPart, $m)) { - $hints['publisher'] = trim($m[1], " .,"); - } - - if ($hints['title'] === '') { - $hints['title'] = $this->parseTitle($bookPart); - } - if ($hints['author'] === '') { - $hints['author'] = $this->parseAuthor($bookPart); + if ($hints['year'] === '') { + $hints['year'] = $parsed['year']; } return $hints; @@ -432,38 +426,6 @@ class BookMetadataService return trim($title, " .,;:"); } - private function parseTitle($text) - { - $parts = preg_split('/\.\s+/u', trim((string)$text), 3); - if (is_array($parts) && count($parts) >= 2) { - return $this->cleanTitle($parts[1]); - } - - return ''; - } - - private function parseAuthor($text) - { - $parts = preg_split('/\.\s+/u', trim((string)$text), 2); - - return is_array($parts) && count($parts) >= 2 ? trim($parts[0]) : ''; - } - - private function parseEdition($text) - { - if (preg_match('/\b(\d{1,2})\s*(?:st|nd|rd|th|d)?\s*(?:ed\.?|edition)\b/iu', $text, $m)) { - return $m[1]; - } - if (preg_match('/\b(first|second|third|fourth|fifth|sixth|seventh|eighth|ninth|tenth)\s+(?:ed\.?|edition)\b/iu', $text, $m)) { - return $m[1]; - } - if (preg_match('/第\s*(\d{1,2})\s*版/u', $text, $m)) { - return $m[1]; - } - - return ''; - } - private function editionOrdinal($edition) { $edition = strtolower(trim((string)$edition)); diff --git a/application/common/ReferenceDispatchService.php b/application/common/ReferenceDispatchService.php index 95953f9d..060276dd 100644 --- a/application/common/ReferenceDispatchService.php +++ b/application/common/ReferenceDispatchService.php @@ -24,11 +24,15 @@ class ReferenceDispatchService /** @var BookMetadataService */ private $bookMetadata; + /** @var BookCitationParser */ + private $bookParser; + public function __construct() { $this->classifier = new ReferenceTypeClassifier(['use_llm' => true]); $this->metadata = new ReferenceMetadataService(); $this->bookMetadata = new BookMetadataService(); + $this->bookParser = new BookCitationParser(); } /** @@ -181,7 +185,9 @@ class ReferenceDispatchService $summary = is_array($meta) ? $meta['crossref_summary'] : null; $raw = is_array($summary) ? ($summary['raw'] ?? []) : []; $hasMeta = is_array($meta) - && (trim((string)$meta['title']) !== '' || trim((string)$meta['author']) !== ''); + && (trim((string)$meta['title']) !== '' || trim((string)$meta['author']) !== '') + // 图书著录里挂的 DOI 常常是被错误关联的期刊文章，题名对不上就不能拿来覆盖 + && $this->metaMatchesContent($meta, $content); if ($hasMeta) { $authorCitation = trim((string)$meta['author']); @@ -217,6 +223,44 @@ class ReferenceDispatchService Db::name('production_article_refer')->where('p_refer_id', $pReferId)->update($update); } + /** + * DOI 抓回来的题名是否确实是这条著录说的那本书 + */ + private function metaMatchesContent($meta, $content) + { + $title = $this->normalizeForMatch((string)$meta['title']); + if ($title === '') { + return false; + } + $blob = $this->normalizeForMatch($content); + if ($blob === '' || strpos($blob, $title) !== false) { + return true; + } + + $words = array_filter(explode(' ', $title), function ($w) { + return strlen($w) > 3; + }); + if (empty($words)) { + return false; + } + $hit = 0; + foreach ($words as $word) { + if (strpos($blob, $word) !== false) { + $hit++; + } + } + + return ($hit / count($words)) >= 0.6; + } + + private function normalizeForMatch($text) + { + $text = strtolower(trim((string)$text)); + $text = preg_replace('/[^a-z0-9\x{4e00}-\x{9fff}\s]+/u', ' ', $text); + + return trim(preg_replace('/\s+/u', ' ', $text)); + } + /** * 按书名/作者/年份/版次反查 ISBN，顺带补齐缺失的出版社与出版年 * @@ -299,50 +343,22 @@ class ReferenceDispatchService } /** - * 无 Crossref 时从原文尽量抽取 book 结构化字段 + * 无可用 DOI 元数据时，从原文著录抽取 book 结构化字段 */ private function parseBookFromContent($content) { - $content = trim((string)$content); - $out = [ - 'author' => '', - 'title' => '', - 'joura' => '', - 'dateno' => '', - 'isbn' => '', + $parsed = $this->bookParser->parse($content); + + $author = trim((string)$parsed['author']); + + return [ + 'author' => $author !== '' ? rtrim($author, '.') . '.' : '', + 'title' => trim((string)$parsed['title']), + 'joura' => trim((string)$parsed['publisher']), + 'dateno' => trim((string)$parsed['year']), + 'isbn' => trim((string)$parsed['isbn']), 'is_ja' => 1, ]; - - if ($content === '') { - return $out; - } - - if (preg_match('/\bISBN[:\s]*([\d\-Xx\s]+)/i', $content, $m)) { - $out['isbn'] = preg_replace('/\s+/', '-', trim($m[1])); - } - - if (preg_match('/\b(19|20)\d{2}\b/', $content, $m)) { - $out['dateno'] = $m[0]; - } - - // Place: Publisher; Year → joura 取 Publisher - if (preg_match('/:\s*([^;]+);\s*(19|20)\d{2}/', $content, $m)) { - $out['joura'] = trim($m[1]); - } elseif (preg_match('/\b([A-Z][A-Za-z .&]+(?:Press|Publishing|Publisher|Books?))\b/i', $content, $m)) { - $out['joura'] = trim($m[1]); - } - - // 作者. 标题. ... 简单拆分 - $parts = preg_split('/\.\s+/', $content, 3); - if (is_array($parts) && count($parts) >= 2) { - $out['author'] = trim($parts[0]); - if (substr($out['author'], -1) !== '.') { - $out['author'] .= '.'; - } - $out['title'] = trim(rtrim($parts[1], '.')); - } - - return $out; } private function normalizeDoi($doi)