zm/tougao
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

参考文献模型校对换阿里云百炼，完善各种校对细节

Browse Source
This commit is contained in:
wyn
2026-08-06 10:14:36 +08:00
parent f4ebd10bfa
commit bffac7755d
7 changed files with 1551 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
application/common/ProductionArticleReferLiteratureService.php
View File

@@ -48,6 +48,8 @@ class ProductionArticleReferLiteratureService
'abstract_text' => '',
'content_text' => '',
'mesh_terms' => '',
'pub_language' => '',
'pub_country' => '',
'refer_content_cleaned' => '',
'literature_pdf_url' => '',
'fetch_sources' => '',
@@ -63,6 +65,8 @@ class ProductionArticleReferLiteratureService
'abstract_text' => trim((string)($stored['abstract_text'] ?? '')),
'content_text' => trim((string)($stored['content_text'] ?? '')),
'mesh_terms' => trim((string)($stored['mesh_terms'] ?? '')),
'pub_language' => trim((string)($stored['pub_language'] ?? '')),
'pub_country' => trim((string)($stored['pub_country'] ?? '')),
'refer_content_cleaned' => trim((string)($stored['refer_content_cleaned'] ?? '')),
'literature_pdf_url' => trim((string)($stored['literature_pdf_url'] ?? '')),
'fetch_sources' => trim((string)($stored['fetch_sources'] ?? '')),
@@ -109,6 +113,8 @@ class ProductionArticleReferLiteratureService
'abstract_text' => (string)($data['abstract_text'] ?? ''),
'content_text' => (string)($data['content_text'] ?? ''),
'mesh_terms' => $this->formatMeshTerms($data['mesh_terms'] ?? ''),
'pub_language' => $this->clip(strtolower(trim((string)($data['pub_language'] ?? ''))), 16),
'pub_country' => $this->clip(strtolower(trim((string)($data['pub_country'] ?? ''))), 64),
'refer_content_cleaned' => (string)($data['refer_content_cleaned'] ?? ''),
'literature_pdf_url' => $this->clip((string)($data['literature_pdf_url'] ?? ''), 1024),
'fetch_sources' => $this->clip($this->formatSources($data['fetch_sources'] ?? ''), 255),

33
application/common/PubmedService.php
View File

@@ -60,8 +60,8 @@ class PubmedService
$pmid = trim($pmid);
if ($pmid === '') return null;
// v2：解析结果新增 journal_iso_abbr / journal_medline_ta，换 key 避免命中旧缓存
$cacheKey = 'pmid_v2_' . $pmid;
// v3：解析结果新增 language / journal_country / affiliations，换 key 避免命中旧缓存
$cacheKey = 'pmid_v3_' . $pmid;
$cached = $this->cacheGet($cacheKey, 30 * 86400);
if (is_array($cached)) return $cached;
@@ -254,6 +254,31 @@ class PubmedService
}
}
// 文献语种：PubMed 用三字母代码（eng/chi/ger…），一篇可有多个
$languages = [];
$langNodes = $xp->query('//PubmedArticle//Article//Language');
if ($langNodes) {
foreach ($langNodes as $n) {
$t = strtolower(trim($n->textContent));
if ($t !== '') $languages[] = $t;
}
}
$languages = array_values(array_unique($languages));
// 期刊出版国（MedlineJournalInfo/Country），非研究开展国，仅作兜底
$journalCountry = $this->xpText($xp, '//PubmedArticle//MedlineJournalInfo//Country');
// 作者单位原文，用于推断研究开展国
$affiliations = [];
$affNodes = $xp->query('//PubmedArticle//AuthorList//Author//AffiliationInfo//Affiliation');
if ($affNodes) {
foreach ($affNodes as $n) {
$t = trim($n->textContent);
if ($t !== '') $affiliations[] = $t;
}
}
$affiliations = array_values(array_unique($affiliations));
if ($title === '' && $abstract === '') {
return null;
}
@@ -267,6 +292,10 @@ class PubmedService
'journal_iso_abbr' => $journalIsoAbbr,
'journal_medline_ta' => $journalMedlineTa,
'year' => $year,
'language' => isset($languages[0]) ? $languages[0] : '',
'languages' => $languages,
'journal_country' => $journalCountry,
'affiliations' => $affiliations,
];
}

19
application/common/ReferenceLiteratureFetchService.php
View File

@@ -22,6 +22,8 @@ class ReferenceLiteratureFetchService
private $cmaJournal;
/** @var ReferenceCheckService */
private $refUtil;
/** @var \app\common\service\BibliographicMetaService */
private $bibMeta;
/** @var bool 预抓取阶段暂不调用 Yiigle 机构 API */
private $skipYiigle = false;
@@ -39,6 +41,7 @@ class ReferenceLiteratureFetchService
$this->unpaywall = new UnpaywallService();
$this->cmaJournal = new CmaJournalLiteratureService();
$this->refUtil = new ReferenceCheckService();
$this->bibMeta = new \app\common\service\BibliographicMetaService();
}
public function setSkipYiigle($skip = true)
@@ -304,6 +307,8 @@ class ReferenceLiteratureFetchService
$fetchLogs = [];
$pdfUrl = '';
$meshTerms = [];
$language = '';
$country = '';
// 0) 中华医学会期刊 DOI：OpenAlex 中文摘要（可选 Yiigle 机构 API）
if (CmaJournalLiteratureService::isCmaJournalDoi($doi)) {
@@ -380,6 +385,16 @@ class ReferenceLiteratureFetchService
if (!empty($pub['mesh_terms']) && is_array($pub['mesh_terms'])) {
$meshTerms = array_values(array_unique(array_merge($meshTerms, $pub['mesh_terms'])));
}
if ($language === '' && trim((string)($pub['language'] ?? '')) !== '') {
$language = strtolower(trim((string)$pub['language']));
}
if ($country === '') {
// 研究开展国以作者单位为准，期刊出版国只作兜底
$country = $this->bibMeta->detectCountry((array)($pub['affiliations'] ?? []));
if ($country === '' && trim((string)($pub['journal_country'] ?? '')) !== '') {
$country = $this->bibMeta->detectCountry((string)$pub['journal_country']);
}
}
$pubBlock = $this->formatPubmedBlock($pub, $doi);
if ($pubBlock !== '' && $abstract === '') {
$blocks[] = $pubBlock;
@@ -444,6 +459,8 @@ class ReferenceLiteratureFetchService
'raw_content' => $raw,
'pdf_url' => $pdfUrl,
'mesh_terms' => $meshTerms,
'language' => $language,
'country' => $country,
'sources' => array_values(array_unique($sources)),
'fetch_log' => trim('doi=' . $doi . '; sources=' . implode(',', $sources) . ($fetchLogs ? '; ' . implode('; ', $fetchLogs) : '')),
];
@@ -615,6 +632,8 @@ class ReferenceLiteratureFetchService
'raw_content' => '',
'pdf_url' => '',
'mesh_terms' => [],
'language' => '',
'country' => '',
'sources' => [],
'fetch_log' => (string)$reason,
];

667
application/common/ReferenceRelevanceCheckService.php
View File

@@ -454,66 +454,377 @@ class ReferenceRelevanceCheckService
if (!empty($fresh) && intval($fresh['status']) === self::RECORD_COMPLETED) {
return $this->formatReturnFromRow($fresh);
}
throw new \RuntimeException('Citation group leader not finished, reference_no=' . $leaderRefNo);
// 组长已完成时，允许任一 pending 成员继续跑剩余分块（断点续跑）
$leaderCompleted = false;
foreach ($groupRows as $gr) {
if (intval($gr['reference_no']) === $leaderRefNo
&& intval($gr['status']) === self::RECORD_COMPLETED) {
$leaderCompleted = true;
break;
}
}
if (!$leaderCompleted) {
// 组员不应单独跑；由 worker 跳过，等组长处理
throw new \RuntimeException(
'Citation group leader not finished, reference_no=' . $leaderRefNo,
9001
);
}
}
}
// 只跑尚未完成的编号；已落库的成功块不再重跑
$pendingGroupRows = [];
foreach ($groupRows as $gr) {
if (intval($gr['status']) !== self::RECORD_COMPLETED) {
$pendingGroupRows[] = $gr;
}
}
if (empty($pendingGroupRows)) {
$fresh = Db::name('article_reference_relevance_check_result')->where('id', $checkId)->find();
return $this->formatReturnFromRow(!empty($fresh) ? $fresh : $row);
}
DbReconnectHelper::release();
DbReconnectHelper::ensure();
$sectionText = $this->refUtil->resolveMainContentForJob($row);
$localContext = $this->resolveLocalContextForJob($row);
$citeGroupRefs = $this->resolveCiteGroupRefs($row, $groupRows);
$referText = $this->buildCombinedReferText($groupRows);
$referTypeMap = $this->resolveReferTypeMap($groupRows);
$citeGroupRefs = $this->buildCiteGroupRefsFromRows($pendingGroupRows);
$referText = $this->buildCombinedReferText($pendingGroupRows);
$referTypeMap = $this->resolveReferTypeMap($pendingGroupRows);
if ($skipLiteratureFetch) {
$literatureBundle = $this->resolveGroupLiteratureBundle($groupRows, $referTypeMap, false);
$literatureBundle = $this->resolveGroupLiteratureBundle($pendingGroupRows, $referTypeMap, false);
$abstractText = $literatureBundle['combined_text'];
} else {
// 优先读 t_production_article_refer；摘要与清洗内容都为空时再抓取并回写 refer 表
DbReconnectHelper::release();
$literatureBundle = $this->resolveGroupLiteratureBundle($groupRows, $referTypeMap, true);
$literatureBundle = $this->resolveGroupLiteratureBundle($pendingGroupRows, $referTypeMap, true);
$abstractText = $literatureBundle['combined_text'];
DbReconnectHelper::ensure();
}
$noLiteratureEvidence = !$literatureBundle['has_verification_evidence'];
if ($noLiteratureEvidence && trim((string)$abstractText) === '') {
// 无摘要/全文时，退化为仅基于参考文献书目信息校对，并打标识供前端区分
$abstractText = "【文献书目信息（无摘要/全文）】\n" . $referText;
}
if ($sectionText === '' || $referText === '') {
$msg = 'Missing section content or refer_text';
$this->failGroupWithQueue($groupRows, $msg);
$this->failGroupWithQueue($pendingGroupRows, $msg);
throw new \RuntimeException($msg);
}
$llmService = new ReferenceRelevanceLlmService();
$appliedInCallback = 0;
$pArticleIdForHeartbeat = intval($row['p_article_id']);
$onChunkDone = function (array $part) use ($groupRows, &$appliedInCallback, $llmService, $pArticleIdForHeartbeat) {
DbReconnectHelper::ensure();
$n = $this->applyPartialGroupResults($groupRows, $part);
$appliedInCallback += $n;
if ($n > 0) {
$this->refreshGroupCombinedFields($groupRows, $llmService);
}
// 分块心跳：避免超大联合组单次跑超过僵尸阈值
if ($pArticleIdForHeartbeat > 0) {
Db::name('article_reference_relevance_check_batch')
->where('p_article_id', $pArticleIdForHeartbeat)
->where('batch_status', 1)
->update(['updated_at' => date('Y-m-d H:i:s')]);
}
$this->log(sprintf(
'relevance chunk persisted applied=%d',
$n
));
};
DbReconnectHelper::release();
$llm = (new ReferenceRelevanceLlmService())->checkRelevance(
$llm = $llmService->checkRelevance(
$sectionText,
$localContext,
$referText,
$abstractText,
$citeGroupRefs,
$referTypeMap
$referTypeMap,
$onChunkDone
);
DbReconnectHelper::ensure();
if (!empty($llm['request_failed']) || !$this->applyGroupResults($groupRows, $llm)) {
// 回调未覆盖的结果（单次调用或回调异常时）再补写一次
$applied = $appliedInCallback;
if (!empty($llm['results']) && is_array($llm['results'])) {
$applied += $this->applyPartialGroupResults($groupRows, $llm);
}
if ($applied <= 0 && !empty($llm['request_failed'])) {
$msg = isset($llm['reason']) ? (string)$llm['reason'] : 'LLM failed or empty results';
$this->failGroupWithQueue($groupRows, $msg);
// 逐篇/分块：单次 LLM 失败不整组标 failed，留 pending 续跑
$this->resetIncompleteGroupToPending($groupRows, $msg);
throw new \RuntimeException($msg);
}
$combinedOverride = [];
if (!empty($llm['combined_locked'])) {
$combinedOverride = [
'combined_relevance_score' => $llm['combined_relevance_score'] ?? 0,
'combined_reason' => $llm['combined_reason'] ?? '',
'combined_author_comment' => $llm['combined_author_comment'] ?? '',
];
}
$this->refreshGroupCombinedFields($groupRows, $llmService, $combinedOverride);
if ($noLiteratureEvidence) {
$this->markGroupNoLiteratureEvidence($groupRows);
$completedRows = [];
foreach ($this->findCitationGroupRows($row) as $gr) {
if (intval($gr['status']) === self::RECORD_COMPLETED) {
$completedRows[] = $gr;
}
}
if (!empty($completedRows)) {
$this->markGroupNoLiteratureEvidence($completedRows);
}
}
$this->markGroupQueueRuntime($groupRows, self::QUEUE_COMPLETED);
// 仍未完成的编号保持 pending，供后续重跑只补缺
$stillPending = $this->listIncompleteGroupRows($this->findCitationGroupRows($row));
foreach ($stillPending as $gr) {
$gid = intval($gr['id']);
if ($gid <= 0) {
continue;
}
if (intval($gr['status']) !== self::RECORD_PENDING) {
$this->updateRow($gid, [
'status' => self::RECORD_PENDING,
'error_msg' => isset($llm['reason']) ? mb_substr((string)$llm['reason'], 0, 512) : 'chunk pending retry',
]);
}
$this->markQueueRuntime($gid, self::QUEUE_PENDING);
}
$fresh = Db::name('article_reference_relevance_check_result')->where('id', $checkId)->find();
if (!empty($stillPending)) {
$this->log(sprintf(
'relevance group partial check_id=%d applied=%d still_pending=%d reason=%s',
$checkId,
$applied,
count($stillPending),
isset($llm['reason']) ? (string)$llm['reason'] : ''
));
}
return $this->formatReturnFromRow(!empty($fresh) ? $fresh : $row);
}
private function buildCiteGroupRefsFromRows(array $rows)
{
$nums = [];
foreach ($rows as $gr) {
$n = intval(isset($gr['reference_no']) ? $gr['reference_no'] : 0);
if ($n > 0) {
$nums[$n] = $n;
}
}
$list = array_values($nums);
sort($list, SORT_NUMERIC);
return implode(',', $list);
}
private function listIncompleteGroupRows(array $groupRows)
{
$out = [];
foreach ($groupRows as $gr) {
if (intval($gr['status']) !== self::RECORD_COMPLETED) {
$out[] = $gr;
}
}
return $out;
}
/**
* LLM 单次失败：未完成行保持/恢复 pending，不标 failed。
*/
private function resetIncompleteGroupToPending(array $groupRows, $msg = '')
{
$msg = mb_substr(trim((string)$msg), 0, 512);
foreach ($this->listIncompleteGroupRows($groupRows) as $gr) {
$gid = intval($gr['id']);
if ($gid <= 0) {
continue;
}
$this->updateRow($gid, [
'status' => self::RECORD_PENDING,
'error_msg' => $msg,
]);
$this->markQueueRuntime($gid, self::QUEUE_PENDING);
}
}
public function isRelevanceLlmFailureMessage($msg)
{
$msg = (string)$msg;
if ($msg === '') {
return false;
}
$needles = [
'LLM curl error',
'LLM HTTP ',
'LLM failed',
'LLM split batch',
'LLM request failed',
'LLM not configured',
'LLM exception',
'LLM response',
'Operation timed out',
'0 bytes received',
];
foreach ($needles as $n) {
if (stripos($msg, $n) !== false) {
return true;
}
}
return false;
}
/**
* 仅落库 results 里出现的编号；返回成功写入条数。
*/
private function applyPartialGroupResults(array $groupRows, array $llmResponse)
{
$results = isset($llmResponse['results']) && is_array($llmResponse['results'])
? $llmResponse['results'] : [];
if (empty($results)) {
return 0;
}
$combinedScore = floatval(isset($llmResponse['combined_relevance_score']) ? $llmResponse['combined_relevance_score'] : 0);
$combinedReason = trim((string)(isset($llmResponse['combined_reason']) ? $llmResponse['combined_reason'] : ''));
$combinedAuthorComment = $this->resolveCombinedAuthorComment(
$llmResponse,
$combinedScore,
$combinedReason
);
$claimsJson = $this->encodeClaimsJson(isset($llmResponse['claims']) ? $llmResponse['claims'] : []);
$byRef = [];
foreach ($results as $item) {
if (!is_array($item)) {
continue;
}
$refNo = intval(isset($item['reference_no']) ? $item['reference_no'] : 0);
if ($refNo > 0) {
$byRef[$refNo] = $item;
}
}
if (empty($byRef)) {
return 0;
}
$applied = 0;
foreach ($groupRows as $gr) {
$refNo = intval($gr['reference_no']);
$gid = intval($gr['id']);
if ($refNo <= 0 || $gid <= 0 || !isset($byRef[$refNo])) {
continue;
}
$freshStatus = Db::name('article_reference_relevance_check_result')
->where('id', $gid)
->value('status');
// 已完成的不覆盖
if (intval($freshStatus) === self::RECORD_COMPLETED) {
continue;
}
$item = $byRef[$refNo];
$rowCombinedScore = $combinedScore > 0
? $combinedScore
: floatval(isset($item['combined_relevance_score']) ? $item['combined_relevance_score'] : $item['relevance_score']);
$rowCombinedReason = $combinedReason !== ''
? $combinedReason
: (string)(isset($item['combined_reason']) ? $item['combined_reason'] : $item['reason']);
$rowCombinedAuthorComment = $combinedAuthorComment !== ''
? $combinedAuthorComment
: $this->resolveCombinedAuthorComment($item, $rowCombinedScore, $rowCombinedReason);
$this->updateRow($gid, [
'is_relevant' => !empty($item['is_relevant']) ? 1 : 0,
'relevance_score' => floatval($item['relevance_score']),
'reason' => (string)$item['reason'],
'author_comment' => (string)($item['author_comment'] ?? ''),
'combined_relevance_score' => $rowCombinedScore,
'combined_reason' => $rowCombinedReason,
'combined_author_comment' => $rowCombinedAuthorComment,
'claims_json' => $claimsJson,
'status' => self::RECORD_COMPLETED,
'error_msg' => '',
]);
$this->markQueueRuntime($gid, self::QUEUE_COMPLETED);
$applied++;
}
return $applied;
}
/**
* 用组内已完成行重算并回写 combined_* / claims 对齐。
*/
private function refreshGroupCombinedFields(array $groupRows, ReferenceRelevanceLlmService $llmService = null, array $combinedOverride = [])
{
DbReconnectHelper::ensure();
$ids = [];
foreach ($groupRows as $gr) {
$id = intval(isset($gr['id']) ? $gr['id'] : 0);
if ($id > 0) {
$ids[] = $id;
}
}
if (empty($ids)) {
return;
}
$freshRows = Db::name('article_reference_relevance_check_result')
->whereIn('id', $ids)
->select();
$completed = [];
foreach ($freshRows as $gr) {
if (intval($gr['status']) === self::RECORD_COMPLETED) {
$completed[] = [
'reference_no' => intval($gr['reference_no']),
'relevance_score' => floatval($gr['relevance_score']),
'is_relevant' => intval($gr['is_relevant']),
'reason' => (string)$gr['reason'],
];
}
}
if (empty($completed)) {
return;
}
if (!empty($combinedOverride)) {
// 整组程序核验的结论不可由逐篇分数反推覆盖，直接沿用
$combinedScore = floatval($combinedOverride['combined_relevance_score'] ?? 0);
$combinedReason = (string)($combinedOverride['combined_reason'] ?? '');
$combinedAuthorComment = (string)($combinedOverride['combined_author_comment'] ?? '');
if ($combinedAuthorComment === '') {
$combinedAuthorComment = $llmService === null
? (new ReferenceRelevanceLlmService())->buildCombinedAuthorCommentFromReason($combinedScore, $combinedReason)
: $llmService->buildCombinedAuthorCommentFromReason($combinedScore, $combinedReason);
}
} else {
if ($llmService === null) {
$llmService = new ReferenceRelevanceLlmService();
}
$summary = $llmService->rebuildCombinedFromResults($completed);
$combinedScore = floatval($summary['combined_relevance_score']);
$combinedReason = (string)$summary['combined_reason'];
$combinedAuthorComment = (string)($summary['combined_author_comment'] ?? '');
}
foreach ($freshRows as $gr) {
if (intval($gr['status']) !== self::RECORD_COMPLETED) {
continue;
}
$this->updateRow(intval($gr['id']), [
'combined_relevance_score' => $combinedScore,
'combined_reason' => $combinedReason,
'combined_author_comment' => $combinedAuthorComment,
]);
}
}
public function getProgressByPArticleId($pArticleId)
{
$pArticleId = intval($pArticleId);
@@ -581,8 +892,13 @@ class ReferenceRelevanceCheckService
}
$claims = $this->decodeClaimsJson(isset($row['claims_json']) ? $row['claims_json'] : '');
$author_comment = $this->resolveAuthorCommentFromRow($row, $claims);
$author_comment = $author_comment?"<b>[文献 ".intval($row['reference_no'])."]: </b>".$author_comment:$author_comment;
$combined_author_comment = $this->resolveCombinedAuthorCommentFromRow($row);
$author_comment = $this->fillAuthorCommentFromCombined(
$this->resolveAuthorCommentFromRow($row, $claims),
$combined_author_comment,
floatval($row['combined_relevance_score'] ?? 0)
);
$author_comment = $author_comment ? ("<b>[文献 " . intval($row['reference_no']) . "]: </b>" . $author_comment) : $author_comment;
$g['records'][] = [
'check_id' => intval($row['id']),
'reference_no' => intval($row['reference_no']),
@@ -595,6 +911,7 @@ class ReferenceRelevanceCheckService
'author_comment' => $author_comment,
'combined_relevance_score' => floatval($row['combined_relevance_score']),
'combined_reason' => (string)$row['combined_reason'],
'combined_author_comment' => $combined_author_comment,
'cite_group_refs' => (string)$row['cite_group_refs'],
'claims' => $claims,
'evidence_mode' => ((string)($row['score_ceiling_trigger'] ?? '') === 'no_literature_evidence')
@@ -1116,6 +1433,8 @@ class ReferenceRelevanceCheckService
'abstract_text' => $abstract,
'content_text' => $content,
'mesh_terms' => $mesh,
'pub_language' => $bundle['language'] ?? '',
'pub_country' => $bundle['country'] ?? '',
'refer_content_cleaned' => '',
'literature_pdf_url' => $pdfUrl,
'fetch_sources' => $bundle['sources'] ?? [],
@@ -1491,6 +1810,113 @@ class ReferenceRelevanceCheckService
return $this->findCitationGroupRows($row);
}
/**
* 联合引用组：仅组长（最小 reference_no）负责跑 LLM；组员等组长分块落库。
* 组长已完成但组内仍有 pending 时，允许任一 pending 行续跑剩余分块。
*/
public function shouldProcessRelevanceRowNow(array $row)
{
$groupRows = $this->findCitationGroupRows($row);
if (!$this->isCitationGroupCheck($groupRows)) {
return true;
}
$leaderRefNo = $this->resolveGroupLeaderRefNo($groupRows);
$currentRefNo = intval($row['reference_no']);
if ($currentRefNo === $leaderRefNo) {
return true;
}
foreach ($groupRows as $gr) {
if (intval($gr['reference_no']) !== $leaderRefNo) {
continue;
}
if (intval($gr['status']) === self::RECORD_COMPLETED) {
return true;
}
return false;
}
return true;
}
/**
* 组员无法处理时，若组长卡在 pending+RUNNING 则立即收回（单消费者下即僵尸）
*/
public function recoverStuckGroupLeaderForRow(array $row)
{
$groupRows = $this->findCitationGroupRows($row);
if (!$this->isCitationGroupCheck($groupRows)) {
return false;
}
$leaderRefNo = $this->resolveGroupLeaderRefNo($groupRows);
foreach ($groupRows as $gr) {
if (intval($gr['reference_no']) !== $leaderRefNo) {
continue;
}
if (intval($gr['status']) !== self::RECORD_PENDING) {
return false;
}
if (intval($gr['queue_status']) !== self::QUEUE_RUNNING) {
return false;
}
$leaderId = $this->resolveCheckRowId($gr);
if ($leaderId <= 0) {
return false;
}
$n = Db::name('article_reference_relevance_check_result')
->where('id', $leaderId)
->where('status', self::RECORD_PENDING)
->where('queue_status', self::QUEUE_RUNNING)
->update([
'queue_status' => self::QUEUE_PENDING,
'updated_at' => date('Y-m-d H:i:s'),
]);
return intval($n) > 0;
}
return false;
}
/**
* 收回卡死的 pending+RUNNING（单消费者场景下 RUNNING 即异常）
*/
public function recoverStuckRunningPendingRows($pArticleId, $staleSeconds = 180)
{
$pArticleId = intval($pArticleId);
if ($pArticleId <= 0) {
return 0;
}
$staleBefore = date('Y-m-d H:i:s', time() - max(60, intval($staleSeconds)));
return Db::name('article_reference_relevance_check_result')
->where('p_article_id', $pArticleId)
->where('status', self::RECORD_PENDING)
->where('queue_status', self::QUEUE_RUNNING)
->where('updated_at', '<', $staleBefore)
->update([
'queue_status' => self::QUEUE_PENDING,
'updated_at' => date('Y-m-d H:i:s'),
]);
}
/** 无时间门槛：收回本篇所有 pending+RUNNING（无可领取行时用） */
public function recoverAllRunningPendingRows($pArticleId)
{
$pArticleId = intval($pArticleId);
if ($pArticleId <= 0) {
return 0;
}
return Db::name('article_reference_relevance_check_result')
->where('p_article_id', $pArticleId)
->where('status', self::RECORD_PENDING)
->where('queue_status', self::QUEUE_RUNNING)
->update([
'queue_status' => self::QUEUE_PENDING,
'updated_at' => date('Y-m-d H:i:s'),
]);
}
private function findCitationGroupRows(array $row)
{
$amId = intval($row['am_id']);
@@ -1658,6 +2084,7 @@ class ReferenceRelevanceCheckService
'author_comment' => '',
'combined_relevance_score' => 0,
'combined_reason' => $reason,
'combined_author_comment' => (new ReferenceRelevanceLlmService())->buildCombinedAuthorCommentFromReason(0, $reason),
'claims_json' => '',
'status' => self::RECORD_COMPLETED,
'queue_status' => self::QUEUE_COMPLETED,
@@ -2077,6 +2504,8 @@ class ReferenceRelevanceCheckService
'abstract_text' => $abstract,
'content_text' => $contentText,
'mesh_terms' => $bundle['mesh_terms'] ?? [],
'pub_language' => $bundle['language'] ?? '',
'pub_country' => $bundle['country'] ?? '',
'refer_content_cleaned' => $cleaned,
'literature_pdf_url' => $pdfUrl,
'fetch_sources' => $bundle['sources'] ?? [],
@@ -2148,7 +2577,7 @@ class ReferenceRelevanceCheckService
if (!empty($pReferIds)) {
DbReconnectHelper::ensure();
$rows = Db::name('production_article_refer')
->field('p_refer_id,refer_type,isbn,refer_doi,doilink,refer_content,refer_frag')
->field('p_refer_id,refer_type,isbn,refer_doi,doilink,refer_content,refer_frag,joura,dateno')
->whereIn('p_refer_id', array_values($pReferIds))
->select();
foreach ($rows as $r) {
@@ -2156,42 +2585,142 @@ class ReferenceRelevanceCheckService
}
}
$metaById = [];
if (!empty($pReferIds)) {
try {
$metaRows = Db::name('production_article_refer_literature')
->field('p_refer_id,pub_language,pub_country,fetch_sources')
->whereIn('p_refer_id', array_values($pReferIds))
->select();
foreach ($metaRows ?: [] as $r) {
$metaById[intval($r['p_refer_id'])] = $r;
}
} catch (\Throwable $e) {
// 迁移未执行时字段不存在，语言/国别核验降级为「元数据缺失」
\think\Log::warning('resolveReferTypeMap literature meta unavailable: ' . $e->getMessage());
}
}
$map = [];
foreach ($groupRows as $gr) {
$refNo = intval($gr['reference_no']);
if ($refNo <= 0) {
continue;
}
$refer = $referById[intval($gr['p_refer_id'] ?? 0)] ?? [];
$type = $this->normalizeReferType($refer, trim((string)($gr['refer_text'] ?? '')));
$pReferId = intval($gr['p_refer_id'] ?? 0);
$refer = $referById[$pReferId] ?? [];
$meta = $metaById[$pReferId] ?? [];
$referText = trim((string)($gr['refer_text'] ?? ''));
$type = $this->normalizeReferType($refer, $referText);
$published = $this->resolvePublishedJournalStatus($type, $refer, $meta, $referText);
$map[$refNo] = [
'type' => $type,
'check_mode' => $type === 'book' ? 'bibliographic_inference' : ($type === 'journal' ? 'abstract_verification' : 'best_effort'),
'type' => $type,
'check_mode' => $type === 'book' ? 'bibliographic_inference' : ($type === 'journal' ? 'abstract_verification' : 'best_effort'),
'language' => strtolower(trim((string)($meta['pub_language'] ?? ''))),
'country' => strtolower(trim((string)($meta['pub_country'] ?? ''))),
'published' => $published['published'],
'published_evidence' => $published['evidence'],
];
}
return $map;
}
/**
* 「已发表期刊论文」判定：类型为 journal 尚不够，还需排除未发表状态，并有正式发表证据。
*
* @return array{published:?bool,evidence:string} published=true 已发表；false 非已发表期刊；null 证据不足
*/
private function resolvePublishedJournalStatus($type, array $refer, array $litMeta, $referTextFallback = '')
{
if ($type !== 'journal') {
return ['published' => false, 'evidence' => 'not_journal'];
}
$sourceText = trim((string)$referTextFallback);
foreach (['refer_content', 'refer_frag', 'joura', 'dateno'] as $field) {
$sourceText .= ' ' . trim((string)($refer[$field] ?? ''));
}
$sourceText = trim($sourceText);
// 明确未发表 / 投稿中 / 仅在刊前预印
if (preg_match('/\b(?:in\s+press|submitted|unpublished|manuscript|under\s+review|accepted\s+manuscript)\b/i', $sourceText)
|| preg_match('/待发表|已接收未刊|未发表|投稿中/u', $sourceText)) {
return ['published' => false, 'evidence' => 'unpublished_marker'];
}
$sources = strtolower(trim((string)($litMeta['fetch_sources'] ?? '')));
$hasPubmed = (strpos($sources, 'pubmed') !== false)
|| trim((string)($litMeta['pub_language'] ?? '')) !== '';
if ($hasPubmed) {
// PubMed 正式收录（非预印本站）即视为已发表期刊论文；预印本已在 normalizeReferType 剔为 other
return ['published' => true, 'evidence' => 'pubmed'];
}
$dateno = trim((string)($refer['dateno'] ?? ''));
$joura = trim((string)($refer['joura'] ?? ''));
$hasYear = (bool)preg_match('/(19|20)\d{2}/', $dateno !== '' ? $dateno : $sourceText);
// 正式卷期页：2021:56:103200 / 1996:23(5):1024-1029 / 2024;23(1):12-18
$hasVolumePages = (bool)preg_match(
'/(?:^|[:;.\s])\d+\s*(?:\([^)]+\))?\s*[:：]\s*\d+/',
$dateno !== '' ? $dateno : $sourceText
) || (bool)preg_match(
'/\d+\s*\(\d+\)\s*:\s*\d+/',
$dateno !== '' ? $dateno : $sourceText
);
$hasDoi = trim((string)($refer['refer_doi'] ?? '')) !== ''
|| trim((string)($refer['doilink'] ?? '')) !== ''
|| (bool)preg_match('/\b10\.\d{4,9}\//', $sourceText);
if ($joura !== '' && $hasYear && $hasVolumePages) {
return ['published' => true, 'evidence' => 'citation_volume'];
}
if ($joura !== '' && $hasYear && $hasDoi) {
// 有刊名+年份+DOI，但无卷期页：多为已正式发表的电子刊/文章编号，作已发表
return ['published' => true, 'evidence' => 'citation_doi'];
}
if ($hasYear && $hasVolumePages && $hasDoi) {
return ['published' => true, 'evidence' => 'citation_volume'];
}
// 仅有 journal 类型或仅有 DOI，不足以断言“已发表”
return ['published' => null, 'evidence' => 'insufficient'];
}
/**
* 归一化文献类型：优先取 refer_type 字段，其次按 ISBN/DOI 规则兜底。
*/
private function normalizeReferType(array $refer, $referTextFallback = '')
{
$type = strtolower(trim((string)($refer['refer_type'] ?? '')));
if ($type === 'book' || $type === 'journal') {
return $type;
$sourceText = $referTextFallback;
foreach (['refer_content', 'refer_frag'] as $field) {
$sourceText .= ' ' . trim((string)($refer[$field] ?? ''));
}
$sourceText = trim($sourceText);
$sourceLower = strtolower($sourceText);
// 预印本/会议摘要等并非“已发表期刊”。
if (strpos($sourceLower, 'medrxiv') !== false
|| strpos($sourceLower, 'biorxiv') !== false
|| strpos($sourceLower, 'arxiv') !== false
|| preg_match('/\bpreprint\b/i', $sourceText)
|| preg_match('/\bconference\b|\bproceedings\b|\bsymposium\b|\bworkshop\b/i', $sourceText)) {
return 'other';
}
if ($type === 'book') {
return 'book';
}
if ($type === 'journal') {
// refer_type 标记为 journal 但命中预印本/会议关键词时，上面已提前返回 other。
return 'journal';
}
$isbn = trim((string)($refer['isbn'] ?? ''));
$hasDoi = trim((string)($refer['refer_doi'] ?? '')) !== ''
|| trim((string)($refer['doilink'] ?? '')) !== '';
$sourceText = $referTextFallback;
foreach (['refer_content', 'refer_frag'] as $field) {
$sourceText .= ' ' . trim((string)($refer[$field] ?? ''));
}
if ($isbn !== '' || preg_match('/\bISBN\b/i', $sourceText)
|| preg_match('/\b\d+(?:st|nd|rd|th)\s+ed\.?/i', $sourceText)) {
return 'book';
@@ -2213,6 +2742,11 @@ class ReferenceRelevanceCheckService
$combinedScore = floatval(isset($llmResponse['combined_relevance_score']) ? $llmResponse['combined_relevance_score'] : 0);
$combinedReason = trim((string)(isset($llmResponse['combined_reason']) ? $llmResponse['combined_reason'] : ''));
$combinedAuthorComment = $this->resolveCombinedAuthorComment(
$llmResponse,
$combinedScore,
$combinedReason
);
$claimsJson = $this->encodeClaimsJson(isset($llmResponse['claims']) ? $llmResponse['claims'] : []);
$byRef = [];
@@ -2244,6 +2778,9 @@ class ReferenceRelevanceCheckService
$rowCombinedReason = $combinedReason !== ''
? $combinedReason
: (string)(isset($item['combined_reason']) ? $item['combined_reason'] : $item['reason']);
$rowCombinedAuthorComment = $combinedAuthorComment !== ''
? $combinedAuthorComment
: $this->resolveCombinedAuthorComment($item, $rowCombinedScore, $rowCombinedReason);
$this->updateRow(intval($gr['id']), [
'is_relevant' => !empty($item['is_relevant']) ? 1 : 0,
'relevance_score' => floatval($item['relevance_score']),
@@ -2251,6 +2788,7 @@ class ReferenceRelevanceCheckService
'author_comment' => (string)($item['author_comment'] ?? ''),
'combined_relevance_score' => $rowCombinedScore,
'combined_reason' => $rowCombinedReason,
'combined_author_comment' => $rowCombinedAuthorComment,
'claims_json' => $claimsJson,
'status' => self::RECORD_COMPLETED,
'error_msg' => '',
@@ -2284,6 +2822,9 @@ class ReferenceRelevanceCheckService
if (isset($fields['combined_reason'])) {
$fields['combined_reason'] = mb_substr(trim((string)$fields['combined_reason']), 0, 2000);
}
if (isset($fields['combined_author_comment'])) {
$fields['combined_author_comment'] = mb_substr(trim((string)$fields['combined_author_comment']), 0, 2000);
}
if (isset($fields['claims_json'])) {
$fields['claims_json'] = mb_substr(trim((string)$fields['claims_json']), 0, 4000);
}
@@ -2305,6 +2846,7 @@ class ReferenceRelevanceCheckService
'author_comment' => '',
'combined_relevance_score' => 0,
'combined_reason' => '',
'combined_author_comment' => '',
'claims_json' => '',
'score_ceiling_trigger' => '',
'error_msg' => '',
@@ -2350,8 +2892,14 @@ class ReferenceRelevanceCheckService
}
}
$author_comment = $this->resolveAuthorCommentFromRow($row, $claims);
if($author_comment){
$reason = $reason . "\n" . $author_comment;
$combined_author_comment = $this->resolveCombinedAuthorCommentFromRow($row);
$author_comment = $this->fillAuthorCommentFromCombined(
$author_comment,
$combined_author_comment,
floatval($row['combined_relevance_score'] ?? 0)
);
if ($author_comment) {
$reason = $reason . "\n" . $author_comment;
}
return [
'check_id' => intval($row['id']),
@@ -2365,6 +2913,7 @@ class ReferenceRelevanceCheckService
'author_comment' => $author_comment,
'combined_relevance_score' => floatval($row['combined_relevance_score']),
'combined_reason' => (string)$row['combined_reason'],
'combined_author_comment' => $combined_author_comment,
'cite_group_refs' => (string)$row['cite_group_refs'],
'claims' => $claims,
'evidence_mode' => ((string)($row['score_ceiling_trigger'] ?? '') === 'no_literature_evidence')
@@ -2375,6 +2924,62 @@ class ReferenceRelevanceCheckService
];
}
/**
* 联合分 <= 0.65 且单条批注为空时，用组合批注回填展示用 author_comment。
*/
private function fillAuthorCommentFromCombined($authorComment, $combinedAuthorComment, $combinedScore)
{
$authorComment = trim((string)$authorComment);
if ($authorComment !== '') {
return $authorComment;
}
if (floatval($combinedScore) > 0.65 + 0.001) {
return '';
}
$combinedAuthorComment = trim((string)$combinedAuthorComment);
return $combinedAuthorComment;
}
/**
* 联合批注：优先读库；score<=0.65 且库空时按 author_comment 规则即时生成。
*/
private function resolveCombinedAuthorCommentFromRow(array $row)
{
$stored = '';
if (array_key_exists('combined_author_comment', $row)) {
$stored = trim((string)$row['combined_author_comment']);
}
$score = floatval($row['combined_relevance_score'] ?? 0);
if ($score > 0.65 + 0.001) {
return '';
}
if ($stored !== '') {
return $stored;
}
return (new ReferenceRelevanceLlmService())->buildCombinedAuthorCommentFromReason(
$score,
(string)($row['combined_reason'] ?? '')
);
}
/**
* 从 LLM/程序结果解析 combined_author_comment；缺省时按联合分规则生成。
*/
private function resolveCombinedAuthorComment(array $payload, $combinedScore, $combinedReason)
{
$comment = trim((string)($payload['combined_author_comment'] ?? ''));
if ($comment !== '') {
return $comment;
}
return (new ReferenceRelevanceLlmService())->buildCombinedAuthorCommentFromReason(
floatval($combinedScore),
(string)$combinedReason
);
}
/**
* @param array|string $claims
*/

129
application/common/mq/ReferenceCheckArticleWorker.php
View File

@@ -66,6 +66,8 @@ class ReferenceCheckArticleWorker
$owned = true;
$finished = false;
$attemptedCheckIds = [];
$idleRecovered = false;
try {
// 续跑时强制把卡死行收回 pending，已完成行不动
$this->svc->recoverQueueRowsForArticle($pArticleId, $resume);
@@ -80,29 +82,81 @@ class ReferenceCheckArticleWorker
);
while (true) {
// 组长卡在 RUNNING 时先收回，避免 worker 误跑组员报 leader not finished
$this->svc->recoverStuckRunningPendingRows($pArticleId, 180);
$row = $this->fetchNextPendingRow($pArticleId);
if (empty($row)) {
if (!$idleRecovered) {
$n = $this->svc->recoverAllRunningPendingRows($pArticleId);
if ($n > 0) {
$idleRecovered = true;
$this->svc->log(
'ReferenceCheckArticleWorker recovered ' . $n . ' running pending rows, retry fetch'
);
continue;
}
}
break;
}
$idleRecovered = false;
$checkId = $this->svc->resolveCheckRowId($row);
if ($checkId <= 0) {
continue;
}
if (!$this->svc->shouldProcessRelevanceRowNow($row)) {
if ($this->svc->recoverStuckGroupLeaderForRow($row)) {
$this->svc->log(
'ReferenceCheckArticleWorker reset stuck group leader for ref='
. intval($row['reference_no'])
);
continue;
}
$this->svc->log(
'ReferenceCheckArticleWorker skip non-leader check_id=' . $checkId
. ' ref=' . intval($row['reference_no'])
);
continue;
}
// 同一批消息内每个 check_id 只尝试一次，避免大联合组部分成功后死循环
if (isset($attemptedCheckIds[$checkId])) {
$this->svc->log('ReferenceCheckArticleWorker stop re-entry check_id=' . $checkId);
break;
}
$attemptedCheckIds[$checkId] = true;
$this->processOneRow($checkId, $row, $trigger === 'recheck_pending_only');
// 每条结束后刷新批次心跳，长文不会被误判为僵尸
$this->touchBatch($batchId);
}
$stats = $this->summarizeArticleCheckStats($pArticleId);
$this->finalizeBatch($batchId, $stats['done'], $stats['failed'], $stats['total']);
$finished = true;
$this->svc->log(
'ReferenceCheckArticleWorker done p_article_id=' . $pArticleId
. ' batch_id=' . $batchId
. ' done=' . $stats['done']
. ' failed=' . $stats['failed']
);
$this->publishNextWaitingBatch();
if (intval($stats['pending']) > 0) {
// 分块落库后仍有缺口：回 WAITING 再投递，下轮只补 pending
Db::name('article_reference_relevance_check_batch')->where('id', intval($batchId))->update([
'batch_status' => self::BATCH_WAITING,
'done_count' => intval($stats['done']),
'failed_count' => intval($stats['failed']),
'updated_at' => date('Y-m-d H:i:s'),
]);
$finished = true;
$this->svc->log(
'ReferenceCheckArticleWorker defer incomplete p_article_id=' . $pArticleId
. ' batch_id=' . $batchId
. ' pending=' . $stats['pending']
. ' done=' . $stats['done']
);
(new ReferenceCheckMqPublisher())->publishArticleStart($pArticleId, $batchId, $trigger);
} else {
$this->finalizeBatch($batchId, $stats['done'], $stats['failed'], $stats['total']);
$finished = true;
$this->svc->log(
'ReferenceCheckArticleWorker done p_article_id=' . $pArticleId
. ' batch_id=' . $batchId
. ' done=' . $stats['done']
. ' failed=' . $stats['failed']
);
$this->publishNextWaitingBatch();
}
} catch (\Throwable $e) {
// 异常不 finalize：保持 RUNNING，靠心跳超时后由后续消息断点续跑
if ($owned) {
@@ -278,12 +332,22 @@ class ReferenceCheckArticleWorker
private function fetchNextPendingRow($pArticleId)
{
return Db::name('article_reference_relevance_check_result')
$rows = Db::name('article_reference_relevance_check_result')
->where('p_article_id', intval($pArticleId))
->where('queue_status', ReferenceRelevanceCheckService::QUEUE_PENDING)
->where('status', ReferenceRelevanceCheckService::RECORD_PENDING)
->order('reference_no asc,am_id asc,text_start asc,id asc')
->find();
->limit(100)
->select();
if (empty($rows)) {
return null;
}
foreach ($rows as $row) {
if ($this->svc->shouldProcessRelevanceRowNow($row)) {
return $row;
}
}
return null;
}
/**
@@ -306,13 +370,43 @@ class ReferenceCheckArticleWorker
$retryCount = intval(isset($row['retry_count']) ? $row['retry_count'] : 0);
try {
$this->svc->runCheckOnce($checkId, $skipLiteratureFetch);
$this->svc->markQueueRuntime($checkId, ReferenceRelevanceCheckService::QUEUE_COMPLETED, $retryCount);
DbReconnectHelper::ensure();
$fresh = Db::name('article_reference_relevance_check_result')->where('id', intval($checkId))->find();
if (empty($fresh)) {
return 'skip';
}
$st = intval($fresh['status']);
if ($st === ReferenceRelevanceCheckService::RECORD_COMPLETED) {
$this->svc->markQueueRuntime($checkId, ReferenceRelevanceCheckService::QUEUE_COMPLETED, $retryCount);
return 'ok';
}
if ($st === ReferenceRelevanceCheckService::RECORD_FAILED) {
$this->svc->markQueueRuntime($checkId, ReferenceRelevanceCheckService::QUEUE_FAILED, $retryCount);
return 'failed';
}
// 分块部分成功：本行仍 pending，留给后续消息补跑
$this->svc->markQueueRuntime($checkId, ReferenceRelevanceCheckService::QUEUE_PENDING, $retryCount);
return 'ok';
} catch (\Exception $e) {
$this->svc->log('ReferenceCheckArticleWorker check_id=' . $checkId . ' err=' . $e->getMessage());
DbReconnectHelper::ensure();
// 联合组组员被提前领取：不算失败，交还 pending 等组长
if (intval($e->getCode()) === 9001
|| strpos($e->getMessage(), 'Citation group leader not finished') !== false) {
$this->svc->markQueueRuntime($checkId, ReferenceRelevanceCheckService::QUEUE_PENDING, $retryCount);
return 'skip';
}
if ($this->svc->isRelevanceLlmFailureMessage($e->getMessage())) {
$this->svc->markQueueRuntime($checkId, ReferenceRelevanceCheckService::QUEUE_PENDING, $retryCount);
return 'ok';
}
try {
$fresh = Db::name('article_reference_relevance_check_result')->where('id', intval($checkId))->find();
if (!empty($fresh) && intval($fresh['status']) === ReferenceRelevanceCheckService::RECORD_COMPLETED) {
// 异常前已有分块落库成功，保留成果，本行按完成处理
$this->svc->markQueueRuntime($checkId, ReferenceRelevanceCheckService::QUEUE_COMPLETED, $retryCount);
return 'ok';
}
if (!empty($fresh) && intval($fresh['status']) === ReferenceRelevanceCheckService::RECORD_FAILED) {
if (intval($fresh['queue_status']) !== ReferenceRelevanceCheckService::QUEUE_FAILED) {
$this->svc->markQueueRuntime($checkId, ReferenceRelevanceCheckService::QUEUE_FAILED, $retryCount);
@@ -321,7 +415,16 @@ class ReferenceCheckArticleWorker
}
$groupRows = !empty($fresh) ? $this->svc->findCitationGroupRowsForWorker($fresh) : [];
if (!empty($groupRows)) {
$this->svc->failGroupWithQueue($groupRows, $e->getMessage(), $retryCount);
// 只失败仍未完成的行，已 completed 的分块结果保留
$incomplete = [];
foreach ($groupRows as $gr) {
if (intval($gr['status']) !== ReferenceRelevanceCheckService::RECORD_COMPLETED) {
$incomplete[] = $gr;
}
}
if (!empty($incomplete)) {
$this->svc->failGroupWithQueue($incomplete, $e->getMessage(), $retryCount);
}
} else {
$this->svc->updateCheckResult($checkId, [
'status' => ReferenceRelevanceCheckService::RECORD_FAILED,

824
application/common/service/ReferenceRelevanceLlmService.php
View File

File diff suppressed because it is too large Load Diff