zm/tougao
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

自动查重

Browse Source
This commit is contained in:
wangjinlei
2026-05-20 14:46:58 +08:00
parent cfa3f791f4
commit 4940db73fe
5 changed files with 290 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

256
application/common/ManuscriptBodyExtractor.php
View File

@@ -2,19 +2,33 @@
namespace app\common;
use PhpOffice\PhpWord\IOFactory;
use PhpOffice\PhpWord\PhpWord;
use DOMDocument;
use DOMElement;
use DOMXPath;
use think\Exception;
use ZipArchive;
/**
* 从投稿 Word 稿件生成「仅正文」版本：去掉文前题名/作者/单位等，去掉参考文献及之后内容。
* 从投稿 Word 生成「仅正文」docx：在 document.xml 上按块裁切，保留表格/图片/样式；
* 边界识别仅用可见文本（w:t），不读取域指令（Zotero/EndNote 的 JSON）。
*/
class ManuscriptBodyExtractor
{
const BODY_SUBDIR = 'public/plagiarism/body_only';
const W_NS = 'http://schemas.openxmlformats.org/wordprocessingml/2006/main';
/** @var DOMDocument */
private $dom;
/** @var DOMElement */
private $bodyNode;
/** @var array<int,DOMElement> */
private $blocks = [];
/** @var array<int,string> */
private $lines = [];
private $blockTexts = [];
/**
* @return array{path:string, rel_path:string, line_count:int, ref_start:int, body_start:int, warnings:array}
@@ -30,9 +44,9 @@ class ManuscriptBodyExtractor
throw new Exception('body_only check requires DOCX manuscript, got: ' . $ext);
}
$this->lines = ArticleParserService::collectParagraphLines($sourcePath);
if (empty($this->lines)) {
throw new Exception('No text extracted from manuscript');
$this->loadDocumentBlocks($sourcePath);
if (empty($this->blocks)) {
throw new Exception('No content blocks in manuscript');
}
$refStart = $this->findReferenceStartIndex();
@@ -41,32 +55,181 @@ class ManuscriptBodyExtractor
if ($refStart < 0) {
$warnings[] = 'references_heading_not_found; using document end';
$refStart = count($this->lines);
$refStart = count($this->blocks);
}
if ($bodyStart >= $refStart) {
throw new Exception('Could not locate main body (front matter may include entire document)');
}
$bodyLines = array_slice($this->lines, $bodyStart, $refStart - $bodyStart);
$bodyLines = $this->normalizeBodyLines($bodyLines);
if (count($bodyLines) < 3) {
throw new Exception('Body text too short after extraction (' . count($bodyLines) . ' paragraphs)');
$kept = 0;
for ($i = $bodyStart; $i < $refStart; $i++) {
if (trim($this->blockTexts[$i]) !== '') {
$kept++;
}
}
if ($kept < 3) {
throw new Exception('Body content too short after extraction (' . $kept . ' non-empty blocks)');
}
$relPath = $this->writeBodyDocx($bodyLines, $articleId);
$relPath = $this->sliceDocxToNewFile($sourcePath, $articleId, $bodyStart, $refStart);
$rootDir = rtrim(ROOT_PATH ?: dirname(dirname(__DIR__)), '/\\');
$absPath = $rootDir . DIRECTORY_SEPARATOR . str_replace(['/', '\\'], DIRECTORY_SEPARATOR, $relPath);
return [
'path' => $absPath,
'rel_path' => $relPath,
'line_count' => count($bodyLines),
'line_count' => $kept,
'ref_start' => $refStart,
'body_start' => $bodyStart,
'warnings' => $warnings,
];
}
private function loadDocumentBlocks($sourcePath)
{
$zip = new ZipArchive();
if ($zip->open($sourcePath) !== true) {
throw new Exception('Cannot open docx: ' . $sourcePath);
}
$xml = $zip->getFromName('word/document.xml');
$zip->close();
if ($xml === false || $xml === '') {
throw new Exception('word/document.xml missing in docx');
}
$this->dom = new DOMDocument();
$this->dom->preserveWhiteSpace = false;
$this->dom->formatOutput = false;
if (@$this->dom->loadXML($xml) === false) {
throw new Exception('Invalid word/document.xml');
}
$xpath = new DOMXPath($this->dom);
$xpath->registerNamespace('w', self::W_NS);
$body = $xpath->query('//w:body')->item(0);
if (!$body instanceof DOMElement) {
throw new Exception('w:body not found');
}
$this->bodyNode = $body;
$this->blocks = [];
$this->blockTexts = [];
foreach ($body->childNodes as $child) {
if ($child->nodeType !== XML_ELEMENT_NODE) {
continue;
}
/** @var DOMElement $child */
if ($child->localName === 'sectPr') {
continue;
}
$this->blocks[] = $child;
$this->blockTexts[] = $this->extractVisibleTextFromBlock($child);
}
}
/**
* 仅拼接 w:t 可见文本，忽略 w:instrText 等域指令（避免 Zotero JSON 参与裁切判断）。
*/
private function extractVisibleTextFromBlock(DOMElement $block)
{
$xpath = new DOMXPath($block->ownerDocument);
$xpath->registerNamespace('w', self::W_NS);
$nodes = $xpath->query('.//w:t', $block);
if (!$nodes || $nodes->length === 0) {
return '';
}
$parts = [];
foreach ($nodes as $node) {
$parts[] = $node->textContent;
}
$text = preg_replace('/\s+/u', ' ', implode('', $parts));
return trim((string) $text);
}
private function sliceDocxToNewFile($sourcePath, $articleId, $bodyStart, $refStart)
{
$rootDir = rtrim(ROOT_PATH ?: dirname(dirname(__DIR__)), '/\\');
$dir = $rootDir . DIRECTORY_SEPARATOR . self::BODY_SUBDIR;
if (!is_dir($dir)) {
@mkdir($dir, 0755, true);
}
$name = sprintf('body_article_%d_%s.docx', intval($articleId), date('Ymd_His'));
$absPath = $dir . DIRECTORY_SEPARATOR . $name;
if (!copy($sourcePath, $absPath)) {
throw new Exception('Failed to copy source docx');
}
$n = count($this->blocks);
$zip = new ZipArchive();
if ($zip->open($absPath) !== true) {
throw new Exception('Cannot open output docx');
}
$xml = $zip->getFromName('word/document.xml');
if ($xml === false) {
$zip->close();
throw new Exception('document.xml missing in output docx');
}
$outDom = new DOMDocument();
$outDom->preserveWhiteSpace = false;
$outDom->formatOutput = false;
if (@$outDom->loadXML($xml) === false) {
$zip->close();
throw new Exception('Invalid document.xml in output docx');
}
$xpath = new DOMXPath($outDom);
$xpath->registerNamespace('w', self::W_NS);
$body = $xpath->query('//w:body')->item(0);
if (!$body instanceof DOMElement) {
$zip->close();
throw new Exception('w:body not found in output docx');
}
$children = [];
foreach ($body->childNodes as $child) {
if ($child->nodeType === XML_ELEMENT_NODE) {
$children[] = $child;
}
}
$blockIdx = 0;
foreach ($children as $child) {
if (!($child instanceof DOMElement)) {
continue;
}
if ($child->localName === 'sectPr') {
continue;
}
if ($blockIdx < $bodyStart || $blockIdx >= $refStart) {
if ($child->parentNode) {
$child->parentNode->removeChild($child);
}
}
$blockIdx++;
}
if ($blockIdx !== $n) {
$zip->close();
@unlink($absPath);
throw new Exception('Document block count mismatch during slice');
}
$zip->addFromString('word/document.xml', $outDom->saveXML());
$zip->close();
if (!is_file($absPath) || filesize($absPath) < 200) {
throw new Exception('Failed to write body-only docx');
}
return self::BODY_SUBDIR . '/' . $name;
}
private function findReferenceStartIndex()
{
$stopKeywords = [
@@ -75,7 +238,7 @@ class ManuscriptBodyExtractor
'致谢', '基金', '附录', '补充材料', '利益冲突', '作者贡献',
];
foreach ($this->lines as $i => $line) {
foreach ($this->blockTexts as $i => $line) {
$t = trim($line);
if ($t === '') {
continue;
@@ -87,7 +250,7 @@ class ManuscriptBodyExtractor
foreach ($stopKeywords as $sk) {
$skLower = strtolower($sk);
if ($lower === $skLower || $lower === $skLower . ':' || $lower === $skLower . '：') {
if ($i > count($this->lines) * 0.4) {
if ($i > count($this->blockTexts) * 0.4) {
return $i;
}
}
@@ -98,12 +261,12 @@ class ManuscriptBodyExtractor
private function findBodyStartIndex()
{
$n = count($this->lines);
$n = count($this->blockTexts);
$introIdx = -1;
$keywordsIdx = -1;
for ($i = 0; $i < $n; $i++) {
$t = trim($this->lines[$i]);
$t = trim($this->blockTexts[$i]);
if ($t === '') {
continue;
}
@@ -145,9 +308,9 @@ class ManuscriptBodyExtractor
private function indexAfterKeywordsBlock($kwIdx)
{
$n = count($this->lines);
$n = count($this->blockTexts);
for ($i = $kwIdx + 1; $i < $n; $i++) {
$t = trim($this->lines[$i]);
$t = trim($this->blockTexts[$i]);
if ($t === '') {
continue;
}
@@ -166,10 +329,10 @@ class ManuscriptBodyExtractor
private function indexAfterFrontMatterFallback()
{
$n = count($this->lines);
$n = count($this->blockTexts);
$maxSkip = min(20, (int) floor($n * 0.15));
for ($i = 0; $i < $maxSkip && $i < $n; $i++) {
$t = trim($this->lines[$i]);
$t = trim($this->blockTexts[$i]);
if ($t === '') {
continue;
}
@@ -190,53 +353,4 @@ class ManuscriptBodyExtractor
}
return false;
}
/**
* @param array<int,string> $bodyLines
* @return array<int,string>
*/
private function normalizeBodyLines(array $bodyLines)
{
$out = [];
foreach ($bodyLines as $line) {
$line = trim($line);
if ($line === '') {
continue;
}
if (preg_match('/^\{\s*ADDIN\s+EN\.REFLIST\s*\}$/i', $line)) {
continue;
}
$out[] = $line;
}
return $out;
}
/**
* @param array<int,string> $bodyLines
*/
private function writeBodyDocx(array $bodyLines, $articleId)
{
$rootDir = rtrim(ROOT_PATH ?: dirname(dirname(__DIR__)), '/\\');
$dir = $rootDir . DIRECTORY_SEPARATOR . self::BODY_SUBDIR;
if (!is_dir($dir)) {
@mkdir($dir, 0755, true);
}
$name = sprintf('body_article_%d_%s.docx', intval($articleId), date('Ymd_His'));
$absPath = $dir . DIRECTORY_SEPARATOR . $name;
$phpWord = new PhpWord();
$section = $phpWord->addSection();
foreach ($bodyLines as $line) {
$section->addText($line);
}
$writer = IOFactory::createWriter($phpWord, 'Word2007');
$writer->save($absPath);
if (!is_file($absPath) || filesize($absPath) < 200) {
throw new Exception('Failed to write body-only docx');
}
return self::BODY_SUBDIR . '/' . $name;
}
}