论文降重算法代码详解

# Python实现简单的文本相似度计算 import jieba import numpy as np from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.metrics.pairwise import cosine_similarity def calculate_similarity(text1, text2): """计算两段文本的余弦相似度""" # 中文分词处理 words1 = " ".join(jieba.cut(text1)) words2 = " ".join(jieba.cut(text2)) # TF-IDF向量化 vectorizer = TfidfVectorizer() tfidf_matrix = vectorizer.fit_transform([words1, words2]) # 计算余弦相似度 similarity = cosine_similarity(tfidf_matrix[0:1], tfidf_matrix[1:2])[0][0] return similarity # 使用示例 original_text = "人工智能技术在教育领域具有广阔的应用前景" rewritten_text = "AI技术在教育教学方面展现出巨大的发展潜力" similarity = calculate_similarity(original_text, rewritten_text) print(f"文本相似度: {similarity:.3f}")

# 基于同义词的简单语义替换算法 class SemanticReplacer: def __init__(self): self.synonym_dict = { '研究': ['探讨', '分析', '调查', '考察', '钻研'], '发展': ['进展', '进步', '演变', '推进', '成长'], '技术': ['手段', '方法', '技艺', '技能', '工艺'], '应用': ['运用', '使用', '采用', '利用', '施行'] } def replace_with_synonyms(self, text): """使用同义词进行文本替换""" words = list(jieba.cut(text)) replaced_words = [] for word in words: if word in self.synonym_dict and np.random.random() > 0.5: # 随机选择一个同义词进行替换 synonym = np.random.choice(self.synonym_dict[word]) replaced_words.append(synonym) else: replaced_words.append(word) return ''.join(replaced_words) # 使用示例 replacer = SemanticReplacer() original = "人工智能技术发展迅速，在各个领域得到广泛应用" rewritten = replacer.replace_with_synonyms(original) print(f"原文: {original}") print(f"改写: {rewritten}")

# 基于Hugging Face Transformers的文本改写示例 from transformers import T5ForConditionalGeneration, T5Tokenizer import torch class TransformerParaphraser: def __init__(self, model_name='t5-base'): self.tokenizer = T5Tokenizer.from_pretrained(model_name) self.model = T5ForConditionalGeneration.from_pretrained(model_name) self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') self.model.to(self.device) def paraphrase(self, text, num_beams=5, num_return_sequences=3): """使用T5模型进行文本改写""" # 准备输入格式 input_text = f"paraphrase: {text}" inputs = self.tokenizer.encode(input_text, return_tensors='pt').to(self.device) # 生成改写结果 outputs = self.model.generate( inputs, max_length=128, num_beams=num_beams, num_return_sequences=num_return_sequences, temperature=0.8, do_sample=True ) # 解码结果 paraphrases = [] for output in outputs: paraphrase = self.tokenizer.decode(output, skip_special_tokens=True) paraphrases.append(paraphrase) return paraphrases # 初始化改写器 paraphraser = TransformerParaphraser() # 使用示例 original_text = "机器学习算法能够从大量数据中自动学习模式和规律" paraphrased_results = paraphraser.paraphrase(original_text) for i, result in enumerate(paraphrased_results, 1): print(f"改写版本{i}: {result}")