导演：陈凯歌
主演：王宝强 郭富城 张震 林志玲 范伟 元华 王学圻
"] labelid = 0 new_doc = [] for i in range(len(new_docs)): doc =
"] labelid = 0 new_doc = [] for i in range(len(new_docs)): doc = new_docs[i] doc = re.sub(r'【[^】]*】', '', doc) doc = doc.strip() new_doc.append(doc) with open(outputpath, 'w', encoding="utf-8") as f: for i in range(len(new_doc)): print(json.dumps({'labelid': labelid, 'doc': new_doc[i]}, ensure_ascii=False), file=f)def load_dict(): # 字典加载，用于初始化字典 dict_path = get_config_dict().get('dict_path') dict = [] with open(dict_path, 'r', encoding='utf-8') as f: for line in f: dict.append(line.strip("\n\r")) return dictdef load_word(): # 字典加载，用于初始化字典 word_path = get_config_dict().get('word_path') word = [] with open(word_path, 'r', encoding='utf-8') as f: for line in f: word.append(line.strip("\n\r")) return worddef get_sy(): dict = load_dict() word = load_word() sy = set() for str1 in dict: str1 = str1.strip(" ") if str1 in word: sy.add(str1) return sydef get_new_synonym(str1,str2): # str1转为list # str2转为list # 将str2组合到str1中 list1 = str1.split(" ") list2 = str2.split(" ") new_word = [] for i in range(len(list1)-1): new_word.append(list1[i]) for i in range(len(list2)): new_word.append(list2[i]) return " ".join(new_word)def synonyms_process(str1): dict1 = load_dict() word = load_word() # str1与word求差，得到不在字典中的同义词 str1 = str1.strip(" ") list1 = str1.split(" ") diff = list(set(list1).difference(set(word))) if len(diff) == 0: return str1 else: #list1与diff求交集 inter_list = list(set(list1).intersection(set(diff))) if len(inter_list) == 1: # 先匹配单词 if str1.index(inter_list[0]) == 0: return dict1.get(inter_list[0]) else: prefix = str1[0:str1.index(inter_list[0])-1] return prefix dict1.get(inter_list[0]) else: # 多个词匹配成一个词 # 获取这些交叉词的后缀 after = [] for i in range(len(inter_list)): after.append(inter_list[i][len(inter_list[i])-1:len(inter_list[i])]) # 求交叉词的并集 for i in range(len(after)): insert = inter_list[i][0:len(inter_list[i])-1] if not insert in inter_list: inter_list.append(insert) inter_set = set(inter_list) dif = list(set(after).difference(inter_set)) str2 = inter_list[0][0:len(inter_list[0])-1] " ".join(dif) str3 = get_new_synonym(inter_list[0],str2) return str3if __name__ == '__main__': #get_summary() #get_keyword() # print(get_full_path("dict1.txt")) # print(synonyms_process("小公司 乱点 点餐 自助")) # print(synonyms_process("成立")) file_name = r"E:\数据集\txt分类数据集\文本分类\text\C4-Fashion.txt" docs = read_file(file_name) with open(r"E:\数据集\txt分类数据集\文本分类\text\C4-Fashion.txt", 'w', encoding="utf-8") as f: for i in range(len(docs)): doc = docs[i] doc = doc.strip() print(doc, file=f)
详情
更多影视：
影林影视
