188 lines
7.9 KiB
Python
188 lines
7.9 KiB
Python
import json,re,os,Levenshtein,fitz
|
|
|
|
#读取存储json数据库相关(不限于题号数据库)
|
|
|
|
def load_dict(filename): #根据filename读取json数据库并转化为python字典
|
|
with open(filename,"r",encoding = "u8") as f:
|
|
adict = json.loads(f.read())
|
|
return adict #返回python字典
|
|
|
|
def save_dict(adict,filename): #将adict字典转化为json文件并保存至filename文件中
|
|
try:
|
|
with open(filename,"w",encoding = "u8") as f:
|
|
f.write(json.dumps(adict,indent=4,ensure_ascii=False))
|
|
return 0 #成功则返回0
|
|
except:
|
|
return 1 #不成功则返回1
|
|
|
|
|
|
def pre_treating(string): #删除字符串中对比较无用的字符, 以供比较
|
|
string = re.sub(r"\\begin\{center\}[\s\S]*?\\end\{center\}","",string)
|
|
string = re.sub(r"(bracket\{\d+\})|(blank\{\d+\})|(fourch)|(twoch)|(onech)","",string)
|
|
string = re.sub(r"[\s\\\{\}\$\(\)\[\]]","",string)
|
|
string = re.sub(r"[\n\t]","",string)
|
|
string = re.sub(r"(displaystyle)|(overrightarrow)|(overline)","",string)
|
|
string = re.sub(r"[,\.:;?]","",string)
|
|
return string #返回处理后的字符串
|
|
|
|
def treat_dict(p_dict): #对整个题库字典中的内容部分进行预处理，删除无用字符
|
|
treated_dict = {}
|
|
for id in p_dict:
|
|
treated_dict[id] = {}
|
|
treated_dict[id]["content"] = pre_treating(p_dict[id]["content"])
|
|
treated_dict[id]["same"] = p_dict[id]["same"]
|
|
return treated_dict #返回处理后的字典, 含内容字段及相同题目字段
|
|
|
|
def detectmaxsim(currentid,excludelist,adict): #检测与已知题目关联程度最大的题目（除外列表之外的部分）
|
|
maxsim = -1
|
|
argmaxsim = "000000"
|
|
for id in adict:
|
|
if not id in excludelist:
|
|
simrate = Levenshtein.jaro(adict[id]["content"],adict[currentid]["content"])
|
|
if simrate > maxsim:
|
|
maxsim = simrate
|
|
argmaxsim = id
|
|
return (maxsim,argmaxsim) #返回最大关联系数与关联程度最大的题号
|
|
|
|
def generate_problem_series(startingid,length,adict): #在adict字典里返回从startingid开始的一系列题号, 每一题都是与上一题的关联程度最大的
|
|
excludelist = [startingid]
|
|
currentid = startingid
|
|
for i in range(length):
|
|
maxsim,currentid = detectmaxsim(currentid,excludelist,adict)
|
|
excludelist.append(currentid)
|
|
return ",".join(excludelist) #返回按顺序的题号列表
|
|
|
|
|
|
def generate_number_set(string): #根据可能含有":"和","的题号字符串生成一个用逗号分隔的六位题号列表, 例如"1:3,5"会生成["000001","000002","000003","000005"]
|
|
string = re.sub(r"[\n\s]","",string)
|
|
string_list = string.split(",")
|
|
numbers_list = []
|
|
for s in string_list:
|
|
if not ":" in s:
|
|
numbers_list.append(s.zfill(6))
|
|
else:
|
|
start,end = s.split(":")
|
|
for ind in range(int(start),int(end)+1):
|
|
numbers_list.append(str(ind).zfill(6))
|
|
return numbers_list #返回六位题号列表
|
|
|
|
def generate_exp(id_list): #根据题号列表生成字符串式的含":"和","的题号字符串, 例如["000001","000002","000003","000005"]生成"000001:000003,000005", 若列表为空则生成"无有效题号"
|
|
if not len(id_list) == 0:
|
|
exp_list = []
|
|
start = id_list[0]
|
|
current = start
|
|
end = start
|
|
for id in id_list[1:]:
|
|
# print(id,current)
|
|
if int(id)-1 == int(current):
|
|
current = id
|
|
end = id
|
|
else:
|
|
if not start == end:
|
|
exp_list.append('"'+start+":"+end+'"')
|
|
else:
|
|
exp_list.append('"'+start+'"')
|
|
start = id
|
|
current = id
|
|
end = id
|
|
if not start == end:
|
|
exp_list.append('"'+start+":"+end+'"')
|
|
else:
|
|
exp_list.append('"'+start+'"')
|
|
exp_str = ",".join(exp_list).replace('"',"")
|
|
else:
|
|
exp_str = "无有效题号"
|
|
return exp_str #返回含有":"或","的题号字符串
|
|
|
|
def parsePDF(filePath): #提取pdf文件中的字符
|
|
with fitz.open(filePath) as doc:
|
|
text = ""
|
|
for page in doc.pages():
|
|
text += page.get_text() + "\n"
|
|
return text
|
|
|
|
def extractIDs(filePath): #提取.txt,.tex或.pdf文件中的题号, 返回含有":"或","的题号字符串
|
|
if filePath[-4:] == ".txt" or filePath[-4:] == ".tex":
|
|
with open(filePath,"r",encoding = "u8") as f:
|
|
data = f.read()
|
|
elif filePath[-4:] == ".pdf":
|
|
data = parsePDF(filePath)
|
|
else:
|
|
return "格式不正确"
|
|
ids = re.findall(r"\((\d{6})\)",data)
|
|
return generate_exp(ids)
|
|
|
|
|
|
def spareIDs(dictname): #返回空闲题号
|
|
idlist = list(dictname.keys())
|
|
used_str = generate_exp(idlist)
|
|
used_list = used_str.split(",")
|
|
output = ""
|
|
for group in range(len(used_list)-1):
|
|
output += "首个空闲id: %s, 直至: %s"%(str(int(used_list[group][-6:])+1).zfill(6),str(int(used_list[group+1][:6])-1).zfill(6)) + "\n"
|
|
output += "首个空闲id: %s, 直至: %s"%(str(int(used_list[-1][-6:])+1).zfill(6),"999999")
|
|
return output #返回的是一个多行的字符串, 每一行中含有一个空闲题号的闭区间
|
|
|
|
def parse_usage(datastring): #对单个usages中的项的结果进行分词
|
|
datastring = re.sub(r"\s+","\t",datastring.strip())
|
|
datalist = datastring.split("\t")
|
|
date = ""
|
|
classname = ""
|
|
diff = []
|
|
for item in datalist:
|
|
if not "." in item and not "高" in item and not "班" in item:
|
|
date = item
|
|
elif "高" in item or "班" in item:
|
|
classname = item
|
|
else:
|
|
diff.append(item)
|
|
return({"date":date,"classname":classname,"difficulty":diff}) #返回一个字典, "date"表示日期, "classname"表示班级, "difficultiy"表示难度列表
|
|
|
|
|
|
def GenerateProblemListFromString(data): #从来自.tex文件的字符串生成题目列表, 每个item是一道题目, 新一行的%用作前缀
|
|
try:
|
|
data = re.findall(r"\\begin\{document\}([\s\S]*?)\\end\{document\}",data)[0]
|
|
except:
|
|
pass
|
|
data = re.sub(r"\n{2,}","\n",data)
|
|
data = re.sub(r"\\item",r"\\enditem\\item",data)
|
|
data = re.sub(r"\\end\{enumerate\}",r"\\enditem",data) #切除无关信息, 保留关键信息
|
|
problempositions = []
|
|
for item in re.finditer(r"\\item([\s\S]*?)\\enditem",data):
|
|
problempositions.append(item.regs[1]) #确定题目内容所在位置
|
|
problem_list = []
|
|
for pos in problempositions:
|
|
content = data[pos[0]:pos[1]].strip()
|
|
content = re.sub(r"\n\%[\s\S]*$","",content) #题目内容
|
|
subdata = data[:pos[0]] #开始寻找出处中缀
|
|
suflist = re.findall(r"\n(\%\s{0,}[\S]+)\n",subdata)
|
|
if len(suflist) == 0:
|
|
suffix = ""
|
|
else:
|
|
suffix = suflist[-1].replace("%","").strip()
|
|
problem_list.append((content,suffix))
|
|
return problem_list #返回一个列表, 每一项是一个由 题目内容 和 题目来源前缀 组成的元组
|
|
|
|
|
|
def CreateEmptyProblem(problem): # 根据已有的题目创建新的空题目
|
|
NewProblem = problem.copy()
|
|
for field in NewProblem:
|
|
if type(NewProblem[field]) == str:
|
|
NewProblem[field] = ""
|
|
elif type(NewProblem[field]) == list:
|
|
NewProblem[field] = []
|
|
elif type(NewProblem[field]) == int or type(NewProblem[field]) == float:
|
|
NewProblem[field] = -1
|
|
return NewProblem #返回一个空题目的字典, ID和内容待赋值
|
|
|
|
# 创建新题目
|
|
def CreateNewProblem(id,content,origin,dict,editor): # 构建一道新题目的字典
|
|
NewProblem = CreateEmptyProblem(dict["000001"])
|
|
NewProblem["id"] = str(id).zfill(6)
|
|
NewProblem["content"] = content
|
|
NewProblem["origin"] = origin
|
|
NewProblem["edit"] = [editor]
|
|
return NewProblem # 返回一道新题目的字典, 已赋新的ID, 内容, 来源和编辑者
|
|
|
|
if __name__ == "__main__":
|
|
print("数据库工具, import用.") |