database_tools中的mathpix文本预处理增加了对逗号的一些分析和预处理

工具v2/database_tools.py

@@ -958,6 +958,7 @@ def RefineMathpix(raw_string): # 进一步修改mathpix得到的字符串
     string = re.sub(r"[\s]*(``|''|\}|\{)[\s]*",lambda matchobj: matchobj.group(1),string) #去除符号前后的空格
     string = re.sub(r"(?:(?:^|\n)+[例]*[\s]*[0-9]+[\s]*[\.、\s]+|\[[\s]*例[\s]*[0-9]*[\s]*\][\s]*)","\\n\\\\item ",string) #将题号替换为\item
     string = re.sub(r"\$\$","$",string) #行间公式替换为行内公式
+    string = re.sub(r"\$\s+\$"," ",string) #删除多余的$符号
     string = re.sub(r"([,.:;?!])\$",lambda x:x.group(1)+" $",string) #标点和$符号分开
     string = re.sub(r"\\frac",r"\\dfrac",string) #替换frac为dfrac
     string = re.sub(r"\n(?:A\.|\(A\))([\s\S]*?)(?:B\.|\(B\))([\s\S]*?)(?:C\.|\(C\))([\s\S]*?)(?:D\.|\(D\))([\s\S]*?)\n",lambda matchobj: "\n\\fourch{%s}{%s}{%s}{%s}\n"%(matchobj.group(1).strip(),matchobj.group(2).strip(),matchobj.group(3).strip(),matchobj.group(4).strip()),string) # 选择题的选择支处理
@@ -983,7 +984,10 @@ def RefineMathpix(raw_string): # 进一步修改mathpix得到的字符串
     string = re.sub(r"(%[^\n]*)\\blank\{50\}\.",lambda matchobj:matchobj.group(1),string) #注释行不加\blank{50}
     string = re.sub(r"[\\\\]*\n(\(\d{1,2}\))(?:(?!\n)\s)*",lambda matchobj: "\\\\\n"+matchobj.group(1)+" ",string) #新一行的小题号回车前加上换行符
     string = re.sub(r"\(([^\(\)]*(?:\\in |=|\\ge|\\le|>|<)[^\(\)]*)\)\$",lambda matchobj: "$($" + matchobj.group(1) + "$)",string) #公式最后的范围陈述的括号放到公式环境外
+    string = SplitMathComma(string) #判断数学环境中的","是否需要用$ $分离, 如果需要则执行分离
+    string = MergeMathComma(string) #判断非数学环境中的","是否需要合并在一个数学环境中, 如果需要则执行合并
     string = RefineChineseComma(string) #改顿号
+
     return string
 
 def RefineChineseComma(string): #顿号如果在数学环境中, 则在两侧加上$符号
@@ -993,7 +997,74 @@ def RefineChineseComma(string): #顿号如果在数学环境中, 则在两侧加
         if string[:pos].count("$") % 2 == 1:
             string = string[:pos].rstrip() + "$、$" + string[(pos+1):].lstrip()
     return string
-    
+
+def SplitMathComma(string): #判断数学环境中的","是否需要用$ $分离, 如果需要则执行分离
+    UnsplittedPairs = [(r"[\(\[]",r"[\)\]]"),(r"\{",r"\}"),(r"\\begin\{cases\}",r"\\end\{cases\}")]
+    lmatter,rmatter = (string,"")
+    while "," in lmatter:
+        pos = lmatter.rfind(",")
+        if lmatter[:pos].count("$") % 2 == 1:
+            stringtemp = lmatter+rmatter
+            start = stringtemp[:pos].rfind("$")
+            end = stringtemp[pos:].find("$")+pos #寻找,两侧的数学环境 stringtemp[start:end+1]
+            locallmatter = stringtemp[start+1:pos]
+            localrmatter = stringtemp[pos+1:end]
+            tosplit = True
+            for p1,p2 in UnsplittedPairs:
+                p1l = SubstringOccurence(p1,locallmatter)
+                p1r = SubstringOccurence(p1,localrmatter)
+                p2l = SubstringOccurence(p2,locallmatter)
+                p2r = SubstringOccurence(p2,localrmatter)
+                if len(p1l)>0 and len(p2l) == 0:
+                    tosplit = False
+                if len(p2r)>0 and len(p1r) == 0:
+                    tosplit = False
+                if len(p1l)*len(p2l)>0:
+                    if p1l[-1]>p2l[-1]:
+                        tosplit = False
+                if len(p1r)*len(p2r)>0:
+                    if p1r[0]>p2r[0]:
+                        tosplit = False
+                if len(SubstringOccurence(r"(?:=|\\ge|\\le|\\in|>|<)",locallmatter))*len(SubstringOccurence(r"(?:=|\\ge|\\le|\\in|>|<)",localrmatter)) == 0:
+                    tosplit = False
+            if tosplit:
+                rmatter = ", $"+lmatter[pos+1:].lstrip()+rmatter.lstrip()
+                lmatter = lmatter[:pos]+"$"
+            else:
+                rmatter = lmatter[pos-1:]+rmatter
+                lmatter = lmatter[:pos-1]
+        else:
+            rmatter = lmatter[pos-1:]+rmatter
+            lmatter = lmatter[:pos-1]
+    return lmatter+rmatter
+
+def MergeMathComma(string): #判断非数学环境中的","是否需要合并在一个数学环境中, 如果需要则执行合并
+    UnsplittedPairs = [(r"[\(\[]",r"[\)\]]"),(r"\{",r"\}"),(r"\\begin\{cases\}",r"\\end\{cases\}")]
+    lmatter,rmatter = (string,"")
+    while not [item for item in re.finditer(r"\$,\s*\$",lmatter)] == []:
+        CommaPos = [(item.start(),item.end()) for item in re.finditer(r"\$,\s*\$",lmatter)]
+        ThePos = CommaPos[-1]
+        tempstring = lmatter+rmatter
+        lpos = tempstring[:ThePos[0]].rfind("$")
+        rpos = tempstring[ThePos[1]+1:].find("$") + ThePos[1] + 1
+        locallmatter = tempstring[lpos+1:ThePos[0]]
+        localrmatter = tempstring[ThePos[1]:rpos]
+        tomerge = False
+        for p1,p2 in UnsplittedPairs:
+            if len(SubstringOccurence(p1,locallmatter))-len(SubstringOccurence(p2,locallmatter)) == 1 and len(SubstringOccurence(p2,localrmatter))-len(SubstringOccurence(p1,localrmatter)) == 1:
+                tomerge = True
+        if tomerge:
+            rmatter = ", "+lmatter[ThePos[1]+1:]+rmatter
+            lmatter = lmatter[:ThePos[0]]
+        else:
+            rmatter = lmatter[ThePos[0]:]+rmatter
+            lmatter = lmatter[:ThePos[0]]
+    return lmatter+rmatter
+
+
+def SubstringOccurence(regex,string): #生成regex在string中出现的所有位置
+    poslist = [item.start() for item in re.finditer(regex,string)]
+    return poslist
 
 if __name__ == "__main__":
     print("数据库工具, import用.")