{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0.789\t1\t005886\n", "0.805\t2\t006207\n", "0.812\t3\t006137\n", "0.798\t4\t012269\n", "0.831\t5\t040063\n", "0.888\t6\t021503\n", "0.932\t7\t040057\n", "0.959\t8\t003206\n", "0.817\t9\t011090\n", "0.796\t10\t003058\n", "0.768\t11\t005984\n", "0.744\t12\t021463\n", "0.872\t13\t008169\n", "0.843\t14\t021559\n", "0.758\t15\t006230\n", "0.888\t16\t008317\n", "0.682\t17\t012005\n", "0.630\t18\t006107\n", "0.765\t19\t021581\n", "0.763\t20\t014242\n", "0.661\t21\t013849\n", "0.716\t22\t008191\n", "0.504\t23\t003537\n", "0.934\t24\t000818\n", "0.755\t25\t012596\n", "0.893\t26\t012355\n", "0.835\t27\t000479\n", "0.950\t28\t001529\n", "0.797\t29\t000577\n", "0.737\t30\t003191\n", "1.000\t31\t012594\n", "0.824\t32\t003150\n", "1.000\t33\t004066\n", "0.707\t34\t003140\n", "0.648\t35\t030294\n", "0.752\t36\t014186\n", "0.621\t37\t014223\n", "0.700\t38\t004442\n", "0.813\t39\t030808\n", "0.669\t40\t013843\n", "0.654\t41\t000738\n", "0.682\t42\t013354\n", "0.830\t43\t030042\n", "0.977\t44\t020411\n", "0.661\t45\t011133\n", "0.681\t46\t004118\n", "0.667\t47\t011943\n", "0.683\t48\t005600\n", "0.691\t49\t011386\n", "0.711\t50\t002922\n", "1.000\t51\t020452\n", "0.736\t52\t020356\n", "0.666\t53\t020430\n", "0.685\t54\t020357\n", "0.713\t55\t005540\n", "0.643\t56\t012868\n", "0.845\t57\t005598\n", "0.860\t58\t002958\n", "0.646\t59\t004184\n" ] } ], "source": [ "import os,re,difflib,Levenshtein,time,json\n", "\n", "# 重要!!! 范围\n", "old_problems_range = \"1:999999\"\n", "threshold = 0.85\n", "\n", "# 待比对的文件\n", "filename = r\"C:\\Users\\weiye\\Documents\\wwy sync\\临时工作区\\自拟题目9.tex\"\n", "\n", "#生成数码列表, 逗号分隔每个区块, 区块内部用:表示整数闭区间\n", "def generate_number_set(string):\n", " string = re.sub(r\"[\\n\\s]\",\"\",string)\n", " string_list = string.split(\",\")\n", " numbers_list = []\n", " for s in string_list:\n", " if not \":\" in s:\n", " numbers_list.append(s.zfill(6))\n", " else:\n", " start,end = s.split(\":\")\n", " for ind in range(int(start),int(end)+1):\n", " numbers_list.append(str(ind).zfill(6))\n", " return numbers_list\n", "\n", "#字符串预处理\n", "def pre_treating(string):\n", " string = re.sub(r\"\\\\begin\\{center\\}[\\s\\S]*?\\\\end\\{center\\}\",\"\",string)\n", " string = re.sub(r\"(bracket\\{\\d+\\})|(blank\\{\\d+\\})|(fourch)|(twoch)|(onech)|(mathrm)|(text)\",\"\",string)\n", " string = re.sub(r\"[\\s\\\\\\{\\}\\$\\(\\)\\[\\]]\",\"\",string)\n", " string = re.sub(r\"[\\n\\t]\",\"\",string)\n", " string = re.sub(r\"(displaystyle)|(overrightarrow)\",\"\",string)\n", " string = re.sub(r\"[,\\.:;?]\",\"\",string)\n", " return string\n", "\n", "#difflab字符串比较\n", "def difflab_get_equal_rate(str1, str2):\n", " return difflib.SequenceMatcher(None, str1, str2).ratio()\n", "\n", "#Levenshtein jaro字符串比较\n", "def jaro_get_equal_rate(str1,str2):\n", " return Levenshtein.jaro(str1,str2)\n", "\n", "#Levenshtein 字符串比较\n", "def Lev_get_equal_rate(str1,str2):\n", " return Levenshtein.ratio(str1,str2)\n", "\n", "def GenerateProblemListFromString(data):\n", " try:\n", " data = re.findall(r\"\\\\begin\\{document\\}([\\s\\S]*?)\\\\end\\{document\\}\",problems_string)[0]\n", " except:\n", " pass\n", " data = re.sub(r\"\\n{2,}\",\"\\n\",data)\n", " data = re.sub(r\"\\\\item\",r\"\\\\enditem\\\\item\",data)\n", " data = re.sub(r\"\\\\end\\{enumerate\\}\",r\"\\\\enditem\",data)\n", " ProblemList_raw = [p.strip() for p in re.findall(r\"\\\\item([\\s\\S]*?)\\\\enditem\",data)]\n", " ProblemsList = []\n", " for p in ProblemList_raw:\n", " startpos = data.index(p)\n", " tempdata = data[:startpos]\n", " suflist = re.findall(r\"\\n\\%[\\dA-Za-z]+\",tempdata)\n", " if len(suflist) > 0:\n", " suffix = suflist[-1].replace(\"%\",\"\").strip()\n", " else:\n", " suffix = \"\"\n", " ProblemsList.append((p,suffix))\n", " return ProblemsList\n", "\n", "\n", "#指定对比方法\n", "sim_test = jaro_get_equal_rate\n", "\n", "#读入题库\n", "with open(r\"../题库0.3/Problems.json\",\"r\",encoding = \"utf8\") as f:\n", " database = f.read()\n", "pro_dict = json.loads(database)\n", "\n", "with open(filename,\"r\",encoding=\"u8\") as f:\n", " newdatabase = f.read()\n", "new_pro_list = GenerateProblemListFromString(newdatabase)\n", "\n", "pro_dict_treated = {}\n", "idrange_raw = generate_number_set(old_problems_range)\n", "idrange = [id for id in pro_dict if id in idrange_raw]\n", "for p in idrange:\n", " pro_dict_treated[p] = pre_treating(pro_dict[p][\"content\"])\n", "\n", "new_dict_treated = {}\n", "for i in range(len(new_pro_list)):\n", " new_dict_treated[i+1] = pre_treating(new_pro_list[i][0])\n", "\n", "for i in new_dict_treated:\n", " new_p = new_dict_treated[i]\n", " maxsim = 0\n", " for p in pro_dict_treated:\n", " old_p = pro_dict_treated[p]\n", " sim = sim_test(new_p,old_p)\n", " if sim > maxsim:\n", " maxsim = sim\n", " argmax = p\n", " print(\"%.3f\\t%d\\t%s\" %(maxsim,i,argmax))\n", " # print(\"\\n新题: %s\" %new_pro_list[i-1][0])\n", " # print(\"\\n原题: %s\\n\\n\\n\" %pro_dict[][\"content\"])\n", "\n" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "999999" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(idrange)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "mathdept", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.15" }, "orig_nbformat": 4, "vscode": { "interpreter": { "hash": "ff3c292c316ba85de6f1ad75f19c731e79d694e741b6f515ec18f14996fe48dc" } } }, "nbformat": 4, "nbformat_minor": 2 }