Skip to content

Commit

Permalink
modified: automated-issue-ref.ipynb
Browse files Browse the repository at this point in the history 
	modified:   autoref/milestone.txt
	modified:   autoref/milestone_sub.txt
	modified:   autoref/missing_refs.txt
  • Loading branch information
@xz-keg
xz-keg committed Jul 28, 2023
1 parent 8881481 commit 502c1d1
Show file tree
Hide file tree
Showing 4 changed files with 254 additions and 38 deletions.
179 changes: 145 additions & 34 deletions automated-issue-ref.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 41,
"id": "f64573a2",
"metadata": {},
"outputs": [],
Expand All @@ -15,9 +15,9 @@
"# the number of milestone we start to read\n",
"start_milestone=1\n",
"start_sub_milestone=1\n",
"end_milestone=795\n",
"end_sub_milestone=13\n",
"def read_milestone(num,di='cov-lineages/pango-designation'):\n",
"end_milestone=801\n",
"end_sub_milestone=51\n",
"def read_milestone(num,di='cov-lineages/pango-designation',output_format='sep'):\n",
" # or sars-cov-2 proj if it starts to add milestones\n",
" url=\"https://github.com/\"+di+\"/milestone/\"+str(num)\n",
" res=request.urlopen(url)\n",
Expand All @@ -27,11 +27,18 @@
" title=html_soup.title\n",
" progress_percent=html_soup.findAll(class_=\"progress-percent\")[0].text\n",
" #updated version, ignore those with 0% progress.\n",
" if progress_percent==\"0%\":\n",
" return []\n",
" \n",
" open_issues=html_soup.findAll(class_=\"btn-link selected\")[0].text\n",
" #print(open_issues)\n",
" closed_issues=html_soup.findAll(class_=\"btn-link\")[1].text\n",
" #print(closed_issues)\n",
" if not('1' in open_issues+closed_issues):\n",
" return ['nothing']\n",
" if output_format=='full':\n",
" title=title.text.split('Milestone')[0].strip().replace(' ','+')\n",
" #print(title)\n",
" return title\n",
" title=title.text.replace(',','')\n",
" \n",
" \n",
" title=title.split()[:-3]\n",
" \n",
" return title\n",
Expand All @@ -40,12 +47,26 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 16,
"id": "769f2d32",
"metadata": {
"scrolled": false
},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Reading Milestone 40\n",
"Reading Milestone 440\n",
"Reading Milestone 610\n",
"801 HTTP Error 404: Not Found\n",
"10 HTTP Error 404: Not Found\n",
"Reading Milestone in sars-cov-2-variants 30\n",
"51 HTTP Error 404: Not Found\n"
]
}
],
"source": [
"# read milestones \n",
"milestone_end=False\n",
Expand All @@ -56,7 +77,8 @@
"import os \n",
"listd=os.listdir()\n",
"\n",
"\n",
"milestone_already=[]\n",
"milestone_sub_already=[]\n",
"if not(\"autoref\" in listd):\n",
" os.mkdir(\"autoref\")\n",
"lista=os.listdir(\"autoref\")\n",
Expand All @@ -72,9 +94,8 @@
" name=linsp[1]\n",
" if (linsp[1]!='·' and linsp[1]!='Milestone'):\n",
" milestone_list.append([linsp[0],linsp[1]])\n",
" # don't read twice for old milestones\n",
" if issue>=milestone:\n",
" milestone=issue+1\n",
" \n",
" milestone_already.append(issue)\n",
" w.close()\n",
"\n",
"if (\"milestone_sub.txt\" in lista):\n",
Expand All @@ -88,8 +109,8 @@
" if (linsp[1]!='·' and linsp[1]!='Milestone'):\n",
" milestone_list_sub.append([linsp[0],linsp[1]])\n",
" # don't read twice for old milestones\n",
" if issue>=milestone_sub:\n",
" milestone_sub=issue+1 \n",
" milestone_sub_already.append(issue)\n",
" \n",
" w.close() \n",
"#rewrite the txt\n",
"\n",
Expand All @@ -99,6 +120,8 @@
"\n",
"\n",
"while (milestone<=end_milestone):\n",
" while milestone in milestone_already:\n",
" milestone+=1\n",
" try:\n",
" title=read_milestone(milestone)\n",
" for item in title:\n",
Expand All @@ -119,6 +142,8 @@
"\n",
"milestone_end=False\n",
"while (milestone_sub<=end_sub_milestone):\n",
" while milestone_sub in milestone_sub_already:\n",
" milestone_sub+=1\n",
" try:\n",
" title=read_milestone(milestone_sub,di=\"sars-cov-2-variants/lineage-proposals\")\n",
" for item in title:\n",
Expand All @@ -129,7 +154,7 @@
" \n",
" milestone_sub+=1\n",
" if milestone_sub%10==0:\n",
" print(\"Reading Milestone in sars-cov-2-variants \",milestone)\n",
" print(\"Reading Milestone in sars-cov-2-variants \",milestone_sub)\n",
"\n",
"w.close()\n",
"\n",
Expand All @@ -140,17 +165,35 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 39,
"id": "88a367db",
"metadata": {},
"outputs": [],
"source": [
"def find_milestone(name,di):\n",
" #print(name,milestone_list_sub)\n",
" if 'cov-lineages' in di:\n",
" for item in milestone_list:\n",
" if item[1]==name:\n",
" return item[0]\n",
" for item in milestone_list_sub:\n",
" if item[1]==name:\n",
" return item[0]\n",
" \n",
"def read_milestone_issues(name,di='cov-lineages/pango-designation'):\n",
" # or sars-cov-2 proj if it starts to add milestones\n",
" \n",
" # first, deal with multiple names\n",
" #print(name)\n",
" to_add=all_milestones[name]\n",
" actual_milestone='\"'+item_to_milestone[to_add]+'\"'\n",
" #realname=name.split('#')[1]\n",
" if item_to_milestone[to_add]!=name: # for multi\n",
" num=find_milestone(name,di=di)\n",
" #print(num)\n",
" real_milestone=read_milestone(num,di=di,output_format='full')\n",
" else:\n",
" real_milestone=name\n",
" actual_milestone='\"'+real_milestone+'\"'\n",
" url=\"https://github.com/\"+di+\"/issues?q=milestone:\"+actual_milestone\n",
" res=request.urlopen(url)\n",
" rp=res.read()\n",
Expand All @@ -169,13 +212,70 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 42,
"id": "a56544a8",
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['FK.1.1.1', 'sars-cov-2-variants/lineage-proposals#368']\n",
"['FK.1.4.1', 'sars-cov-2-variants/lineage-proposals#88']\n",
"['HM.1', 'sars-cov-2-variants/lineage-proposals#59']\n",
"39\n",
"FT.3+FT.3.1\n",
"['FT.3', 'sars-cov-2-variants/lineage-proposals#266']\n",
"39\n",
"FT.3+FT.3.1\n",
"['FT.3.1', 'sars-cov-2-variants/lineage-proposals#266']\n",
"['FT.3.1.1', '#2140']\n",
"['HR.1', 'sars-cov-2-variants/lineage-proposals#323']\n",
"HQ.1 list index out of range\n",
"31\n",
"XBB.1.5.93+HD.1++HD.1.1\n",
"['XBB.1.5.93', 'sars-cov-2-variants/lineage-proposals#363']\n",
"31\n",
"XBB.1.5.93+HD.1++HD.1.1\n",
"['HD.1.1', 'sars-cov-2-variants/lineage-proposals#363']\n",
"['HN.1', 'sars-cov-2-variants/lineage-proposals#408']\n",
"['FL.21.1', '#2143']\n",
"['FL.21.2', 'sars-cov-2-variants/lineage-proposals#187']\n",
"['FL.29', 'sars-cov-2-variants/lineage-proposals#119']\n",
"['HK.1', 'sars-cov-2-variants/lineage-proposals#359']\n",
"['HK.2', 'sars-cov-2-variants/lineage-proposals#432']\n",
"['HK.3', 'sars-cov-2-variants/lineage-proposals#414']\n",
"['EG.5.1.3', 'sars-cov-2-variants/lineage-proposals#222']\n",
"['EG.5.1.4', 'sars-cov-2-variants/lineage-proposals#428']\n",
"['EG.5.1.6', 'sars-cov-2-variants/lineage-proposals#394']\n",
"['EG.5.2.1', 'sars-cov-2-variants/lineage-proposals#189']\n",
"['EG.5.2.3', 'sars-cov-2-variants/lineage-proposals#353']\n",
"['EG.9', 'sars-cov-2-variants/lineage-proposals#280']\n",
"['EG.9.1', 'sars-cov-2-variants/lineage-proposals#57']\n",
"38\n",
"EG.10++EG.10.1\n",
"['EG.10', 'sars-cov-2-variants/lineage-proposals#346']\n",
"38\n",
"EG.10++EG.10.1\n",
"['EG.10.1', 'sars-cov-2-variants/lineage-proposals#346']\n",
"['GY.4', 'sars-cov-2-variants/lineage-proposals#345']\n",
"['XBB.1.16.17', 'sars-cov-2-variants/lineage-proposals#354']\n",
"['XBB.1.16.18', 'sars-cov-2-variants/lineage-proposals#276']\n",
"['GA.4', 'sars-cov-2-variants/lineage-proposals#265']\n",
"['GW.5', 'sars-cov-2-variants/lineage-proposals#61']\n",
"['XBB.1.19.2', 'sars-cov-2-variants/lineage-proposals#233']\n",
"['FY.4.1.1', 'sars-cov-2-variants/lineage-proposals#302']\n",
"['FY.7', 'sars-cov-2-variants/lineage-proposals#257']\n",
"['HU.2', 'sars-cov-2-variants/lineage-proposals#364']\n",
"['XBB.1.31.1', 'sars-cov-2-variants/lineage-proposals#141']\n",
"['HL.1', 'sars-cov-2-variants/lineage-proposals#352']\n"
]
}
],
"source": [
"# read lineage_notes.txt\n",
"\n",
"\n",
"def milestone2issue(num):\n",
" return 0\n",
"w=open(\"lineage_notes.txt\",'r')\n",
Expand All @@ -186,20 +286,22 @@
"all_milestones={}\n",
"item_to_milestone={}\n",
"for item in milestone_list:\n",
" to_add=\"#\"+str(item[0])\n",
" all_milestones[item[1]]=to_add\n",
" if to_add in item_to_milestone:\n",
" item_to_milestone[to_add]+='+'+item[1]\n",
" else:\n",
" item_to_milestone[to_add]=item[1]\n",
" if item[1]!='nothing':\n",
" to_add=\"#\"+str(item[0])\n",
" all_milestones[item[1]]=to_add\n",
" if to_add in item_to_milestone:\n",
" item_to_milestone[to_add]+='+'+item[1]\n",
" else:\n",
" item_to_milestone[to_add]=item[1]\n",
" \n",
"for item in milestone_list_sub:\n",
" to_add='sars-cov-2-variants/lineage-proposals#'+str(item[0])\n",
" all_milestones[item[1]]=to_add\n",
" if to_add in item_to_milestone:\n",
" item_to_milestone[to_add]+='+'+item[1]\n",
" else:\n",
" item_to_milestone[to_add]=item[1]\n",
" if item[1]!='nothing':\n",
" to_add=\"sars#\"+str(item[0])\n",
" all_milestones[item[1]]=to_add\n",
" if to_add in item_to_milestone:\n",
" item_to_milestone[to_add]+='+'+item[1]\n",
" else:\n",
" item_to_milestone[to_add]=item[1]\n",
"\n",
"changed_items=0\n",
"missing_milestones=[]\n",
Expand Down Expand Up @@ -233,7 +335,7 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 13,
"id": "1c9369e1",
"metadata": {},
"outputs": [],
Expand All @@ -250,10 +352,19 @@
"w=open(\"lineage_notes.txt\",'r')\n",
"data=w.readlines()\n",
"w.close()\n",
"normal_count=0\n",
"sub_count=0\n",
"for i in range(len(data)):\n",
" linsp=data[i].split()\n",
" if linsp[0] in missing_refs:\n",
" data[i]=data[i].replace('\\n','')+\" From \"+missing_refs[linsp[0]]+'\\n'\n",
" if 'sars' in missing_refs[linsp[0]]:\n",
" sub_count+=1\n",
" else:\n",
" normal_count+=1\n",
"\n",
"print(\"Add \",normal_count,\" references from cov-lineages/pango-designation.\")\n",
"print(\"Add \",sub_count,\" references from sars-cov-2-variants/lineage-proposals.\")\n",
"\n",
"w=open(\"lineage_notes.txt\",'w',newline='\\n')\n",
"w.writelines(data)\n",
Expand Down
30 changes: 30 additions & 0 deletions autoref/milestone.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -864,3 +864,33 @@
793 FL.10.1
793 FL.25
795 FR.1.3
796 FL.20.1
797 GW.5
798 FL.21.1
800 FL.19.1
612 BN.1.4.1
799 FT.3.1.1
13 nothing
22 nothing
39 nothing
146 nothing
155 nothing
156 nothing
275 nothing
284 nothing
302 nothing
423 nothing
434 nothing
439 nothing
440 nothing
456 nothing
462 nothing
463 nothing
571 nothing
595 nothing
609 nothing
634 nothing
663 nothing
692 nothing
776 nothing
794 nothing
43 changes: 43 additions & 0 deletions autoref/milestone_sub.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,46 @@
11 FL.4.3
12 GJ.1.1
13 GW.1
14 HL.1
15 EG.5.1.6
17 HM.1
18 EG.5.1.4
20 XBB.1.16.17
21 FY.4.1.1
22 FY.2.1
23 HK.1
24 EG.5.1.3
25 GA.4
26 FY.7
27 FL.29
6 GY.4
19 HK.3
31 XBB.1.5.93
31 HD.1
31 HD.1.1
32 HK.2
33 FK.1.1.1
34 FK.1.4.1
35 HN.1
36 EG.9.1
37 EG.9
38 EG.10
38 EG.10.1
39 FT.3
39 FT.3.1
41 EG.5.2.3
42 EG.5.2.1
43 XBB.1.16.18
44 HQ.1
45 HR.1
46 GW.5
47 XBB.1.19.2
48 XBB.1.31.1
49 FL.21.2
50 HU.2
3 nothing
16 nothing
28 nothing
29 nothing
30 nothing
40 nothing
Loading

0 comments on commit 502c1d1

Please sign in to comment.