From 502c1d19d53207a788d48c4d1c154612937b8449 Mon Sep 17 00:00:00 2001 From: aviczhl2 Date: Fri, 28 Jul 2023 12:15:32 +0800 Subject: [PATCH] modified: automated-issue-ref.ipynb modified: autoref/milestone.txt modified: autoref/milestone_sub.txt modified: autoref/missing_refs.txt --- automated-issue-ref.ipynb | 179 ++++++++++++++++++++++++++++++-------- autoref/milestone.txt | 30 +++++++ autoref/milestone_sub.txt | 43 +++++++++ autoref/missing_refs.txt | 40 ++++++++- 4 files changed, 254 insertions(+), 38 deletions(-) diff --git a/automated-issue-ref.ipynb b/automated-issue-ref.ipynb index 67a64928..80e0093e 100755 --- a/automated-issue-ref.ipynb +++ b/automated-issue-ref.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 7, + "execution_count": 41, "id": "f64573a2", "metadata": {}, "outputs": [], @@ -15,9 +15,9 @@ "# the number of milestone we start to read\n", "start_milestone=1\n", "start_sub_milestone=1\n", - "end_milestone=795\n", - "end_sub_milestone=13\n", - "def read_milestone(num,di='cov-lineages/pango-designation'):\n", + "end_milestone=801\n", + "end_sub_milestone=51\n", + "def read_milestone(num,di='cov-lineages/pango-designation',output_format='sep'):\n", " # or sars-cov-2 proj if it starts to add milestones\n", " url=\"https://github.com/\"+di+\"/milestone/\"+str(num)\n", " res=request.urlopen(url)\n", @@ -27,11 +27,18 @@ " title=html_soup.title\n", " progress_percent=html_soup.findAll(class_=\"progress-percent\")[0].text\n", " #updated version, ignore those with 0% progress.\n", - " if progress_percent==\"0%\":\n", - " return []\n", - " \n", + " open_issues=html_soup.findAll(class_=\"btn-link selected\")[0].text\n", + " #print(open_issues)\n", + " closed_issues=html_soup.findAll(class_=\"btn-link\")[1].text\n", + " #print(closed_issues)\n", + " if not('1' in open_issues+closed_issues):\n", + " return ['nothing']\n", + " if output_format=='full':\n", + " title=title.text.split('Milestone')[0].strip().replace(' ','+')\n", + " #print(title)\n", + " return title\n", " title=title.text.replace(',','')\n", - " \n", + " \n", " title=title.split()[:-3]\n", " \n", " return title\n", @@ -40,12 +47,26 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 16, "id": "769f2d32", "metadata": { "scrolled": false }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Reading Milestone 40\n", + "Reading Milestone 440\n", + "Reading Milestone 610\n", + "801 HTTP Error 404: Not Found\n", + "10 HTTP Error 404: Not Found\n", + "Reading Milestone in sars-cov-2-variants 30\n", + "51 HTTP Error 404: Not Found\n" + ] + } + ], "source": [ "# read milestones \n", "milestone_end=False\n", @@ -56,7 +77,8 @@ "import os \n", "listd=os.listdir()\n", "\n", - "\n", + "milestone_already=[]\n", + "milestone_sub_already=[]\n", "if not(\"autoref\" in listd):\n", " os.mkdir(\"autoref\")\n", "lista=os.listdir(\"autoref\")\n", @@ -72,9 +94,8 @@ " name=linsp[1]\n", " if (linsp[1]!='·' and linsp[1]!='Milestone'):\n", " milestone_list.append([linsp[0],linsp[1]])\n", - " # don't read twice for old milestones\n", - " if issue>=milestone:\n", - " milestone=issue+1\n", + " \n", + " milestone_already.append(issue)\n", " w.close()\n", "\n", "if (\"milestone_sub.txt\" in lista):\n", @@ -88,8 +109,8 @@ " if (linsp[1]!='·' and linsp[1]!='Milestone'):\n", " milestone_list_sub.append([linsp[0],linsp[1]])\n", " # don't read twice for old milestones\n", - " if issue>=milestone_sub:\n", - " milestone_sub=issue+1 \n", + " milestone_sub_already.append(issue)\n", + " \n", " w.close() \n", "#rewrite the txt\n", "\n", @@ -99,6 +120,8 @@ "\n", "\n", "while (milestone<=end_milestone):\n", + " while milestone in milestone_already:\n", + " milestone+=1\n", " try:\n", " title=read_milestone(milestone)\n", " for item in title:\n", @@ -119,6 +142,8 @@ "\n", "milestone_end=False\n", "while (milestone_sub<=end_sub_milestone):\n", + " while milestone_sub in milestone_sub_already:\n", + " milestone_sub+=1\n", " try:\n", " title=read_milestone(milestone_sub,di=\"sars-cov-2-variants/lineage-proposals\")\n", " for item in title:\n", @@ -129,7 +154,7 @@ " \n", " milestone_sub+=1\n", " if milestone_sub%10==0:\n", - " print(\"Reading Milestone in sars-cov-2-variants \",milestone)\n", + " print(\"Reading Milestone in sars-cov-2-variants \",milestone_sub)\n", "\n", "w.close()\n", "\n", @@ -140,17 +165,35 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 39, "id": "88a367db", "metadata": {}, "outputs": [], "source": [ + "def find_milestone(name,di):\n", + " #print(name,milestone_list_sub)\n", + " if 'cov-lineages' in di:\n", + " for item in milestone_list:\n", + " if item[1]==name:\n", + " return item[0]\n", + " for item in milestone_list_sub:\n", + " if item[1]==name:\n", + " return item[0]\n", + " \n", "def read_milestone_issues(name,di='cov-lineages/pango-designation'):\n", " # or sars-cov-2 proj if it starts to add milestones\n", " \n", " # first, deal with multiple names\n", + " #print(name)\n", " to_add=all_milestones[name]\n", - " actual_milestone='\"'+item_to_milestone[to_add]+'\"'\n", + " #realname=name.split('#')[1]\n", + " if item_to_milestone[to_add]!=name: # for multi\n", + " num=find_milestone(name,di=di)\n", + " #print(num)\n", + " real_milestone=read_milestone(num,di=di,output_format='full')\n", + " else:\n", + " real_milestone=name\n", + " actual_milestone='\"'+real_milestone+'\"'\n", " url=\"https://github.com/\"+di+\"/issues?q=milestone:\"+actual_milestone\n", " res=request.urlopen(url)\n", " rp=res.read()\n", @@ -169,13 +212,70 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 42, "id": "a56544a8", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['FK.1.1.1', 'sars-cov-2-variants/lineage-proposals#368']\n", + "['FK.1.4.1', 'sars-cov-2-variants/lineage-proposals#88']\n", + "['HM.1', 'sars-cov-2-variants/lineage-proposals#59']\n", + "39\n", + "FT.3+FT.3.1\n", + "['FT.3', 'sars-cov-2-variants/lineage-proposals#266']\n", + "39\n", + "FT.3+FT.3.1\n", + "['FT.3.1', 'sars-cov-2-variants/lineage-proposals#266']\n", + "['FT.3.1.1', '#2140']\n", + "['HR.1', 'sars-cov-2-variants/lineage-proposals#323']\n", + "HQ.1 list index out of range\n", + "31\n", + "XBB.1.5.93+HD.1++HD.1.1\n", + "['XBB.1.5.93', 'sars-cov-2-variants/lineage-proposals#363']\n", + "31\n", + "XBB.1.5.93+HD.1++HD.1.1\n", + "['HD.1.1', 'sars-cov-2-variants/lineage-proposals#363']\n", + "['HN.1', 'sars-cov-2-variants/lineage-proposals#408']\n", + "['FL.21.1', '#2143']\n", + "['FL.21.2', 'sars-cov-2-variants/lineage-proposals#187']\n", + "['FL.29', 'sars-cov-2-variants/lineage-proposals#119']\n", + "['HK.1', 'sars-cov-2-variants/lineage-proposals#359']\n", + "['HK.2', 'sars-cov-2-variants/lineage-proposals#432']\n", + "['HK.3', 'sars-cov-2-variants/lineage-proposals#414']\n", + "['EG.5.1.3', 'sars-cov-2-variants/lineage-proposals#222']\n", + "['EG.5.1.4', 'sars-cov-2-variants/lineage-proposals#428']\n", + "['EG.5.1.6', 'sars-cov-2-variants/lineage-proposals#394']\n", + "['EG.5.2.1', 'sars-cov-2-variants/lineage-proposals#189']\n", + "['EG.5.2.3', 'sars-cov-2-variants/lineage-proposals#353']\n", + "['EG.9', 'sars-cov-2-variants/lineage-proposals#280']\n", + "['EG.9.1', 'sars-cov-2-variants/lineage-proposals#57']\n", + "38\n", + "EG.10++EG.10.1\n", + "['EG.10', 'sars-cov-2-variants/lineage-proposals#346']\n", + "38\n", + "EG.10++EG.10.1\n", + "['EG.10.1', 'sars-cov-2-variants/lineage-proposals#346']\n", + "['GY.4', 'sars-cov-2-variants/lineage-proposals#345']\n", + "['XBB.1.16.17', 'sars-cov-2-variants/lineage-proposals#354']\n", + "['XBB.1.16.18', 'sars-cov-2-variants/lineage-proposals#276']\n", + "['GA.4', 'sars-cov-2-variants/lineage-proposals#265']\n", + "['GW.5', 'sars-cov-2-variants/lineage-proposals#61']\n", + "['XBB.1.19.2', 'sars-cov-2-variants/lineage-proposals#233']\n", + "['FY.4.1.1', 'sars-cov-2-variants/lineage-proposals#302']\n", + "['FY.7', 'sars-cov-2-variants/lineage-proposals#257']\n", + "['HU.2', 'sars-cov-2-variants/lineage-proposals#364']\n", + "['XBB.1.31.1', 'sars-cov-2-variants/lineage-proposals#141']\n", + "['HL.1', 'sars-cov-2-variants/lineage-proposals#352']\n" + ] + } + ], "source": [ "# read lineage_notes.txt\n", "\n", + "\n", "def milestone2issue(num):\n", " return 0\n", "w=open(\"lineage_notes.txt\",'r')\n", @@ -186,20 +286,22 @@ "all_milestones={}\n", "item_to_milestone={}\n", "for item in milestone_list:\n", - " to_add=\"#\"+str(item[0])\n", - " all_milestones[item[1]]=to_add\n", - " if to_add in item_to_milestone:\n", - " item_to_milestone[to_add]+='+'+item[1]\n", - " else:\n", - " item_to_milestone[to_add]=item[1]\n", + " if item[1]!='nothing':\n", + " to_add=\"#\"+str(item[0])\n", + " all_milestones[item[1]]=to_add\n", + " if to_add in item_to_milestone:\n", + " item_to_milestone[to_add]+='+'+item[1]\n", + " else:\n", + " item_to_milestone[to_add]=item[1]\n", " \n", "for item in milestone_list_sub:\n", - " to_add='sars-cov-2-variants/lineage-proposals#'+str(item[0])\n", - " all_milestones[item[1]]=to_add\n", - " if to_add in item_to_milestone:\n", - " item_to_milestone[to_add]+='+'+item[1]\n", - " else:\n", - " item_to_milestone[to_add]=item[1]\n", + " if item[1]!='nothing':\n", + " to_add=\"sars#\"+str(item[0])\n", + " all_milestones[item[1]]=to_add\n", + " if to_add in item_to_milestone:\n", + " item_to_milestone[to_add]+='+'+item[1]\n", + " else:\n", + " item_to_milestone[to_add]=item[1]\n", "\n", "changed_items=0\n", "missing_milestones=[]\n", @@ -233,7 +335,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 13, "id": "1c9369e1", "metadata": {}, "outputs": [], @@ -250,10 +352,19 @@ "w=open(\"lineage_notes.txt\",'r')\n", "data=w.readlines()\n", "w.close()\n", + "normal_count=0\n", + "sub_count=0\n", "for i in range(len(data)):\n", " linsp=data[i].split()\n", " if linsp[0] in missing_refs:\n", " data[i]=data[i].replace('\\n','')+\" From \"+missing_refs[linsp[0]]+'\\n'\n", + " if 'sars' in missing_refs[linsp[0]]:\n", + " sub_count+=1\n", + " else:\n", + " normal_count+=1\n", + "\n", + "print(\"Add \",normal_count,\" references from cov-lineages/pango-designation.\")\n", + "print(\"Add \",sub_count,\" references from sars-cov-2-variants/lineage-proposals.\")\n", "\n", "w=open(\"lineage_notes.txt\",'w',newline='\\n')\n", "w.writelines(data)\n", diff --git a/autoref/milestone.txt b/autoref/milestone.txt index 2c860f92..58d5c533 100755 --- a/autoref/milestone.txt +++ b/autoref/milestone.txt @@ -864,3 +864,33 @@ 793 FL.10.1 793 FL.25 795 FR.1.3 +796 FL.20.1 +797 GW.5 +798 FL.21.1 +800 FL.19.1 +612 BN.1.4.1 +799 FT.3.1.1 +13 nothing +22 nothing +39 nothing +146 nothing +155 nothing +156 nothing +275 nothing +284 nothing +302 nothing +423 nothing +434 nothing +439 nothing +440 nothing +456 nothing +462 nothing +463 nothing +571 nothing +595 nothing +609 nothing +634 nothing +663 nothing +692 nothing +776 nothing +794 nothing diff --git a/autoref/milestone_sub.txt b/autoref/milestone_sub.txt index a31c0afd..1ff6f408 100755 --- a/autoref/milestone_sub.txt +++ b/autoref/milestone_sub.txt @@ -9,3 +9,46 @@ 11 FL.4.3 12 GJ.1.1 13 GW.1 +14 HL.1 +15 EG.5.1.6 +17 HM.1 +18 EG.5.1.4 +20 XBB.1.16.17 +21 FY.4.1.1 +22 FY.2.1 +23 HK.1 +24 EG.5.1.3 +25 GA.4 +26 FY.7 +27 FL.29 +6 GY.4 +19 HK.3 +31 XBB.1.5.93 +31 HD.1 +31 HD.1.1 +32 HK.2 +33 FK.1.1.1 +34 FK.1.4.1 +35 HN.1 +36 EG.9.1 +37 EG.9 +38 EG.10 +38 EG.10.1 +39 FT.3 +39 FT.3.1 +41 EG.5.2.3 +42 EG.5.2.1 +43 XBB.1.16.18 +44 HQ.1 +45 HR.1 +46 GW.5 +47 XBB.1.19.2 +48 XBB.1.31.1 +49 FL.21.2 +50 HU.2 +3 nothing +16 nothing +28 nothing +29 nothing +30 nothing +40 nothing diff --git a/autoref/missing_refs.txt b/autoref/missing_refs.txt index ec8c5134..19ca666c 100755 --- a/autoref/missing_refs.txt +++ b/autoref/missing_refs.txt @@ -1,4 +1,36 @@ -GK.1.3 sars-cov-2-variants/lineage-proposals#179 -FL.4.3 sars-cov-2-variants/lineage-proposals#153 -XBB.1.41.1 sars-cov-2-variants/lineage-proposals#150 -GJ.1.1 sars-cov-2-variants/lineage-proposals#109 +FK.1.1.1 sars-cov-2-variants/lineage-proposals#368 +FK.1.4.1 sars-cov-2-variants/lineage-proposals#88 +HM.1 sars-cov-2-variants/lineage-proposals#59 +FT.3 sars-cov-2-variants/lineage-proposals#266 +FT.3.1 sars-cov-2-variants/lineage-proposals#266 +FT.3.1.1 #2140 +HR.1 sars-cov-2-variants/lineage-proposals#323 +XBB.1.5.93 sars-cov-2-variants/lineage-proposals#363 +HD.1.1 sars-cov-2-variants/lineage-proposals#363 +HN.1 sars-cov-2-variants/lineage-proposals#408 +FL.21.1 #2143 +FL.21.2 sars-cov-2-variants/lineage-proposals#187 +FL.29 sars-cov-2-variants/lineage-proposals#119 +HK.1 sars-cov-2-variants/lineage-proposals#359 +HK.2 sars-cov-2-variants/lineage-proposals#432 +HK.3 sars-cov-2-variants/lineage-proposals#414 +EG.5.1.3 sars-cov-2-variants/lineage-proposals#222 +EG.5.1.4 sars-cov-2-variants/lineage-proposals#428 +EG.5.1.6 sars-cov-2-variants/lineage-proposals#394 +EG.5.2.1 sars-cov-2-variants/lineage-proposals#189 +EG.5.2.3 sars-cov-2-variants/lineage-proposals#353 +EG.9 sars-cov-2-variants/lineage-proposals#280 +EG.9.1 sars-cov-2-variants/lineage-proposals#57 +EG.10 sars-cov-2-variants/lineage-proposals#346 +EG.10.1 sars-cov-2-variants/lineage-proposals#346 +GY.4 sars-cov-2-variants/lineage-proposals#345 +XBB.1.16.17 sars-cov-2-variants/lineage-proposals#354 +XBB.1.16.18 sars-cov-2-variants/lineage-proposals#276 +GA.4 sars-cov-2-variants/lineage-proposals#265 +GW.5 sars-cov-2-variants/lineage-proposals#61 +XBB.1.19.2 sars-cov-2-variants/lineage-proposals#233 +FY.4.1.1 sars-cov-2-variants/lineage-proposals#302 +FY.7 sars-cov-2-variants/lineage-proposals#257 +HU.2 sars-cov-2-variants/lineage-proposals#364 +XBB.1.31.1 sars-cov-2-variants/lineage-proposals#141 +HL.1 sars-cov-2-variants/lineage-proposals#352