Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: emit bulk redirect configuration for iesg artifacts #7086

Merged
merged 2 commits into from
Feb 22, 2024
Merged

fix: emit bulk redirect configuration for iesg artifacts #7086

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e926e49
fix: emit bulk redirect configuration for iesg artifacts
rjsparks Feb 22, 2024
4d352df
chore: remove unintended debug statement
rjsparks Feb 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 16 additions & 14 deletions ietf/group/management/commands/import_iesg_appeals.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# Copyright The IETF Trust 2023, All Rights Reserved

import csv
import datetime
import re
import shutil
Expand Down Expand Up @@ -148,9 +149,9 @@ def handle(self, *args, **options):
part_times["klensin-2023-08-15.txt"] = "2023-08-15 15:03:55 -0400"
part_times["response-to-klensin-2023-08-15.txt"] = "2023-08-24 18:54:13 +0300"
part_times["hardie-frindell-2023-07-19.txt"] = "2023-07-19 07:17:16PDT"
part_times[
"response-to-hardie-frindell-2023-07-19.txt"
] = "2023-08-15 11:58:26PDT"
part_times["response-to-hardie-frindell-2023-07-19.txt"] = (
"2023-08-15 11:58:26PDT"
)
part_times["mcsweeney-2020-07-08.txt"] = "2020-07-08 14:45:00 -0400"
part_times["response-to-mcsweeney-2020-07-08.pdf"] = "2020-07-28 12:54:04 -0000"
part_times["gont-2020-04-22.txt"] = "2020-04-22 22:26:20 -0400"
Expand Down Expand Up @@ -200,9 +201,9 @@ def handle(self, *args, **options):
part_times["response-to-sayre-2006-08-29.txt"] = "2006-10-16 13:07:18 -0400"
part_times["morfin-2006-08-16.pdf"] = "2006-08-16 18:28:19 -0400"
part_times["response-to-morfin-2006-08-17.txt"] = "2006-08-22 12:05:42 -0400"
part_times[
"response-to-morfin-2006-08-17-part2.txt"
] = "2006-11-07 13:00:58 -0500"
part_times["response-to-morfin-2006-08-17-part2.txt"] = (
"2006-11-07 13:00:58 -0500"
)
# part_times["anderson-2006-06-13.txt"]="2006-06-13 21:51:18EDT"
part_times["anderson-2006-06-13.txt"] = "2006-06-13 21:51:18 -0400"
part_times["response-to-anderson-2006-06-14.txt"] = "2006-07-10 14:31:08 -0400"
Expand Down Expand Up @@ -279,14 +280,15 @@ def handle(self, *args, **options):
bits=bits,
)
redirects.append(
(
part.replace(".md", ".html")
if part.endswith(".md")
else part,
artifact.pk,
)
[
f'www6.ietf.org/iesg/appeal/{part.replace(".md", ".html") if part.endswith(".md") else part}',
f"https://datatracker.ietf.org/group/iesg/appeals/artifact/{artifact.pk}",
302,
]
)

shutil.rmtree(tmpdir)
with open("iesg_appeal_redirects.txt", "w") as f:
f.write(str(redirects))
with open("iesg_appeal_redirects.csv", "w", newline="") as f:
csvwriter = csv.writer(f)
rjsparks marked this conversation as resolved.
Show resolved Hide resolved
for row in redirects:
csvwriter.writerow(row)
91 changes: 88 additions & 3 deletions ietf/group/management/commands/import_iesg_statements.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import debug # pyflakes:ignore

import csv
import datetime
import os
import shutil
Expand Down Expand Up @@ -41,8 +42,11 @@ def handle(self, *args, **options):
self.stdout.write(f"Clean up {tmpdir} manually")
exit(-1)

redirects = []
for item in self.get_work_items():
replaced = item.title.endswith(" SUPERSEDED") or item.doc_time.date() == datetime.date(2007,7,30)
replaced = item.title.endswith(
" SUPERSEDED"
) or item.doc_time.date() == datetime.date(2007, 7, 30)
title = item.title
if title.endswith(" - SUPERSEDED"):
title = title[: -len(" - SUPERSEDED")]
Expand Down Expand Up @@ -93,20 +97,34 @@ def handle(self, *args, **options):
os.makedirs(dest.parent, exist_ok=True)
shutil.copy(source, dest)

redirects.append(
[
f"www.ietf.org/about/groups/iesg/statements/{item.slug}",
f"https://datatracker.ietf.org/group/iesg/statements/{name}",
302,
]
)

shutil.rmtree(tmpdir)
with open("iesg_statement_redirects.csv", "w", newline="") as f:
csvwriter = csv.writer(f)
rjsparks marked this conversation as resolved.
Show resolved Hide resolved
for row in redirects:
csvwriter.writerow(row)

def get_work_items(self):
Item = namedtuple("Item", "doc_time source_filename title")
Item = namedtuple("Item", "doc_time source_filename title slug")
items = []
dressed_rows = " ".join(
self.cut_paste_from_www().expandtabs(1).split(" ")
).split("\n")
old_slugs = self.get_old_slugs()
# Rube-Goldberg-esque dance to deal with conflicting directions of the scrape and
# what order we want the result to sort to
dressed_rows.reverse()
old_slugs.reverse()
total_times_date_seen = Counter([row.split(" ")[0] for row in dressed_rows])
count_date_seen_so_far = Counter()
for row in dressed_rows:
for row, slug in zip(dressed_rows, old_slugs):
rjsparks marked this conversation as resolved.
Show resolved Hide resolved
date_part = row.split(" ")[0]
title_part = row[len(date_part) + 1 :]
datetime_args = list(map(int, date_part.replace("-0", "-").split("-")))
Expand All @@ -120,6 +138,7 @@ def get_work_items(self):
doc_time,
f"{date_part}-{total_times_date_seen[date_part] - count_date_seen_so_far[date_part]}.md",
title_part,
slug,
)
)
return items
Expand Down Expand Up @@ -187,3 +206,69 @@ def cut_paste_from_www(self):
2000-11-20 A New IETF Work Area
2000-08-29 Guidance on Interim IETF Working Group Meetings and Conference Calls - SUPERSEDED
2000-08-29 IESG Guidance on the Moderation of IETF Working Group Mailing Lists"""

def get_old_slugs(self):
return [
"support-documents",
"interim-meetings-guidance",
"ethertypes",
"second-report-on-the-rfc-8989-experiment",
"interim-meetings-guidance-2023-01-27",
"statement-on-restricting-access",
"handling-ballot-positions",
"report-on-rfc8989-experiment",
"email-addresses-ietf-domain",
"on-inclusive-language",
"internet-draft-authorship",
"processing-errata-ietf-stream",
"last-call-guidance",
"statement-on-oppressive-exclusionary-language",
"interim-meetings-guidance-2020-05-01",
"meeting-photography-policy",
"interim-meetings-guidance-2018-01-11",
"open-source-repositories-license",
"support-documents-2016-11-13",
"interim-meetings-guidance-2016-02-05",
"interim-meetings-guidance-2016-01-11",
"maximizing-encrypted-access",
"internet-draft-authorship-2015-06-11",
"designating-rfcs-historic",
"iesg-discuss-criteria",
"writable-mib-module",
"anti-harassment-policy",
"ethertypes-2012-10-25",
"internet-draft-removal",
"designating-rfcs-historic-2011-10-20",
"designating-rfcs-historic-2011-06-27",
"rfc-metadata-errata",
"document-shepherds",
"assignable-codepoints-addresses-names",
"nomcom-eligibility-day-passes",
"copyright-2009-09-08",
"reserving-resources-examples",
"interim-meetings-guidance-2008-09-02",
"processing-rfc-errata",
"spam-control-2008-04-14",
"registration-requests-uris",
"urn-namespaces-registry",
"off-topic-postings",
"appeals-actions-decisions",
"experimental-congestion-control",
"area-director-sponsoring-documents",
"last-call-guidance-2007-01-15",
"normative-informative-references",
"disruptive-posting",
"spam-control-2006-01-09",
"auth48",
"syntax-format-definitions",
"idn",
"copyright-2002-11-27",
"spam-control-2002-03-13",
"design-teams",
"formal-languages-use",
"sub-ip-area-2001-03-21",
"sub-ip-area-2000-11-20",
"sub-ip-area-2000-12-06",
"interim-meetings-guidance-2000-08-29",
"mailing-lists-moderation",
]
1 change: 0 additions & 1 deletion ietf/group/views.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2209,7 +2209,6 @@ def statements(request, acronym, group_type=None):
)
.order_by("-published")
)
debug.show("statements.first().status")
return render(
request,
"group/statements.html",
Expand Down
23 changes: 22 additions & 1 deletion ietf/meeting/management/commands/import_iesg_minutes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# Copyright The IETF Trust 2023, All Rights Reserved

from collections import namedtuple
import csv
import datetime
import os
import re
Expand Down Expand Up @@ -158,6 +159,7 @@ def handle(self, *args, **options):
minutes_dir = Path(old_minutes_root)
date_re = re.compile(r"\d{4}-\d{2}-\d{2}")
meeting_times = set()
redirects = []
for file_prefix in ["minutes", "narrative"]:
paths = list(minutes_dir.glob(f"[12][09][0129][0-9]/{file_prefix}*.txt"))
paths.extend(
Expand Down Expand Up @@ -259,6 +261,13 @@ def handle(self, *args, **options):
else:
os.makedirs(dest.parent, exist_ok=True)
shutil.copy(source, dest)
redirects.append(
[
f"www6.ietf.org/iesg/minutes/{dt.year}/{bof_coord_data[dt].source_name}",
f"https://datatracker.ietf.org/doc/{doc_name}",
302,
]
)
else:
for type_id in ["minutes", "narrativeminutes"]:
source_file_prefix = (
Expand Down Expand Up @@ -334,10 +343,22 @@ def handle(self, *args, **options):
html_content = re.sub(
'<a href="http://validator.w3.org/[^>]*> *<img[^>]*></a>',
"",
html_content
html_content,
)
dest.write_text(html_content, encoding="utf-8")
else:
shutil.copy(txt_source, dest)
redirects.append(
[
f"www6.ietf.org/iesg/minutes/{dt.year}/{txt_source.name if txt_source.exists() else html_source.name}",
f"https://datatracker.ietf.org/doc/{doc_name}",
302,
]
)

counter += 1

with open("iesg_minutes_redirects.csv", "w", newline="") as f:
csvwriter = csv.writer(f)
rjsparks marked this conversation as resolved.
Show resolved Hide resolved
for row in redirects:
csvwriter.writerow(row)
Loading