Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

WIP: snapshot support #70

Merged
merged 8 commits into from
May 29, 2018
Merged

WIP: snapshot support #70

Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
3c74bd2
WIP: snapshot support
annevk May 15, 2018
0acb4ae
fixup
annevk May 15, 2018
4924d1d
more modest progress
annevk May 16, 2018
0751161
Abuse typed constant to set sha-link attribute
sideshowbarker May 17, 2018
d77afcf
add review functionality
annevk May 17, 2018
6d066f9
Use Rope.Append to set sha-link attribute
sideshowbarker May 19, 2018
3705cb8
move Result := True down again
annevk May 22, 2018
c974041
avoid too much document cloning
annevk May 22, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 97 additions & 36 deletions src/wattsi.pas
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,12 +52,13 @@
IsFirstSearchIndexItem: Boolean = true;

type
TAllVariants = (vHTML, vDEV, vSplit);
TVariants = vHTML..vDEV;
TAllVariants = (vHTML, vDEV, vSnap, vReview, vSplit);
TVariants = vHTML..vReview;

const
kSuffixes: array[TVariants] of UTF8String = ('html', 'dev');
kExcludingAttribute: array[TAllVariants] of UTF8String = ('w-nohtml', 'w-nodev', 'w-nosplit');
kSuffixes: array[TVariants] of UTF8String = ('html', 'dev', 'snap', 'review');
kExcludingAttribute: array[TAllVariants] of UTF8String = ('w-nohtml', 'w-nodev', 'w-nosnap', 'w-noreview', 'w-nosplit');
kDEVAttribute = 'w-dev';
kCrossRefAttribute = 'data-x';
kCrossSpecRefAttribute = 'data-x-href';
kCrossRefInternalLinkAttribute = 'data-x-internal';
Expand Down Expand Up @@ -121,7 +122,7 @@ procedure TFeature.Reset();
Browsers: array[TBrowserIndex] of TBrowser;
Features: TFeatureMap;

procedure ProcessDocument(const Document: TDocument; const Variant: TVariants; out BigTOC: TElement);
procedure ProcessDocument(const Document: TDocument; const Variant: TVariants; out BigTOC: TElement; const SourceGitSHA: AnsiString);
type
PElementListNode = ^TElementListNode;
TElementListNode = record
Expand Down Expand Up @@ -322,7 +323,9 @@ TCrossReferences = record
repeat
if (Current is TElement) then
begin
if (TElement(Current).HasAttribute(kExcludingAttribute[Variant])) then
if ((TElement(Current).HasAttribute(kExcludingAttribute[Variant])) or
((Variant <> vDEV) and
(TElement(Current).HasAttribute(kDEVAttribute)))) then
begin
DropNode();
continue;
Expand Down Expand Up @@ -598,6 +601,8 @@ TCrossReferences = record
end;

function ProcessNode(var Node: TNode): Boolean; // return True if we are to keep this node, False if we drop it
const
SourceGitBaseURL: AnsiString = 'https://github.com/whatwg/html/commit/';
var
CandidateChild, SelectedForTransfer: TNode;
CurrentHeadingRank: THeadingRank;
Expand Down Expand Up @@ -883,6 +888,14 @@ TCrossReferences = record
end;
end
else
// For vReview the title is already taken care of
if (Element.IsIdentity(nsHTML, eTitle)) and (Variant = vSnap) then
begin
Element.AppendChild(TText.Create(' (Commit Snapshot '));
Element.AppendChild(TText.Create(SourceGitSHA));
Element.AppendChild(TText.Create(')'));
end
else
if (Element.IsIdentity(nsHTML, eDFN)) then
begin
if (Element.HasAttribute(kLTAttribute)) then
Expand Down Expand Up @@ -934,6 +947,16 @@ TCrossReferences = record
Result := False;
end
else
if (Element.isIdentity(nsHTML, eA) and (Element.GetAttribute('class').AsString = 'sha-link') and (Variant = vSnap)) then
begin
Element.AppendChild(TText.Create(SourceGitSHA));
Element.AppendChild(TText.Create(' commit'));
Scratch := Default(Rope);
Scratch.Append(@SourceGitBaseURL);
Scratch.Append(@SourceGitSHA);
Element.SetAttributeDestructively('href', Scratch);
end
else
if (Element.isIdentity(nsHTML, eA) and (not Element.HasAttribute(kHrefAttribute))) then
begin
Fail('<a> without href found: ' + Describe(Element));
Expand Down Expand Up @@ -1757,7 +1780,8 @@ procedure Save(const Document: TDocument; const FileName: AnsiString; const InSp
for Variant in TAllVariants do
if (AttributeName = kExcludingAttribute[Variant]) then
Skip := True;
if (Skip or (AttributeName = kCrossRefAttribute) or
if (Skip or (AttributeName = kDEVAttribute) or
(AttributeName = kCrossRefAttribute) or
(AttributeName = kSubDFNAttribute) or
(AttributeName = kCrossSpecRefAttribute) or
(AttributeName = kUndefinedAttribute) or
Expand Down Expand Up @@ -2507,6 +2531,8 @@ function Main(): Boolean;
var
ParamOffset: Integer = 0;
SourceFile: AnsiString;
SourceGitSHA: AnsiString;
BuildType: AnsiString;
Source: TFileData;
Parser: THTMLParser;
BigTOC: TElement;
Expand All @@ -2515,8 +2541,8 @@ function Main(): Boolean;
Variant: TAllVariants;
begin
Result := False;
if (ParamCount() <> 4) then
if ((ParamCount() = 5) and (ParamStr(1) = '--quiet')) then
if (ParamCount() <> 6) then
if ((ParamCount() = 7) and (ParamStr(1) = '--quiet')) then
begin
Quiet := true;
ParamOffset := 1;
Expand All @@ -2531,12 +2557,14 @@ function Main(): Boolean;
begin
Writeln('wattsi: invalid arguments');
Writeln('syntax:');
Writeln(' wattsi [--quiet] <source-file> <output-directory> <caniuse.json> <bugs.csv>');
Writeln(' wattsi [--quiet] <source-file> <source-git-sha> <output-directory> <default-or-review> <caniuse.json> <bugs.csv>');
Writeln(' wattsi --version');
exit;
end;
SourceFile := ParamStr(1 + ParamOffset);
OutputDirectory := ParamStr(2 + ParamOffset);
SourceGitSHA := ParamStr(2 + ParamOffset);
OutputDirectory := ParamStr(3 + ParamOffset);
BuildType := ParamStr(4 + ParamOffset);
if (not IsEmptyDirectory(OutputDirectory)) then
begin
// only act if, when we start, the output directory is empty, to make sure that the
Expand All @@ -2546,8 +2574,8 @@ function Main(): Boolean;
end;
Features := TFeatureMap.Create(@UTF8StringHash32);
try
PreProcessCanIUseData(ParamStr(3 + ParamOffset));
PreProcessBugsData(ParamStr(4 + ParamOffset));
PreProcessCanIUseData(ParamStr(5 + ParamOffset));
PreProcessBugsData(ParamStr(6 + ParamOffset));
{$IFDEF VERBOSE_PREPROCESSORS}
if (Assigned(Features)) then
for ID in Features do
Expand Down Expand Up @@ -2592,35 +2620,61 @@ function Main(): Boolean;
Parser.Free();
end;
{$IFDEF TIMINGS} Writeln('Elapsed time: ', MillisecondsBetween(StartTime, Now()), 'ms'); {$ENDIF}
{$IFDEF TIMINGS} Writeln('Cloning...'); {$ENDIF}
{$IFDEF TIMINGS} StartTime := Now(); {$ENDIF}
for Variant in OtherVariants do
Documents[Variant] := Documents[Low(TVariants)].CloneNode(True);
{$IFDEF TIMINGS} Writeln('Elapsed time: ', MillisecondsBetween(StartTime, Now()), 'ms'); {$ENDIF}
if (BuildType = 'default') then
begin
{$IFDEF TIMINGS} Writeln('Cloning...'); {$ENDIF}
{$IFDEF TIMINGS} StartTime := Now(); {$ENDIF}
for Variant in OtherVariants do
Documents[Variant] := Documents[Low(TVariants)].CloneNode(True);
{$IFDEF TIMINGS} Writeln('Elapsed time: ', MillisecondsBetween(StartTime, Now()), 'ms'); {$ENDIF}
end;
try
try
// gen...
for Variant in TVariants do
if (BuildType = 'default') then
begin
MkDir(OutputDirectory + '/multipage-' + kSuffixes[Variant]);
Inform('Generating ' + Uppercase(kSuffixes[Variant]) + ' variant...');
{$IFDEF TIMINGS} StartTime := Now(); {$ENDIF}
ProcessDocument(Documents[Variant], Variant, BigTOC); // $R-
{$IFDEF TIMINGS} Writeln('Elapsed time: ', MillisecondsBetween(StartTime, Now()), 'ms'); {$ENDIF}
// output...
if (Variant <> vDEV) then
for Variant in TVariants do
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There's some copying of documents earlier on that also does that for all variants. I suppose we could add additional branching there too to save a bit of a processing time.

begin
{$IFDEF TIMINGS} Writeln('Saving single-page version...'); {$ENDIF}
if (Variant = vReview) then
begin
continue;
end;
// Create this directory early as ProcessDocument relies on it to store
// /multipage-dev/search-index.json
if (Variant <> vSnap) then
begin
MkDir(OutputDirectory + '/multipage-' + kSuffixes[Variant]);
end;
Inform('Generating ' + Uppercase(kSuffixes[Variant]) + ' variant...');
{$IFDEF TIMINGS} StartTime := Now(); {$ENDIF}
Save(Documents[Variant], OutputDirectory + '/index-' + kSuffixes[Variant]);
ProcessDocument(Documents[Variant], Variant, BigTOC, SourceGitSHA); // $R-
{$IFDEF TIMINGS} Writeln('Elapsed time: ', MillisecondsBetween(StartTime, Now()), 'ms'); {$ENDIF}
// output...
if (Variant <> vDEV) then
begin
{$IFDEF TIMINGS} Writeln('Saving single-page version...'); {$ENDIF}
{$IFDEF TIMINGS} StartTime := Now(); {$ENDIF}
Save(Documents[Variant], OutputDirectory + '/index-' + kSuffixes[Variant]);
{$IFDEF TIMINGS} Writeln('Elapsed time: ', MillisecondsBetween(StartTime, Now()), 'ms'); {$ENDIF}
end;
// multipage...
if (Variant <> vSnap) then
begin
{$IFDEF TIMINGS} Writeln('Splitting spec...'); {$ENDIF}
{$IFDEF TIMINGS} StartTime := Now(); {$ENDIF}
if (not Split(Documents[Variant], BigTOC, OutputDirectory + '/multipage-' + kSuffixes[Variant] + '/')) then
raise EAbort.Create('Could not split specification');
{$IFDEF TIMINGS} Writeln('Elapsed time: ', MillisecondsBetween(StartTime, Now()), 'ms'); {$ENDIF}
end;
end;
// multipage...
{$IFDEF TIMINGS} Writeln('Splitting spec...'); {$ENDIF}
{$IFDEF TIMINGS} StartTime := Now(); {$ENDIF}
if (not Split(Documents[Variant], BigTOC, OutputDirectory + '/multipage-' + kSuffixes[Variant] + '/')) then
raise EAbort.Create('Could not split specification');
{$IFDEF TIMINGS} Writeln('Elapsed time: ', MillisecondsBetween(StartTime, Now()), 'ms'); {$ENDIF}
end
else
begin
Assert(BuildType = 'review');
// Skip timing information here as it should be roughly equivalent
Inform('Generating ' + Uppercase(kSuffixes[vReview]) + ' exclusively...');
ProcessDocument(Documents[Low(TVariants)], vReview, BigTOC, SourceGitSHA);
Save(Documents[Low(TVariants)], OutputDirectory + '/index-' + kSuffixes[vReview]);
end;
Result := True;
except
Expand All @@ -2633,8 +2687,15 @@ function Main(): Boolean;
end;
finally
try
for Variant in TVariants do
Documents[Variant].Free();
if (BuildType = 'default') then
begin
for Variant in TVariants do
Documents[Variant].Free();
end
else
begin
Documents[Low(TVariants)].Free();
end;
except
ReportCurrentException();
end;
Expand Down