Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] adding exit code to the preprocessor #1533

Draft
wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

[WIP] adding exit code to the preprocessor #1533

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
96b33b1
[WIP] adding exit code to the preprocessor
karimtera Nov 16, 2022
5662702
[WIP] implementing exit codes into VerilogPreprocess
karimtera Nov 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 56 additions & 19 deletions verilog/preprocessor/verilog_preprocess.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,11 +59,14 @@ VerilogPreprocess::VerilogPreprocess(const Config& config, FileOpener opener)
}

TokenStreamView::const_iterator VerilogPreprocess::GenerateBypassWhiteSpaces(
const StreamIteratorGenerator& generator) {
auto iterator =
generator(); // iterator should be pointing to a non-whitespace token;
const StreamIteratorGenerator& generator,
std::vector<TokenStreamView::const_iterator>* backup_view = nullptr) {
// iterator should be pointing to a non-whitespace token;
auto iterator = generator();
if (backup_view != nullptr) backup_view->push_back(iterator);
while (verilog::VerilogLexer::KeepSyntaxTreeTokens(**iterator) == 0) {
iterator = generator();
if (backup_view != nullptr) backup_view->push_back(iterator);
}
return iterator;
}
Expand Down Expand Up @@ -223,7 +226,8 @@ std::unique_ptr<VerilogPreprocessError> VerilogPreprocess::ParseMacroDefinition(
absl::Status VerilogPreprocess::ConsumeAndParseMacroCall(
TokenStreamView::const_iterator iter,
const StreamIteratorGenerator& generator, verible::MacroCall* macro_call,
const verible::MacroDefinition& macro_definition) {
const verible::MacroDefinition& macro_definition,
std::vector<TokenStreamView::const_iterator>& backup_view) {
// Parsing the macro .
const absl::string_view macro_name_str = (*iter)->text().substr(1);
verible::TokenInfo macro_name_token(MacroCallId, macro_name_str);
Expand All @@ -238,10 +242,11 @@ absl::Status VerilogPreprocess::ConsumeAndParseMacroCall(

// Parsing parameters.
TokenStreamView::const_iterator token_iter =
GenerateBypassWhiteSpaces(generator);
GenerateBypassWhiteSpaces(generator, &backup_view);
int parameters_size = macro_definition.Parameters().size();
if ((*token_iter)->text() == "(") {
token_iter = GenerateBypassWhiteSpaces(generator); // skip the "("
token_iter =
GenerateBypassWhiteSpaces(generator, &backup_view); // skip the "("
} else {
return absl::InvalidArgumentError(
"Error it is illegal to call a callable macro without ().");
Expand All @@ -250,15 +255,15 @@ absl::Status VerilogPreprocess::ConsumeAndParseMacroCall(
while (parameters_size > 0) {
if ((*token_iter)->token_enum() == MacroArg) {
macro_call->positional_arguments.emplace_back(**token_iter);
token_iter = GenerateBypassWhiteSpaces(generator);
token_iter = GenerateBypassWhiteSpaces(generator, &backup_view);
if ((*token_iter)->text() == ",")
token_iter = GenerateBypassWhiteSpaces(generator);
token_iter = GenerateBypassWhiteSpaces(generator, &backup_view);
parameters_size--;
continue;
} else if ((*token_iter)->text() == ",") {
macro_call->positional_arguments.emplace_back(
verible::DefaultTokenInfo());
token_iter = GenerateBypassWhiteSpaces(generator);
token_iter = GenerateBypassWhiteSpaces(generator, &backup_view);
parameters_size--;
continue;
} else if ((*token_iter)->text() == ")")
Expand Down Expand Up @@ -287,22 +292,40 @@ absl::Status VerilogPreprocess::HandleMacroIdentifier(
const auto* found =
FindOrNull(preprocess_data_.macro_definitions, sv.substr(1));
if (!found) {
// Outputs the macro call if it couldn't be expanded.
preprocess_data_.preprocessed_token_stream.push_back(*iter);

// Updates the exit code.
preprocess_data_.exit_code |= kMacroUseNotExpandedBit;

preprocess_data_.errors.push_back(VerilogPreprocessError(
**iter,
"Error expanding macro identifier, might not be defined before."));
return absl::InvalidArgumentError(
"Error expanding macro identifier, might not be defined before.");
}

if (config_.expand_macros) {
verible::MacroCall macro_call;
if (auto status =
ConsumeAndParseMacroCall(iter, generator, &macro_call, *found);
!status.ok())
return status;
if (auto status = ExpandMacro(macro_call, found); !status.ok())
return status;
// Backup of const_iterators to push back into
// preprocessor_data_.preprocessed_token_stream in case we couldn't open the
// file.
std::vector<TokenStreamView::const_iterator> backup_view;
backup_view.push_back(iter);

verible::MacroCall macro_call;
if (auto status = ConsumeAndParseMacroCall(iter, generator, &macro_call,
*found, backup_view);
!status.ok()) {
// Outputs the macro call if it couldn't be expanded.
for (const auto& u : backup_view)
preprocess_data_.preprocessed_token_stream.push_back(*u);

// Updates the exit code.
preprocess_data_.exit_code |= kMacroUseNotExpandedBit;

return status;
}
if (auto status = ExpandMacro(macro_call, found); !status.ok()) return status;

auto& lexed = preprocess_data_.lexed_macros_backup.back();
if (!forward) return absl::OkStatus();
auto iter_generator = verible::MakeConstIteratorStreamer(lexed);
Expand Down Expand Up @@ -592,11 +615,17 @@ absl::Status VerilogPreprocess::HandleInclude(
if (!file_opener_)
return absl::FailedPreconditionError("file_opener_ is not defined");

// Backup of const_iterators to push back into
// preprocessor_data_.preprocessed_token_stream in case we couldn't open the
// file.
std::vector<TokenStreamView::const_iterator> backup_view;
backup_view.push_back(iter);

// TODO(karimtera): Support inclduing <file>,
// which should look for files defined by language standard in a compiler
// dependent path.
TokenStreamView::const_iterator token_iter =
GenerateBypassWhiteSpaces(generator);
GenerateBypassWhiteSpaces(generator, &backup_view);
auto file_token_iter = *token_iter;
if (file_token_iter->token_enum() != TK_StringLiteral) {
preprocess_data_.errors.push_back(
Expand All @@ -614,6 +643,14 @@ absl::Status VerilogPreprocess::HandleInclude(
if (!status_or_file.ok()) {
preprocess_data_.errors.push_back(
{**token_iter, std::string(status_or_file.status().message())});
// Outputs the `include directive as it is, in case we couldn't open the
// file.
for (const auto& u : backup_view)
preprocess_data_.preprocessed_token_stream.push_back(*u);

// Add this error to the exit code.
preprocess_data_.exit_code |= kIncludeFileNotFoundBit;

return status_or_file.status();
}
const absl::string_view source_contents = *status_or_file;
Expand Down Expand Up @@ -740,7 +777,7 @@ VerilogPreprocessData VerilogPreprocess::ScanStream(
const auto status = HandleTokenIterator(iter, iter_generator);
if (!status.ok()) {
// Detailed errors are already in preprocessor_data_.errors.
break; // For now, stop after first error.
/* break; // For now, stop after first error. */
}
}

Expand Down
33 changes: 24 additions & 9 deletions verilog/preprocessor/verilog_preprocess.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.

// VerilogPreprocess is a *pseudo*-preprocessor for Verilog.
// Unlike a conventional preprocessor, this pseudo-preprocessor does not open
Expand All @@ -35,6 +35,7 @@
#ifndef VERIBLE_VERILOG_PREPROCESSOR_VERILOG_PREPROCESS_H_
#define VERIBLE_VERILOG_PREPROCESSOR_VERILOG_PREPROCESS_H_

#include <cstdint>
#include <functional>
#include <map>
#include <memory>
Expand Down Expand Up @@ -87,6 +88,10 @@ struct VerilogPreprocessData {
// are two separate vectors.
std::vector<VerilogPreprocessError> errors;
std::vector<VerilogPreprocessError> warnings;

// An exit code of 4 bits is used to distinguish between errors met during
// preprocessing.
std::uint32_t exit_code{0};
};

// VerilogPreprocess transforms a TokenStreamView.
Expand Down Expand Up @@ -164,10 +169,10 @@ class VerilogPreprocess {
absl::Status HandleElse(TokenStreamView::const_iterator else_pos);
absl::Status HandleEndif(TokenStreamView::const_iterator endif_pos);

static absl::Status ConsumeAndParseMacroCall(TokenStreamView::const_iterator,
const StreamIteratorGenerator&,
verible::MacroCall*,
const verible::MacroDefinition&);
static absl::Status ConsumeAndParseMacroCall(
TokenStreamView::const_iterator, const StreamIteratorGenerator&,
verible::MacroCall*, const verible::MacroDefinition&,
std::vector<TokenStreamView::const_iterator>&);

// The following functions return nullptr when there is no error:
absl::Status ConsumeMacroDefinition(const StreamIteratorGenerator&,
Expand All @@ -188,7 +193,8 @@ class VerilogPreprocess {

// Generate a const_iterator to a non-whitespace token.
static TokenStreamView::const_iterator GenerateBypassWhiteSpaces(
const StreamIteratorGenerator&);
const StreamIteratorGenerator&,
std::vector<TokenStreamView::const_iterator>*);

const Config config_;

Expand Down Expand Up @@ -253,6 +259,15 @@ class VerilogPreprocess {
// A pointer to a file opener function.
// This is needed for opening new files while handling includes.
const FileOpener file_opener_ = nullptr;

enum ResultBits {
// file not found or tokenization error
kFileErrorBit = (1 << 0),
// At least one macro could not be expanded
kMacroUseNotExpandedBit = (1 << 1),
// At least one include file could not be found
kIncludeFileNotFoundBit = (1 << 2)
};
};

} // namespace verilog
Expand Down
7 changes: 5 additions & 2 deletions verilog/tools/preprocessor/verilog_preprocessor.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,9 +124,12 @@ static absl::Status PreprocessSingleFile(
preprocessor.ScanStream(lexed_streamview);
auto& preprocessed_stream = preprocessed_data.preprocessed_token_stream;
for (auto u : preprocessed_stream) outs << u->text();
for (auto& u : preprocessed_data.errors) outs << u.error_message << '\n';
if (!preprocessed_data.errors.empty())
for (auto& u : preprocessed_data.errors)
message_stream << u.error_message << '\n';
if (!preprocessed_data.errors.empty()) {
message_stream << "Exit code: " << preprocessed_data.exit_code << "\n";
return absl::InvalidArgumentError("Error: The preprocessing has failed.");
}
return absl::OkStatus();
}

Expand Down