[WIP] Fix issue 166

.appveyor.yml

@@ -13,7 +13,7 @@ environment:
     - GENERATOR: -G"Visual Studio 16 2019" -A x64
       APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
       BUILD_TYPE: Release
-      PYTHON: "C:\\Python39-x64"
+      PYTHON: "C:\\Python312-x64"
       # add to path otherwise Cython is not found
       PATH: C:\Program Files\Java\jdk11\bin;$(PYTHON)\Scripts;$(PATH)
       PLATFORM: x64

.gitattributes

@@ -0,0 +1,10 @@
+.appveyor.yml export-ignore
+.circleci/ export-ignore
+.github export-ignore
+Dockerfile export-ignore
+.dockerignore export-ignore
+tests/** export-ignore
+tests/ghdl/** export-ignore
+tests/ghdl/ export-ignore
+tests/ghdl/logo.png export-ignore
+notebooks export-ignore

.github/workflows/build_and_release.yml

@@ -8,14 +8,14 @@ permissions:
 jobs:
   # Create a Debug build and test it.
   build_and_test:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
       with:
         submodules: recursive
     - uses: actions/setup-python@v5
       with:
-        python-version: "3.10"
+        python-version: "3.12"
 
     # Don't need to install Python / Create venv, as you are running in a container.
     # You don't even need to install g++/cmake as it comes with the docker image.
@@ -26,6 +26,7 @@ jobs:
                 libantlr4-runtime-dev antlr4 libssl-dev -yq
         pip install setuptools wheel cmake ninja patchelf auditwheel
         pip install -r requirements.txt
+        pip install build nbconvert
 
     - name: Check Your Environment
       run: |
@@ -36,16 +37,18 @@ jobs:
 
     - name: Build
       run: |
-        echo Your Build Script here
-        echo Make sure the Debug and Coverage Switch is on, only in this job.
+        python -m build -Ccompile-args="--verbose" -Ccompile-args="-j4" . # -j4 because we do not have enough RAM
+        pip install dist/*.whl
 
     - name: Test
       run: |
-        echo Your Test Script here
+        mv hdlConvertor tmp_hdlConvertor
+        python -m tests.all
+        mv tmp_hdlConvertor hdlConvertor
 
     - name: Extract Coverage Report
       run: |
-        echo Your Coverage Report
+        echo "[todo] Coverage Report"
 
 
   # Build on Different Platform.
@@ -63,8 +66,8 @@ jobs:
 
     strategy:
         matrix:
-          python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11", "3.12" ]
-    runs-on: ubuntu-20.04
+          python-version: [ "3.9", "3.10", "3.11", "3.12" ]
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
       with:
@@ -79,6 +82,7 @@ jobs:
                 libantlr4-runtime-dev antlr4 libssl-dev -yq
         pip install setuptools wheel cmake ninja patchelf auditwheel
         pip install -r requirements.txt
+        pip install build
 
     # Include ANTLR4's License file as we are redistributing their binary
     - name: Adding ANTLR4 License
@@ -91,11 +95,11 @@ jobs:
     # Use them with `-j 4` flag
     - name: Build Library
       run: |
-        python setup.py bdist_wheel -j 4
+        python -m build .
         auditwheel repair --plat manylinux_2_31_x86_64 dist/*.whl
 
     - name: Archive Artifacts
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: PythonPackage
         path: |
@@ -105,7 +109,7 @@ jobs:
   # Developer can still build on their machine with the source bundle
   # (i.e. the original `pip install` flow if the platform is not included by above steps)  
   build_sdist:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     if: ${{ success() }}
 
     needs:
@@ -117,7 +121,7 @@ jobs:
         submodules: recursive
     - uses: actions/setup-python@v5
       with:
-        python-version: "3.9"
+        python-version: "3.12"
     - name: Install Dependencies
       run: |
         sudo apt update -y
@@ -133,7 +137,7 @@ jobs:
         python setup.py sdist
 
     - name: Archive Artifacts
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: PythonPackage
         path: |
@@ -160,7 +164,7 @@ jobs:
 
     steps:
     - name: Download Package Built
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: PythonPackage
         path: dist

grammars/verilogPreprocLexer.g4

@@ -359,11 +359,15 @@ mode KEYWOORDS_MODE;
 // include argument processing
 mode INCLUDE_MODE;
     INCLUDE_MODE_STR
-        :  STR -> type(STR),popMode
+        :  STR -> type(STR)
         ;
+
     INCLUDE_MODE_STR_CHEVRONS
-        :  '<' ( ~('\\'|'>') )* '>' -> type(STR),popMode
+        :  '<' ( ~('\\'|'>') )* '>' -> type(STR)
         ;
+    INCLUDE_LINE_COMMENT : LINE_COMMENT -> popMode,type(LINE_COMMENT),channel(CH_LINE_COMMENT);
+    INCLUDE_MODE_NEW_LINE: CRLF -> type(NEW_LINE),popMode,skip;
+
     INCLUDE_MODE_MACRO_ENTER: '`' -> popMode,pushMode(DIRECTIVE_MODE),skip;
     INCLUDE_MODE_WS : WS ->skip;
 

hdlConvertor/toPy.cpp

@@ -190,6 +190,8 @@ PyObject* ToPy::toPy(const CodePosition o) {
 		return nullptr;
 	if (toPy_property(py_inst, "stop_column", o.stop_column))
 		return nullptr;
+	if (toPy_property(py_inst, "file_name", o.file_name))
+		return nullptr;
 	return py_inst;
 }
 

include/hdlConvertor/hdlAst/codePosition.h

@@ -2,6 +2,8 @@
 #include <stddef.h>
 #include <limits>
 
+#include <hdlConvertor/hdlConvertorToken.h>
+
 namespace hdlConvertor {
 namespace hdlAst {
 
@@ -13,22 +15,34 @@ namespace hdlAst {
 class CodePosition {
 public:
 	static constexpr size_t INVALID = std::numeric_limits<size_t>::max();
+	static constexpr char INVALID_STR[] = "<INVSTR>";
 
 	size_t start_line;
 	size_t stop_line;
 	size_t start_column;
 	size_t stop_column;
+	std::string file_name;
 
 	CodePosition();
 	CodePosition(size_t startLine, size_t stopLine, size_t startColumn,
-			size_t stopColumn);
+			size_t stopColumn, std::string fileName);
 	template<class ELEM_T>
 	void update_from_elem(ELEM_T *elem) {
-		start_line = elem->getStart()->getLine();
-		stop_line = elem->getStop()->getLine();
-		start_column = elem->getStart()->getCharPositionInLine() + 1;
-		stop_column = elem->getStop()->getCharPositionInLine() +
-				(elem->getStop()->getStopIndex() - elem->getStop()->getStartIndex()) + 1;
+		/*
+		 * Cast antlr4::CommonToken to our custum hdlConvertorToken
+		 * So we can retrive the file name information of origin of
+		 * current token.
+		 */
+		hdlConvertor::hdlConvertorToken * tokenStart_ptr = dynamic_cast<hdlConvertor::hdlConvertorToken *>(elem->getStart());
+		hdlConvertor::hdlConvertorToken * tokenStop_ptr = dynamic_cast<hdlConvertor::hdlConvertorToken *>(elem->getStop());
+
+		/* pick up data from token */
+		start_line = tokenStart_ptr->getLine();
+		stop_line = tokenStop_ptr->getLine();
+		start_column = tokenStart_ptr->getCharPositionInLine() + 1;
+		stop_column = tokenStop_ptr->getCharPositionInLine() +
+				(tokenStop_ptr->getStopIndex() - tokenStop_ptr->getStartIndex()) + 1;
+		file_name = tokenStart_ptr->getSrcName();
 	}
 	bool isKnown() const;
 };

include/hdlConvertor/hdlConvertor.h

@@ -12,6 +12,7 @@
 #include <hdlConvertor/parserContainer.h>
 #include <hdlConvertor/hdlAst/hdlContext.h>
 #include <hdlConvertor/verilogPreproc/verilogPreproc.h>
+#include <hdlConvertor/hdlConvertorTokenFactory.h>
 
 namespace hdlConvertor {
 

include/hdlConvertor/hdlConvertorToken.h

@@ -0,0 +1,23 @@
+#pragma once
+
+#include <antlr4-runtime.h>
+
+namespace hdlConvertor {
+
+class hdlConvertorToken : public antlr4::CommonToken {
+	public:
+	hdlConvertorToken(size_t type);
+    	hdlConvertorToken(std::pair<antlr4::TokenSource*, antlr4::CharStream*> source, size_t type, size_t channel, size_t start, size_t stop);
+	hdlConvertorToken(size_t type, const std::string &text);
+	hdlConvertorToken(antlr4::Token *oldToken);
+
+	void setSrcName(const std::string fileName);
+	std::string getSrcName();
+	std::string toString() const override;
+
+	private:
+		std::string srcName;
+
+};
+
+}

include/hdlConvertor/hdlConvertorTokenFactory.h

@@ -0,0 +1,36 @@
+#pragma once
+
+#include <hdlConvertor/verilogPreproc/file_line_map.h>
+#include <hdlConvertor/hdlConvertorToken.h>
+#include <antlr4-runtime.h>
+namespace hdlConvertor {
+
+class hdlConvertorTokenFactory : public antlr4::CommonTokenFactory {
+	public:
+        static const std::unique_ptr<antlr4::TokenFactory<antlr4::CommonToken>> DEFAULT;
+
+	hdlConvertorTokenFactory(bool copy_text);
+	hdlConvertorTokenFactory();
+	void setFileLineMap( verilog_pp::FileLineMap * file_line_map);
+
+	std::unique_ptr<antlr4::CommonToken> create(
+			std::pair<antlr4::TokenSource*, antlr4::CharStream*> source,
+			size_t type,
+      			const std::string &text,
+			size_t channel,
+			size_t start,
+			size_t stop,
+			size_t line,
+			size_t charPositionInLine) override;
+
+	std::unique_ptr<antlr4::CommonToken> create(
+			size_t type,
+			const std::string &text) override;
+
+
+	private:
+	static verilog_pp::FileLineMap *_file_line_map;
+
+};
+
+}

meson.build

@@ -23,7 +23,7 @@ cc = meson.get_compiler('cpp')
 EXTRA_CXX_FLAGS = []
 EXTRA_LD_FLAGS = []
 if meson.get_compiler('cpp').get_id() == 'msvc'
-	EXTRA_CXX_FLAGS.append('/bigobj')
+	EXTRA_CXX_FLAGS += ['-MP /wd4251', '/bigobj'] # https://github.com/antlr/antlr4/blob/dev/runtime/Cpp/demo/CMakeLists.txt
 	# using /MD flag for antlr4_runtime (for Visual C++ compilers only)
 elif host_machine.system() == 'cygwin'
 	EXTRA_CXX_FLAGS += ['-Wa,-mbig-obj']
@@ -50,7 +50,7 @@ endif
 	antlr4_runtime_opt.set_install(false) # because meson-python does not know where to install it
 	antlr4_runtime_proj = cmake.subproject('antlr4-runtime', options: antlr4_runtime_opt)
 	antlr4_cpp_dep = antlr4_runtime_proj.dependency('antlr4_static')
-	antlr4_cpp_version = '4.13.1'
+	antlr4_cpp_version = '4.13.2' # :note: must match version in antlr4-runtime.wrap
 	import('java') # to check that java is installed
 	antlr4_jar = join_paths(meson.current_build_dir() , 'antlr4-complete.jar')
 	fs = import('fs')
@@ -88,6 +88,7 @@ curSrcDir = meson.current_source_dir()
 mainIncludeDir = 'include/'
 mainIncludeDir_inc = include_directories(mainIncludeDir)
 all_parser_libs = []
+all_parser_deps = []
 foreach parserTuple : ANTLR_PARSERS
 	name = parserTuple[0]
 	folder_name = parserTuple[1]
@@ -157,10 +158,21 @@ foreach parserTuple : ANTLR_PARSERS
         include_directories: [mainIncludeDir_inc, include_directories(genIncludeDir)],
     )
     all_parser_libs += parser_lib
+    parser_lib_dependency = declare_dependency(
+         link_with : [parser_lib],)
+    all_parser_deps += parser_lib_dependency
 endforeach
 
 subdir('src')
 
 if get_option('python_package')
 	subdir('hdlConvertor')
 endif
+
+# if tests files are in the package mark them as test data
+test_data = run_command(python_prog, 'utils/rec_glob.py', 'tests', '*', check:true)\
+	.stdout().strip().split('\n')
+message('test_data:\n' + '\n'.join(test_data))
+
+install_data(test_data, install_tag : 'tests')
+

pyproject.toml

@@ -3,8 +3,8 @@ requires = [
   "wheel", # python package distribution lib.
   "Cython",
   "cmake>=3.26.3", # cmake buildsystem for antrl
-  "meson>=1.1.0", # meson buildsystem
-  'meson-python>=0.12.1', # meson to pip adapter
+  "meson>=1.8.2", # meson buildsystem
+  'meson-python>=0.18.0', # meson to pip adapter
   "pybind11>=2.10.4", # c++ to Python binding
   "ninja>=1.11.1", # build tool for meson to use
 ]
@@ -19,7 +19,7 @@ dynamic = ["version", ] # specified in meson.build
 # readme = {file = ["README.md"]}
 readme = "README.md"
 license = { file = 'LICENSE' }
-keyword = ['FPGA', 'ASIC', 'compiler', 'RTL', 'synthesis', 'logicsynthesis',
+keywords = ['FPGA', 'ASIC', 'compiler', 'RTL', 'synthesis', 'logicsynthesis',
            'hdl', 'vhdl', 'parser', 'preprocessor', 'antlr4', 'verification',
            'Verilog', 'SystemVerilog', 'system verilog']
 authors = [

src/hdlAst/codePosition.cpp

@@ -4,20 +4,21 @@ namespace hdlConvertor {
 namespace hdlAst {
 
 CodePosition::CodePosition() :
-		CodePosition(INVALID, INVALID, INVALID, INVALID) {
+		CodePosition(INVALID, INVALID, INVALID, INVALID,INVALID_STR) {
 }
 
 CodePosition::CodePosition(size_t startLine, size_t stopLine, size_t startColumn,
-		size_t stopColumn) {
+		size_t stopColumn,std::string fileName) {
 	this->start_line = startLine;
 	this->stop_line = stopLine;
 	this->start_column = startColumn;
 	this->stop_column = stopColumn;
+	this->file_name = fileName;
 }
 
 bool CodePosition::isKnown() const {
 	return start_line != INVALID || stop_line != INVALID || start_column != INVALID
-			|| stop_column != INVALID;
+			|| stop_column != INVALID || file_name != INVALID_STR;
 }
 
 }