Use compressed grammar tables for core instructions and operands

Android.mk

@@ -25,6 +25,7 @@ SPVTOOLS_SRC_FILES := \
 		source/spirv_target_env.cpp \
 		source/spirv_validator_options.cpp \
 		source/table.cpp \
+		source/table2.cpp \
 		source/text.cpp \
 		source/text_handler.cpp \
 		source/to_string.cpp \
@@ -206,24 +207,20 @@ SPV_CLDEBUGINFO100_GRAMMAR=$(SPVHEADERS_LOCAL_PATH)/include/spirv/unified1/extin
 SPV_VKDEBUGINFO100_GRAMMAR=$(SPVHEADERS_LOCAL_PATH)/include/spirv/unified1/extinst.nonsemantic.shader.debuginfo.100.grammar.json
 
 define gen_spvtools_grammar_tables
-$(call generate-file-dir,$(1)/core.insts-unified1.inc)
-$(1)/core.insts-unified1.inc $(1)/operand.kinds-unified1.inc \
+$(call generate-file-dir,$(1)/core_tables.inc)
+$(1)/core_tables.inc \
 : \
         $(LOCAL_PATH)/utils/generate_grammar_tables.py \
         $(SPV_COREUNIFIED1_GRAMMAR) \
-        $(SPV_GLSL_GRAMMAR) \
-        $(SPV_OpenCL_GRAMMAR) \
         $(SPV_DEBUGINFO_GRAMMAR) \
         $(SPV_CLDEBUGINFO100_GRAMMAR)
-		@$(HOST_PYTHON) $(LOCAL_PATH)/utils/generate_grammar_tables.py \
+		@$(HOST_PYTHON) $(LOCAL_PATH)/utils/ggt.py \
 		                --spirv-core-grammar=$(SPV_COREUNIFIED1_GRAMMAR) \
 		                --extinst-debuginfo-grammar=$(SPV_DEBUGINFO_GRAMMAR) \
 		                --extinst-cldebuginfo100-grammar=$(SPV_CLDEBUGINFO100_GRAMMAR) \
-		                --core-insts-output=$(1)/core.insts-unified1.inc \
-		                --operand-kinds-output=$(1)/operand.kinds-unified1.inc
+		                --core-tables-output=$(1)/core_tables.inc
 		@echo "[$(TARGET_ARCH_ABI)] Grammar (from unified1)  : instructions & operands <= grammar JSON files"
-$(LOCAL_PATH)/source/opcode.cpp: $(1)/core.insts-unified1.inc
-$(LOCAL_PATH)/source/operand.cpp: $(1)/operand.kinds-unified1.inc
+$(LOCAL_PATH)/source/table2.cpp: $(1)/core_tables.inc
 $(LOCAL_PATH)/source/ext_inst.cpp: \
 	$(1)/glsl.std.450.insts.inc \
 	$(1)/opencl.std.100.insts.inc \

BUILD.bazel

@@ -5,7 +5,7 @@ load(
     "DEBUGINFO_GRAMMAR_JSON_FILE",
     "SHDEBUGINFO100_GRAMMAR_JSON_FILE",
     "TEST_COPTS",
-    "generate_core_tables",
+    "generate_compressed_tables",
     "generate_enum_string_mapping",
     "generate_extinst_lang_headers",
     "generate_vendor_tables",
@@ -27,16 +27,35 @@ exports_files([
 ])
 
 py_binary(
+    # Generates tables for extensions and extended instruction sets.
+    # TODO(crbug.com/266223071) Replace this with ggt.py
     name = "generate_grammar_tables",
     srcs = ["utils/generate_grammar_tables.py"],
 )
 
+py_binary(
+    # The script that generates compressed grammar tables for
+    # instructions and operands.
+    # TODO(crbug.com/266223071) Eventually this will fully replace
+    # utils/generate_grammar_tables.py.
+    name = "ggt",
+    main = "utils/ggt.py", # The file found by $(location :ggt)
+    srcs = [
+        "utils/ggt.py",
+        "utils/Table/__init__.py",
+        "utils/Table/Context.py",
+        "utils/Table/IndexRange.py",
+        "utils/Table/Operand.py",
+        "utils/Table/StringList.py",
+    ],
+)
+
 py_binary(
     name = "generate_language_headers",
     srcs = ["utils/generate_language_headers.py"],
 )
 
-generate_core_tables(version = "unified1")
+generate_compressed_tables()
 
 generate_enum_string_mapping(version = "unified1")
 
@@ -142,7 +161,7 @@ cc_library(
         "source/val/*.cpp",
     ]) + [
         ":build_version_inc",
-        ":gen_core_tables_unified1",
+        ":gen_compressed_tables",
         ":gen_enum_string_mapping",
         ":gen_extinst_lang_headers_DebugInfo",
         ":gen_extinst_lang_headers_NonSemanticShaderDebugInfo100",

BUILD.gn

@@ -34,41 +34,35 @@ spirv_headers = spirv_tools_spirv_headers_dir
 spirv_is_winuwp = is_win && target_os == "winuwp"
 
 template("spvtools_core_tables") {
-  assert(defined(invoker.version), "Need version in $target_name generation.")
-
+  # Generate a compressed table describing SPIR-V core spec instructions and operands.
   action("spvtools_core_tables_" + target_name) {
-    script = "utils/generate_grammar_tables.py"
-
-    version = invoker.version
+    script = "utils/ggt.py"
 
     core_json_file =
-        "${spirv_headers}/include/spirv/$version/spirv.core.grammar.json"
-    core_insts_file = "${target_gen_dir}/core.insts-$version.inc"
-    operand_kinds_file = "${target_gen_dir}/operand.kinds-$version.inc"
+        "${spirv_headers}/include/spirv/unified1/spirv.core.grammar.json"
     debuginfo_insts_file =
         "${spirv_headers}/include/spirv/unified1/extinst.debuginfo.grammar.json"
     cldebuginfo100_insts_file = "${spirv_headers}/include/spirv/unified1/extinst.opencl.debuginfo.100.grammar.json"
 
+    core_tables_file = "${target_gen_dir}/core_tables.inc"
+
     sources = [
       cldebuginfo100_insts_file,
       core_json_file,
       debuginfo_insts_file,
     ]
     outputs = [
-      core_insts_file,
-      operand_kinds_file,
+      core_tables_file,
     ]
     args = [
       "--spirv-core-grammar",
       rebase_path(core_json_file, root_build_dir),
-      "--core-insts-output",
-      rebase_path(core_insts_file, root_build_dir),
       "--extinst-debuginfo-grammar",
       rebase_path(debuginfo_insts_file, root_build_dir),
       "--extinst-cldebuginfo100-grammar",
       rebase_path(cldebuginfo100_insts_file, root_build_dir),
-      "--operand-kinds-output",
-      rebase_path(operand_kinds_file, root_build_dir)
+      "--core-tables-output",
+      rebase_path(core_tables_file, root_build_dir)
     ]
   }
 }
@@ -267,7 +261,6 @@ action("spvtools_build_version") {
 }
 
 spvtools_core_tables("unified1") {
-  version = "unified1"
 }
 spvtools_core_enums("unified1") {
   version = "unified1"

CMakeLists.txt

@@ -240,6 +240,20 @@ endif()
 # Tests require Python3
 find_host_package(Python3 REQUIRED)
 
+# Check type annotations in Perl code. Assumes mypy.
+add_custom_target(spirv-tools-check-python-types
+	COMMAND mypy Table
+	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/utils
+)
+set_target_properties(spirv-tools-check-python-types
+  PROPERTIES EXCLUDE_FROM_ALL ON)
+
+# Run Python unit tests
+add_custom_target(spirv-tools-check-python-tests
+	COMMAND Python3::Interpreter -m unittest discover -v -p "*test.py"
+	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/utils
+)
+
 # Check for symbol exports on Linux.
 # At the moment, this check will fail on the OSX build machines for the Android NDK.
 # It appears they don't have objdump.

build_defs.bzl

@@ -47,6 +47,7 @@ def incompatible_with(incompatible_constraints):
         for constraint in incompatible_constraints
     }]))
 
+SPIRV_CORE_GRAMMAR_JSON_FILE = "@spirv_headers//:spirv_core_grammar_unified1"
 DEBUGINFO_GRAMMAR_JSON_FILE = "@spirv_headers//:spirv_ext_inst_debuginfo_grammar_unified1"
 CLDEBUGINFO100_GRAMMAR_JSON_FILE = "@spirv_headers//:spirv_ext_inst_opencl_debuginfo_100_grammar_unified1"
 SHDEBUGINFO100_GRAMMAR_JSON_FILE = "@spirv_headers//:spirv_ext_inst_nonsemantic_shader_debuginfo_100_grammar_unified1"
@@ -57,6 +58,7 @@ def _merge_dicts(dicts):
         merged.update(d)
     return merged
 
+# TODO(b/413743565): Remove after legacy grammars removed.
 def generate_core_tables(version):
     if not version:
         fail("Must specify version", "version")
@@ -91,6 +93,35 @@ def generate_core_tables(version):
         visibility = ["//visibility:private"],
     )
 
+def generate_compressed_tables():
+    grammars = dict(
+        core_grammar = SPIRV_CORE_GRAMMAR_JSON_FILE,
+        debuginfo_grammar = DEBUGINFO_GRAMMAR_JSON_FILE,
+        cldebuginfo_grammar = CLDEBUGINFO100_GRAMMAR_JSON_FILE,
+    )
+
+    outs = dict(
+        core_tables_output = "core_tables.inc",
+    )
+
+    cmd = (
+        "$(location :ggt)" +
+        " --spirv-core-grammar=$(location {core_grammar})" +
+        " --extinst-debuginfo-grammar=$(location {debuginfo_grammar})" +
+        " --extinst-cldebuginfo100-grammar=$(location {cldebuginfo_grammar})" +
+        " --core-tables-output=$(location {core_tables_output})"
+    ).format(**_merge_dicts([grammars, outs]))
+
+    native.genrule(
+        name = "gen_compressed_tables",
+        srcs = grammars.values(),
+        outs = outs.values(),
+        cmd = cmd,
+        cmd_bat = cmd,
+        tools = [":ggt"],
+        visibility = ["//visibility:private"],
+    )
+
 def generate_enum_string_mapping(version):
     if not version:
         fail("Must specify version", "version")

source/CMakeLists.txt

@@ -16,35 +16,27 @@ set(GRAMMAR_PROCESSING_SCRIPT "${spirv-tools_SOURCE_DIR}/utils/generate_grammar_
 set(VIMSYNTAX_PROCESSING_SCRIPT "${spirv-tools_SOURCE_DIR}/utils/vim/generate_syntax.py")
 set(XML_REGISTRY_PROCESSING_SCRIPT "${spirv-tools_SOURCE_DIR}/utils/generate_registry_tables.py")
 set(LANG_HEADER_PROCESSING_SCRIPT "${spirv-tools_SOURCE_DIR}/utils/generate_language_headers.py")
+set(GGT_SCRIPT "${spirv-tools_SOURCE_DIR}/utils/ggt.py")
 
 # Pull in grammar files that have migrated to SPIRV-Headers
+set(SPIRV_CORE_GRAMMAR_JSON_FILE "${SPIRV_HEADER_INCLUDE_DIR}/spirv/unified1/spirv.core.grammar.json")
 set(DEBUGINFO_GRAMMAR_JSON_FILE "${SPIRV_HEADER_INCLUDE_DIR}/spirv/unified1/extinst.debuginfo.grammar.json")
 set(CLDEBUGINFO100_GRAMMAR_JSON_FILE "${SPIRV_HEADER_INCLUDE_DIR}/spirv/unified1/extinst.opencl.debuginfo.100.grammar.json")
 set(VKDEBUGINFO100_GRAMMAR_JSON_FILE "${SPIRV_HEADER_INCLUDE_DIR}/spirv/unified1/extinst.nonsemantic.shader.debuginfo.100.grammar.json")
 
-# macro() definitions are used in the following because we need to append .inc
-# file paths into some global lists (*_CPP_DEPENDS). And those global lists are
-# later used by set_source_files_properties() calls.
-# function() definitions are not suitable because they create new scopes.
-macro(spvtools_core_tables CONFIG_VERSION)
-  set(GRAMMAR_JSON_FILE "${SPIRV_HEADER_INCLUDE_DIR}/spirv/${CONFIG_VERSION}/spirv.core.grammar.json")
-  set(GRAMMAR_INSTS_INC_FILE "${spirv-tools_BINARY_DIR}/core.insts-${CONFIG_VERSION}.inc")
-  set(GRAMMAR_KINDS_INC_FILE "${spirv-tools_BINARY_DIR}/operand.kinds-${CONFIG_VERSION}.inc")
-  add_custom_command(OUTPUT ${GRAMMAR_INSTS_INC_FILE} ${GRAMMAR_KINDS_INC_FILE}
-    COMMAND Python3::Interpreter ${GRAMMAR_PROCESSING_SCRIPT}
-      --spirv-core-grammar=${GRAMMAR_JSON_FILE}
+set(CORE_TABLES_INC_FILE ${spirv-tools_BINARY_DIR}/core_tables.inc)
+add_custom_command(OUTPUT ${CORE_TABLES_INC_FILE}
+    COMMAND Python3::Interpreter ${GGT_SCRIPT}
+      --spirv-core-grammar=${SPIRV_CORE_GRAMMAR_JSON_FILE}
       --extinst-debuginfo-grammar=${DEBUGINFO_GRAMMAR_JSON_FILE}
       --extinst-cldebuginfo100-grammar=${CLDEBUGINFO100_GRAMMAR_JSON_FILE}
-      --core-insts-output=${GRAMMAR_INSTS_INC_FILE}
-      --operand-kinds-output=${GRAMMAR_KINDS_INC_FILE}
-    DEPENDS ${GRAMMAR_PROCESSING_SCRIPT}
-            ${GRAMMAR_JSON_FILE}
+      --core-tables-output=${CORE_TABLES_INC_FILE}
+    DEPENDS ${GGT_SCRIPT}
+            ${SPIRV_CORE_GRAMMAR_JSON_FILE}
             ${DEBUGINFO_GRAMMAR_JSON_FILE}
             ${CLDEBUGINFO100_GRAMMAR_JSON_FILE}
-    COMMENT "Generate info tables for SPIR-V ${CONFIG_VERSION} core instructions and operands.")
-  list(APPEND OPCODE_CPP_DEPENDS ${GRAMMAR_INSTS_INC_FILE})
-  list(APPEND OPERAND_CPP_DEPENDS ${GRAMMAR_KINDS_INC_FILE})
-endmacro(spvtools_core_tables)
+    COMMENT "Generate core tables")
+add_custom_target(spirv-tools-tables DEPENDS ${CORE_TABLES_INC_FILE})
 
 macro(spvtools_enum_string_mapping CONFIG_VERSION)
   set(GRAMMAR_JSON_FILE "${SPIRV_HEADER_INCLUDE_DIR}/spirv/${CONFIG_VERSION}/spirv.core.grammar.json")
@@ -115,7 +107,6 @@ macro(spvtools_extinst_lang_headers NAME GRAMMAR_FILE)
   list(APPEND EXTINST_CPP_DEPENDS spirv-tools-header-${NAME})
 endmacro(spvtools_extinst_lang_headers)
 
-spvtools_core_tables("unified1")
 spvtools_enum_string_mapping("unified1")
 spvtools_vendor_tables("glsl.std.450" "glsl" "")
 spvtools_vendor_tables("opencl.std.100" "opencl" "")
@@ -153,7 +144,7 @@ list(APPEND OPCODE_CPP_DEPENDS ${GENERATOR_INC_FILE})
 # We need to wrap the .inc files with a custom target to avoid problems when
 # multiple targets depend on the same custom command.
 add_custom_target(core_tables
-  DEPENDS ${OPCODE_CPP_DEPENDS} ${OPERAND_CPP_DEPENDS})
+  DEPENDS ${OPCODE_CPP_DEPENDS} ${OPERAND_CPP_DEPENDS} ${CORE_TABLES_INC_FILE})
 add_custom_target(enum_string_mapping
   DEPENDS ${EXTENSION_H_DEPENDS} ${ENUM_STRING_MAPPING_CPP_DEPENDS})
 add_custom_target(extinst_tables
@@ -235,6 +226,7 @@ set(SPIRV_SOURCES
   ${CMAKE_CURRENT_SOURCE_DIR}/spirv_target_env.h
   ${CMAKE_CURRENT_SOURCE_DIR}/spirv_validator_options.h
   ${CMAKE_CURRENT_SOURCE_DIR}/table.h
+  ${CMAKE_CURRENT_SOURCE_DIR}/table2.h
   ${CMAKE_CURRENT_SOURCE_DIR}/text.h
   ${CMAKE_CURRENT_SOURCE_DIR}/text_handler.h
   ${CMAKE_CURRENT_SOURCE_DIR}/to_string.h
@@ -264,6 +256,7 @@ set(SPIRV_SOURCES
   ${CMAKE_CURRENT_SOURCE_DIR}/spirv_target_env.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/spirv_validator_options.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/table.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/table2.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/text.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/text_handler.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/to_string.cpp