Merge branch 'feature/espcoredump_py_riscv_support_v4.3' into 'releas…

…e/v4.3'

feature: espcoredump py riscv support (v4.3)

See merge request espressif/esp-idf!16840

.gitlab-ci.yml

@@ -70,7 +70,7 @@ variables:
   export IDF_MIRROR_PREFIX_MAP=
   fi
   if [[ "$SETUP_TOOLS" == "1" || "$CI_JOB_STAGE" != "target_test" ]]; then
-  tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)" || exit 1
+    tools/idf_tools.py --non-interactive install ${SETUP_TOOLS_LIST:-} && eval "$(tools/idf_tools.py --non-interactive export)" || exit 1
   fi
 
 before_script:

components/espcoredump/corefile/__init__.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2021 Espressif Systems (Shanghai) PTE LTD
+# Copyright 2021 Espressif Systems (Shanghai) CO., LTD
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,7 +16,22 @@
 
 __version__ = '0.4-dev'
 
+import abc
+import os
 from abc import abstractmethod
+from importlib import import_module
+
+from future.utils import with_metaclass
+
+try:
+    from typing import Optional, Tuple
+except ImportError:
+    pass
+
+IDF_PATH = os.path.normpath(os.getenv('IDF_PATH', '.'))
+XTENSA_TARGETS = ['esp32', 'esp32s2']
+RISCV_TARGETS = ['esp32c3']
+SUPPORTED_TARGETS = XTENSA_TARGETS + RISCV_TARGETS
 
 
 class ESPCoreDumpError(RuntimeError):
@@ -27,42 +42,84 @@ class ESPCoreDumpLoaderError(ESPCoreDumpError):
     pass
 
 
-class _TargetMethodsBase(object):
+class BaseArchMethodsMixin(with_metaclass(abc.ABCMeta)):  # type: ignore
     @staticmethod
     @abstractmethod
-    def tcb_is_sane(tcb_addr, tcb_size):
+    def get_registers_from_stack(data, grows_down):
+        # type: (bytes, bool) -> Tuple[list[int], Optional[dict[int, int]]]
         """
-        Check tcb address if it is correct
+        Parse stack data, growing up stacks are not supported for now.
+        :param data: stack data
+        :param grows_down: stack grow direction
+        :return: return tuple (regs, exception_regs)
         """
-        return False
+        pass
 
     @staticmethod
     @abstractmethod
-    def stack_is_sane(sp):
+    def build_prstatus_data(tcb_addr, task_regs):  # type: (int, list[int]) -> str
         """
-        Check stack address if it is correct
+        Build PrStatus note section
+        :param tcb_addr: tcb addr
+        :param task_regs: registers
+        :return: str
         """
-        return False
+        pass
 
-    @staticmethod
-    @abstractmethod
-    def addr_is_fake(addr):
-        """
-        Check if address is in fake area
-        """
+
+class BaseTargetMethods(with_metaclass(abc.ABCMeta, BaseArchMethodsMixin)):  # type: ignore
+    UNKNOWN = 'unknown'
+    TARGET = UNKNOWN
+
+    COREDUMP_FAKE_STACK_START = 0x20000000
+    COREDUMP_FAKE_STACK_LIMIT = 0x30000000
+    COREDUMP_MAX_TASK_STACK_SIZE = 64 * 1024
+
+    def __init__(self):  # type: () -> None
+        if self.TARGET == self.UNKNOWN:
+            raise ValueError('Please use the derived child-class with valid TARGET')
+
+        self._set_attr_from_soc_header()
+
+    def _set_attr_from_soc_header(self):  # type: () -> None
+        module = import_module('corefile.soc_headers.{}'.format(self.TARGET))
+        for k, v in module.__dict__.items():
+            if k.startswith('SOC_'):
+                setattr(self, k, v)
+
+    def _esp_ptr_in_dram(self, addr):  # type: (int) -> bool
+        return self.SOC_DRAM_LOW <= addr < self.SOC_DRAM_HIGH  # type: ignore
+
+    def _esp_ptr_in_iram(self, addr):  # type: (int) -> bool
+        return self.SOC_IRAM_LOW <= addr < self.SOC_IRAM_HIGH  # type: ignore
+
+    def _esp_ptr_in_rtc_slow(self, addr):  # type: (int) -> bool
+        return self.SOC_RTC_DATA_LOW <= addr < self.SOC_RTC_DATA_HIGH  # type: ignore
+
+    def _esp_ptr_in_rtc_dram_fast(self, addr):  # type: (int) -> bool
+        return self.SOC_RTC_DRAM_LOW <= addr < self.SOC_RTC_DRAM_HIGH  # type: ignore
+
+    def tcb_is_sane(self, tcb_addr, tcb_size):  # type: (int, int) -> bool
+        for func in [self._esp_ptr_in_dram,
+                     self._esp_ptr_in_iram,
+                     self._esp_ptr_in_rtc_slow,
+                     self._esp_ptr_in_rtc_dram_fast]:
+            res = func(tcb_addr) and func(tcb_addr + tcb_size - 1)
+            if res:
+                return True
         return False
 
+    def _esp_stack_ptr_in_dram(self, addr):  # type: (int) -> bool
+        return not (addr < self.SOC_DRAM_LOW + 0x10
+                    or addr > self.SOC_DRAM_HIGH - 0x10
+                    or (addr & 0xF) != 0)
 
-class _ArchMethodsBase(object):
-    @staticmethod
-    @abstractmethod
-    def get_registers_from_stack(data, grows_down):
-        """
-        Returns list of registers (in GDB format) from stack frame
-        """
-        return [], {}
+    def stack_is_sane(self, stack_start, stack_end):  # type: (int, int) -> bool
+        return (self._esp_stack_ptr_in_dram(stack_start)
+                and self._esp_ptr_in_dram(stack_end)
+                and stack_start < stack_end
+                and (stack_end - stack_start) < self.COREDUMP_MAX_TASK_STACK_SIZE)
 
-    @staticmethod
-    @abstractmethod
-    def build_prstatus_data(tcb_addr, task_regs):
-        return b''
+    def addr_is_fake(self, addr):  # type: (int) -> bool
+        return (self.COREDUMP_FAKE_STACK_START <= addr < self.COREDUMP_FAKE_STACK_LIMIT
+                or addr > 2 ** 31 - 1)

components/espcoredump/corefile/_parse_soc_header.py

@@ -0,0 +1,44 @@
+"""
+This file is used to generate soc header constants into sub-package soc_headers
+"""
+import os
+from ast import literal_eval
+
+from . import IDF_PATH, SUPPORTED_TARGETS
+
+
+def main():  # type: () -> None
+    constants = [
+        'SOC_DRAM_LOW',
+        'SOC_DRAM_HIGH',
+        'SOC_IRAM_LOW',
+        'SOC_IRAM_HIGH',
+        'SOC_RTC_DATA_LOW',
+        'SOC_RTC_DATA_HIGH',
+        'SOC_RTC_DRAM_LOW',
+        'SOC_RTC_DRAM_HIGH',
+    ]
+
+    for target in SUPPORTED_TARGETS:
+        target_constants = {}
+        soc_header_fp = os.path.join(IDF_PATH, 'components/soc/{}/include/soc/soc.h'.format(target))
+        module_fp = os.path.join(IDF_PATH, 'components', 'espcoredump', 'corefile', 'soc_headers',
+                                 '{}.py'.format(target))
+
+        with open(soc_header_fp) as fr:
+            for line in fr.readlines():
+                for attr in constants:
+                    if '#define {}'.format(attr) in line:
+                        target_constants[attr] = literal_eval(line.strip().split()[-1])
+
+        for attr in constants:
+            if attr not in target_constants:
+                raise ValueError('ERROR: Attr {} is missing in {}'.format(attr, soc_header_fp))
+
+        with open(module_fp, 'w') as fw:
+            for k, v in target_constants.items():
+                fw.write('{} = {}\n'.format(k, hex(v)))
+
+
+if __name__ == '__main__':
+    main()

components/espcoredump/corefile/elf.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2021 Espressif Systems (Shanghai) PTE LTD
+# Copyright 2021 Espressif Systems (Shanghai) CO., LTD
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,8 +17,13 @@
 import hashlib
 import os
 
-from construct import (AlignedStruct, Bytes, Const, GreedyRange, Int16ul, Int32ul, Padding, Pointer, Sequence, Struct,
-                       this)
+from construct import (AlignedStruct, Bytes, Const, Container, GreedyRange, Int16ul, Int32ul, Padding, Pointer,
+                       Sequence, Struct, this)
+
+try:
+    from typing import Optional
+except ImportError:
+    pass
 
 # Following structs are based on spec
 # https://refspecs.linuxfoundation.org/elf/elf.pdf
@@ -110,12 +115,13 @@ class ElfFile(object):
     EV_CURRENT = 0x01
 
     def __init__(self, elf_path=None, e_type=None, e_machine=None):
+        # type: (Optional[str], Optional[int], Optional[int]) -> None
         self.e_type = e_type
         self.e_machine = e_machine
 
-        self._struct = None  # construct Struct
-        self._model = None  # construct Container
-        self._section_names = []  # type: list[str]
+        self._struct = None  # type: Optional[Struct]
+        self._model = None  # type: Optional[Container]
+        self._section_names = {}  # type: dict[int, str]
 
         self.sections = []  # type: list[ElfSection]
         self.load_segments = []  # type: list[ElfSegment]
@@ -171,7 +177,7 @@ def _parse_string_table(byte_str):  # type: (bytes) -> dict
                 name += c
         return res
 
-    def _generate_struct_from_headers(self, header_tables):
+    def _generate_struct_from_headers(self, header_tables):  # type: (Container) -> Struct
         """
         Generate ``construct`` Struct for this file
         :param header_tables: contains elf_header, program_headers, section_headers
@@ -219,12 +225,12 @@ def _generate_struct_from_headers(self, header_tables):
         return Struct(*args)
 
     @property
-    def sha256(self):
+    def sha256(self):  # type: () -> bytes
         """
         :return: SHA256 hash of the input ELF file
         """
         sha256 = hashlib.sha256()
-        sha256.update(self._struct.build(self._model))
+        sha256.update(self._struct.build(self._model))  # type: ignore
         return sha256.digest()
 
 
@@ -234,13 +240,13 @@ class ElfSection(object):
     SHF_EXECINSTR = 0x04
     SHF_MASKPROC = 0xf0000000
 
-    def __init__(self, name, addr, data, flags):
+    def __init__(self, name, addr, data, flags):  # type: (str, int, bytes, int) -> None
         self.name = name
         self.addr = addr
         self.data = data
         self.flags = flags
 
-    def attr_str(self):
+    def attr_str(self):  # type: () -> str
         if self.flags & self.SHF_MASKPROC:
             return 'MS'
 
@@ -250,7 +256,7 @@ def attr_str(self):
         res += 'A' if self.flags & self.SHF_ALLOC else ' '
         return res
 
-    def __repr__(self):
+    def __repr__(self):  # type: () -> str
         return '{:>32} [Addr] 0x{:>08X}, [Size] 0x{:>08X} {:>4}' \
             .format(self.name, self.addr, len(self.data), self.attr_str())
 
@@ -260,36 +266,36 @@ class ElfSegment(object):
     PF_W = 0x02
     PF_R = 0x04
 
-    def __init__(self, addr, data, flags):
+    def __init__(self, addr, data, flags):  # type: (int, bytes, int) -> None
         self.addr = addr
         self.data = data
         self.flags = flags
         self.type = ElfFile.PT_LOAD
 
-    def attr_str(self):
+    def attr_str(self):  # type: () -> str
         res = ''
         res += 'R' if self.flags & self.PF_R else ' '
         res += 'W' if self.flags & self.PF_W else ' '
         res += 'E' if self.flags & self.PF_X else ' '
         return res
 
     @staticmethod
-    def _type_str():
+    def _type_str():  # type: () -> str
         return 'LOAD'
 
-    def __repr__(self):
+    def __repr__(self):  # type: () -> str
         return '{:>8} Addr 0x{:>08X}, Size 0x{:>08X} Flags {:4}' \
             .format(self._type_str(), self.addr, len(self.data), self.attr_str())
 
 
 class ElfNoteSegment(ElfSegment):
-    def __init__(self, addr, data, flags):
+    def __init__(self, addr, data, flags):  # type: (int, bytes, int) -> None
         super(ElfNoteSegment, self).__init__(addr, data, flags)
         self.type = ElfFile.PT_NOTE
         self.note_secs = NoteSections.parse(self.data)
 
     @staticmethod
-    def _type_str():
+    def _type_str():  # type: () -> str
         return 'NOTE'
 
 
@@ -316,13 +322,15 @@ class ESPCoreDumpElfFile(ElfFile):
 
     # ELF file machine type
     EM_XTENSA = 0x5E
+    EM_RISCV = 0xF3
 
     def __init__(self, elf_path=None, e_type=None, e_machine=None):
+        # type: (Optional[str], Optional[int], Optional[int]) -> None
         _e_type = e_type or self.ET_CORE
         _e_machine = e_machine or self.EM_XTENSA
         super(ESPCoreDumpElfFile, self).__init__(elf_path, _e_type, _e_machine)
 
-    def add_segment(self, addr, data, seg_type, flags):
+    def add_segment(self, addr, data, seg_type, flags):  # type: (int, bytes, int, int) -> None
         if seg_type != self.PT_NOTE:
             self.load_segments.append(ElfSegment(addr, data, flags))
         else:
@@ -352,7 +360,7 @@ def dump(self, output_path):  # type: (str) -> None
         })
 
         offset = ElfHeader.sizeof() + (len(self.load_segments) + len(self.note_segments)) * ProgramHeader.sizeof()
-        _segments = self.load_segments + self.note_segments
+        _segments = self.load_segments + self.note_segments  # type: ignore
         for seg in _segments:
             res += ProgramHeader.build({
                 'p_type': seg.type,