From 35e1dcd00220734c446f6f2945f913b9cc76eb3c Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Fri, 1 May 2026 22:52:25 +0200 Subject: [PATCH 01/95] remove binja bindings no longer maintained by the person that created them --- ida/ffxiv_idarename.py | 106 ----------------- ida/ffxiv_structimporter.py | 222 ------------------------------------ 2 files changed, 328 deletions(-) diff --git a/ida/ffxiv_idarename.py b/ida/ffxiv_idarename.py index 076ca9d0b4..166a61c938 100644 --- a/ida/ffxiv_idarename.py +++ b/ida/ffxiv_idarename.py @@ -416,112 +416,6 @@ def format_func_name(self, ea, current_func_name, proposed_func_name, class_name api = GhidraApi() -# endregion -# region Binja Api - -if api is None: - try: - import binaryninja # noqa - except ImportError: - print("Warning: Unable to load Binary Ninja") - else: - # noinspection PyUnresolvedReferences - class BinjaApi(BaseApi): - @property - def data_file_path(self): - return os.path.join(os.path.dirname(__file__), "data.yml") - - def get_image_base(self): - return bv.start - - def is_offset(self, ea): - return bv.get_data_var_at(ea) is not None - - def xrefs_to(self, ea): - return [xref.address for xref in bv.get_code_refs(ea)] - - def get_qword(self, ea): - bytes = bv.read(ea, 8) - return int.from_bytes(bytes, byteorder='little') - - def get_addr_name(self, ea): - func = bv.get_function_at(ea) - if func: - return func.name - - sym = bv.get_symbol_at(ea) - if sym: - return sym.name - - return "" - - def set_addr_name(self, ea, name): - func = bv.get_function_at(ea) - if func: - func.name = name - return True - - sym = bv.get_symbol_at(ea) - if sym: - # No good way to clone a symbol (afaik), so remake it - new_sym = binaryninja.types.Symbol( - sym.type, - ea, - name, - sym.full_name, - sym.raw_name, - sym.binding, - sym.namespace, - sym.ordinal - ) - - if sym.auto: - bv.undefine_auto_symbol(sym) - else: - bv.undefine_user_symbol(sym) - bv.define_user_symbol(new_sym) - return True - - data_var = bv.get_data_var_at(ea) - if data_var: - new_sym = binaryninja.types.Symbol( - binaryninja.types.SymbolType.DataSymbol, - ea, - name - ) - bv.define_user_symbol(new_sym) - return True - - return False - - def get_comment(self, ea): - return bv.get_comment_at(ea) - - def set_comment(self, ea, comment): - bv.set_comment_at(ea, comment) - - def format_vfunc_name(self, ea, current_func_name, proposed_func_name, class_name, base_class_names): - # Previously renamed as a vfunc - if current_func_name.startswith(class_name): - # return the proposed func name in case it was updated since last run - current_class_name = current_func_name.rsplit(".", 1)[0] - return "{0}.{1}".format(current_class_name, proposed_func_name) - - # This should have been handled in the base class - if any(current_func_name.startswith(name) for name in base_class_names): - return "" - - return "{0}.{1}".format(class_name, proposed_func_name) - - def format_func_name(self, ea, current_func_name, proposed_func_name, class_name): - proposed_qualified_func_name = "{0}.{1}".format(class_name, proposed_func_name) - if current_func_name == proposed_qualified_func_name: - return "" - return proposed_qualified_func_name - - - api = BinjaApi() - # endregion if api is None: diff --git a/ida/ffxiv_structimporter.py b/ida/ffxiv_structimporter.py index 10d4943eef..086c2abfb5 100644 --- a/ida/ffxiv_structimporter.py +++ b/ida/ffxiv_structimporter.py @@ -2070,228 +2070,6 @@ def preprocess_yaml(self, yaml: DefinedStructExport): api = GhidraApi() -if api is None: - try: - import binaryninja - import struct - except ImportError: - print("Warning: Unable to load Binary Ninja") - else: - # TODO: VTables, Unions - class BinjaApi(BaseApi): - def can_run(self): - return True - - def get_binja_type(self, name): - # type: (str) -> str - lookup = { - "__int8": "int8_t", - "__int16": "int16_t", - "__int32": "int32_t", - "__int64": "int64_t", - "unsigned __int8": "uint8_t", - "unsigned __int16": "uint16_t", - "unsigned __int32": "uint32_t", - "unsigned __int64": "uint64_t", - "unsigned int": "uint32_t", - "_DWORD": "uint32_t", - "float": "float", - } - if name in lookup: - return lookup[name] - else: - return name - - def get_type(self, name): - # type: (str) -> binaryninja.Type - if name == "__fastcall": - return None - - fixed_name = self.get_binja_type(name) - - pointer_count = 0 - while fixed_name.endswith("*"): - fixed_name = fixed_name[:-1] - pointer_count += 1 - - type = None - - try: - type = bv.parse_type_string(fixed_name)[0] - except: - # Sometimes it just throws. Dunno why - type = bv.types[fixed_name] - - if pointer_count > 0: - type = binaryninja.Type.pointer(type=type, arch=bv.arch) - - return type - - def get_size_from_type(self, name): - # type: (str) -> int - type_obj = self.get_type(name) - if type_obj is None: - return 4 - return type_obj.width - - @property - def get_file_path(self): - # type: () -> str - return os.path.join(os.path.dirname(__file__), "ffxiv_structs.yml") - - def create_enum_struct(self, enum): - # type: (DefinedStructEnum) -> None - members = [] - for value in enum.values: - members.append((value, enum.values[value])) - enum_type = binaryninja.Type.enumeration( - members=members, width=self.get_size_from_type(enum.underlying) - ) - bv.define_user_type(enum.type, enum_type) - - def create_struct(self, struct): - # type: (DefinedStruct) -> None - struct_type = binaryninja.Type.structure( - type=binaryninja.StructureVariant.ClassStructureType - ) - bv.define_user_type(struct.type, struct_type) - - def create_struct_members(self, struct): - # type: (DefinedStruct) -> None - struct_type = bv.types[ - struct.type - ].mutable_copy() # type: binaryninja.StructureBuilder - - for field in struct.fields: - field_type = self.get_type(field.type) - if field_type is None: - continue - - if isinstance(field, DefinedStructFixedField): - field_type = binaryninja.Type.array(field_type, int(field.size)) - struct_type.add_member_at_offset( - field.name, - field_type, - int(field.offset), - ) - - bv.define_user_type(struct.type, struct_type) - - def get_func_ea_by_sig(self, pattern): - # type: (str) -> int - regex = "" - for part in pattern.split(" "): - if part == "??": - regex = regex + "." - else: - regex = regex + "\\x" + part - compiled = re.compile(regex.encode("utf-8")) - - for segment in bv.segments: - data = bv.read(segment.start, segment.end - segment.start) - match = compiled.search(data) - if match: - match_start = match.start() - addr = segment.start + match_start - if data[match_start] == 0xE8 or data[match_start] == 0xE9: - addr += 5 - addr += struct.unpack( - " None - func_name = "{0}.{1}".format(struct.type, member_func.name) - - func = None - symbol = bv.get_symbol_by_raw_name(func_name) - if symbol: - func = bv.get_function_at(symbol.address) - - if not func: - func_addr = self.get_func_ea_by_sig(member_func.signature) - if func_addr is not None: - func = bv.get_function_at(func_addr) - - if not func: - return - - if member_func.return_type != "void": - new_return_type = self.get_type(member_func.return_type) - if new_return_type is not None: - func.return_type = new_return_type - - for i, param in enumerate(member_func.parameters): - if i < len(func.parameter_vars): - param_var = func.parameter_vars[i] - if param_var is None: - continue - - new_param_type = self.get_type(param.type) - if new_param_type is not None: - param_var.type = new_param_type - param_var.name = param.name - - def update_virt_func(self, virt_func, struct): - # type: (DefinedStructVFunc, DefinedStruct) -> None - func_name = "{0}.{1}".format(struct.type, virt_func.name) - - func = None - symbol = bv.get_symbol_by_raw_name(func_name) - if symbol: - func = bv.get_function_at(symbol.address) - - if not func: - return - - if virt_func.return_type != "void": - new_return_type = self.get_type(virt_func.return_type) - if new_return_type is not None: - func.return_type = new_return_type - - for i, param in enumerate(virt_func.parameters): - if i < len(func.parameter_vars): - param_var = func.parameter_vars[i] - if param_var is None: - continue - - new_param_type = self.get_type(param.type) - if new_param_type is not None: - param_var.type = new_param_type - param_var.name = param.name - - def update_static_member(self, static_member, struct): - # type: (DefinedStructStaticMember, DefinedStruct) -> None - pass - - def should_update_member_func(self): - # type: () -> bool - return ( - binaryninja.get_choice_input( - "Update member function types?", - "ffxiv_structimporter", - ["Yes", "No"], - ) - == 0 - ) - - def should_update_virt_func(self): - # type: () -> bool - return ( - binaryninja.get_choice_input( - "Update virtual function types?", - "ffxiv_structimporter", - ["Yes", "No"], - ) - == 0 - ) - - def preprocess_yaml(self, yaml: DefinedStructExport): - return - - api = BinjaApi() - if api is None: raise Exception("Unable to load API (supported: IDA, Ghidra, Binary Ninja)") From c513d21dbb3a87dc524c82480fd023afdc7021de Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Sat, 2 May 2026 12:28:40 +0200 Subject: [PATCH 02/95] rename top folder from ida to scripts - Also converts structs_schema.py to dataclass types --- CExporter/Program.cs | 6 +- {ida => scripts}/README.md | 0 {ida => scripts}/classinformer.csv | 0 {ida => scripts}/classinformer_lysander.csv | 0 {ida => scripts}/data-validator.js | 0 {ida => scripts}/data.yml | 0 {ida => scripts}/data_schema.py | 0 .../__init__.py => scripts/errors.txt | 0 {ida => scripts}/ffxiv_addonmagic.py | 0 {ida => scripts}/ffxiv_data_struct_matcher.py | 0 {ida => scripts}/ffxiv_exdgetters.py | 0 {ida => scripts}/ffxiv_extract_uld_names.py | 0 {ida => scripts}/ffxiv_fullrun_ida.py | 0 {ida => scripts}/ffxiv_idarename.py | 0 {ida => scripts}/ffxiv_luafuncs_ida.py | 0 {ida => scripts}/ffxiv_sigmaker.py | 0 {ida => scripts}/ffxiv_structimporter.py | 20 ++ {ida => scripts}/ffxiv_structs.yml | 0 {ida => scripts}/ida_wrapper.py | 261 ++++++++++++++++-- {ida => scripts}/idauser.cfg | 0 scripts/luminapie/__init__.py | 0 {ida => scripts}/luminapie/definitions.py | 0 {ida => scripts}/luminapie/enums.py | 0 {ida => scripts}/luminapie/excel.py | 0 {ida => scripts}/luminapie/exdschema.py | 0 {ida => scripts}/luminapie/file_handlers.py | 0 {ida => scripts}/luminapie/game_data.py | 0 {ida => scripts}/luminapie/se_crc.py | 0 {ida => scripts}/luminapie/sqpack.py | 0 .../old/data_2020.12.29.0000.0000.yml | 0 .../old/data_2021.01.28.0000.0000.yml | 0 .../old/data_2021.02.10.0000.0000.yml | 0 .../old/data_2021.04.07.0000.0000.yml | 0 .../old/data_2021.04.21.0000.0000.yml | 0 .../old/data_2021.05.18.0000.0000.yml | 0 .../old/data_2021.05.29.0000.0000.yml | 0 .../old/data_2021.06.16.0000.0000.yml | 0 .../old/data_2021.07.15.0000.0000.yml | 0 .../old/data_2021.08.17.0000.0000.yml | 0 .../old/data_2021.11.28.0000.0000.yml | 0 .../old/data_2021.12.16.0000.0000.yml | 0 .../old/data_2021.12.24.0000.0000.yml | 0 .../old/data_2022.01.25.0000.0000.yml | 0 .../old/data_2022.03.01.0000.0000.yml | 0 .../old/data_2022.04.07.0000.0000.yml | 0 .../old/data_2022.04.20.0000.0000.yml | 0 .../old/data_2022.05.19.0000.0000.yml | 0 .../old/data_2022.05.27.0000.0000.yml | 0 .../old/data_2022.06.21.0000.0000.yml | 0 .../old/data_2022.07.08.0000.0000.yml | 0 .../old/data_2022.08.17.0000.0000.yml | 0 .../old/data_2022.08.25.0000.0000.yml | 0 .../old/data_2022.09.07.0000.0000.yml | 0 .../old/data_2022.10.05.0000.0000.yml | 0 .../old/data_2022.10.26.0000.0000.yml | 0 .../old/data_2022.11.09.0000.0000.yml | 0 .../old/data_2022.12.23.0000.0000.yml | 0 .../old/data_2023.01.11.0000.0000.yml | 0 .../old/data_2023.01.17.0000.0000.yml | 0 .../old/data_2023.02.03.0000.0000.yml | 0 .../old/data_2023.02.28.0000.0000.yml | 0 .../old/data_2023.03.24.0000.0000.yml | 0 .../old/data_2023.05.18.0000.0000.yml | 0 .../old/data_2023.05.31.0000.0000.yml | 0 .../old/data_2023.06.14.0000.0000.yml | 0 .../old/data_2023.07.06.0000.0000.yml | 0 .../old/data_2023.07.26.0000.0000.yml | 0 .../old/data_2023.09.28.0000.0000.yml | 0 .../old/data_2023.10.23.0000.0000.yml | 0 .../old/data_2023.11.09.0000.0000.yml | 0 .../old/data_2024.01.06.0000.0000.yml | 0 .../old/data_2024.02.05.0000.0000.yml | 0 .../old/data_2024.03.08.0000.0000.yml | 0 .../old/data_2024.03.27.0000.0000.yml | 0 .../old/data_2024.04.23.0000.0000.yml | 0 .../old/data_2024.06.18.0000.0000.yml | 0 .../old/data_2024.07.06.0000.0000.yml | 0 .../old/data_2024.07.10.0001.0000.yml | 0 .../old/data_2024.07.24.0000.0000.yml | 0 .../old/data_2024.08.02.0000.0000.yml | 0 .../old/data_2024.08.21.0000.0000.yml | 0 .../old/data_2024.11.06.0000.0000.yml | 0 .../old/data_2024.11.20.0000.0000.yml | 0 .../old/data_2024.12.07.0000.0000.yml | 0 .../old/data_2025.01.14.0000.0000.yml | 0 .../old/data_2025.01.28.0000.0000.yml | 0 .../old/data_2025.02.19.0000.0000.yml | 0 .../old/data_2025.02.27.0000.0000.yml | 0 .../old/data_2025.03.18.0000.0000.yml | 0 .../old/data_2025.03.27.0000.0000.yml | 0 .../old/data_2025.04.16.0000.0000.yml | 0 .../old/data_2025.05.17.0000.0000.yml | 0 .../old/data_2025.06.10.0000.0000.yml | 0 .../old/data_2025.06.19.0000.0000.yml | 0 .../old/data_2025.06.28.0000.0000.yml | 0 .../old/data_2025.07.30.0000.0000.yml | 0 .../old/data_2025.08.07.0000.0000.yml | 0 .../old/data_2025.08.22.0000.0000.yml | 0 .../old/data_2025.09.04.0000.0000.yml | 0 .../old/data_2025.09.30.0000.0000.yml | 0 .../old/data_2025.10.13.0000.0000.yml | 0 .../old/data_2025.10.30.0000.0000.yml | 0 .../old/data_2025.12.09.0000.0000.yml | 0 .../old/data_2025.12.18.0000.0000.yml | 0 .../old/data_2025.12.23.0000.0000.yml | 0 .../old/data_2026.01.21.0000.0000.yml | 0 .../old/data_2026.01.30.0000.0000.yml | 0 .../old/data_2026.02.20.0000.0000.yml | 0 .../old/data_2026.03.07.0000.0000.yml | 0 .../old/data_2026.03.17.0000.0000.yml | 0 .../old/data_2026.04.21.0000.0000.yml | 0 .../old/data_2026.05.01.0000.0000.yml | 0 .../old/data_2026.05.25.0000.0000.yml | 0 .../old/data_2026.06.10.0000.0000.yml | 0 .../old/data_2026.06.18.0000.0000.yml | 0 .../old/data_2026.07.16.0001.0000.yml | 0 .../old/data_2026.08.05.0000.0000.yml | 0 .../old/data_2026.08.11.0000.0000.yml | 0 .../old/data_2026.09.01.0000.0000.yml | 0 .../old/data_2026.09.15.0000.0000.yml | 0 .../old/data_2026.10.06.0000.0000.yml | 0 .../ffxiv_idarename_2020.10.21.0001.0000.py | 0 .../ffxiv_idarename_2020.12.02.0000.0000.py | 0 .../ffxiv_idarename_2020.12.15.0000.0000.py | 0 {ida => scripts}/poetry.lock | 0 {ida => scripts}/pyproject.toml | 0 scripts/structs_schema.py | 81 ++++++ 127 files changed, 341 insertions(+), 27 deletions(-) rename {ida => scripts}/README.md (100%) rename {ida => scripts}/classinformer.csv (100%) rename {ida => scripts}/classinformer_lysander.csv (100%) rename {ida => scripts}/data-validator.js (100%) rename {ida => scripts}/data.yml (100%) rename {ida => scripts}/data_schema.py (100%) rename ida/luminapie/__init__.py => scripts/errors.txt (100%) rename {ida => scripts}/ffxiv_addonmagic.py (100%) rename {ida => scripts}/ffxiv_data_struct_matcher.py (100%) rename {ida => scripts}/ffxiv_exdgetters.py (100%) rename {ida => scripts}/ffxiv_extract_uld_names.py (100%) rename {ida => scripts}/ffxiv_fullrun_ida.py (100%) rename {ida => scripts}/ffxiv_idarename.py (100%) rename {ida => scripts}/ffxiv_luafuncs_ida.py (100%) rename {ida => scripts}/ffxiv_sigmaker.py (100%) rename {ida => scripts}/ffxiv_structimporter.py (99%) rename {ida => scripts}/ffxiv_structs.yml (100%) rename {ida => scripts}/ida_wrapper.py (82%) rename {ida => scripts}/idauser.cfg (100%) create mode 100644 scripts/luminapie/__init__.py rename {ida => scripts}/luminapie/definitions.py (100%) rename {ida => scripts}/luminapie/enums.py (100%) rename {ida => scripts}/luminapie/excel.py (100%) rename {ida => scripts}/luminapie/exdschema.py (100%) rename {ida => scripts}/luminapie/file_handlers.py (100%) rename {ida => scripts}/luminapie/game_data.py (100%) rename {ida => scripts}/luminapie/se_crc.py (100%) rename {ida => scripts}/luminapie/sqpack.py (100%) rename {ida => scripts}/old/data_2020.12.29.0000.0000.yml (100%) rename {ida => scripts}/old/data_2021.01.28.0000.0000.yml (100%) rename {ida => scripts}/old/data_2021.02.10.0000.0000.yml (100%) rename {ida => scripts}/old/data_2021.04.07.0000.0000.yml (100%) rename {ida => scripts}/old/data_2021.04.21.0000.0000.yml (100%) rename {ida => scripts}/old/data_2021.05.18.0000.0000.yml (100%) rename {ida => scripts}/old/data_2021.05.29.0000.0000.yml (100%) rename {ida => scripts}/old/data_2021.06.16.0000.0000.yml (100%) rename {ida => scripts}/old/data_2021.07.15.0000.0000.yml (100%) rename {ida => scripts}/old/data_2021.08.17.0000.0000.yml (100%) rename {ida => scripts}/old/data_2021.11.28.0000.0000.yml (100%) rename {ida => scripts}/old/data_2021.12.16.0000.0000.yml (100%) rename {ida => scripts}/old/data_2021.12.24.0000.0000.yml (100%) rename {ida => scripts}/old/data_2022.01.25.0000.0000.yml (100%) rename {ida => scripts}/old/data_2022.03.01.0000.0000.yml (100%) rename {ida => scripts}/old/data_2022.04.07.0000.0000.yml (100%) rename {ida => scripts}/old/data_2022.04.20.0000.0000.yml (100%) rename {ida => scripts}/old/data_2022.05.19.0000.0000.yml (100%) rename {ida => scripts}/old/data_2022.05.27.0000.0000.yml (100%) rename {ida => scripts}/old/data_2022.06.21.0000.0000.yml (100%) rename {ida => scripts}/old/data_2022.07.08.0000.0000.yml (100%) rename {ida => scripts}/old/data_2022.08.17.0000.0000.yml (100%) rename {ida => scripts}/old/data_2022.08.25.0000.0000.yml (100%) rename {ida => scripts}/old/data_2022.09.07.0000.0000.yml (100%) rename {ida => scripts}/old/data_2022.10.05.0000.0000.yml (100%) rename {ida => scripts}/old/data_2022.10.26.0000.0000.yml (100%) rename {ida => scripts}/old/data_2022.11.09.0000.0000.yml (100%) rename {ida => scripts}/old/data_2022.12.23.0000.0000.yml (100%) rename {ida => scripts}/old/data_2023.01.11.0000.0000.yml (100%) rename {ida => scripts}/old/data_2023.01.17.0000.0000.yml (100%) rename {ida => scripts}/old/data_2023.02.03.0000.0000.yml (100%) rename {ida => scripts}/old/data_2023.02.28.0000.0000.yml (100%) rename {ida => scripts}/old/data_2023.03.24.0000.0000.yml (100%) rename {ida => scripts}/old/data_2023.05.18.0000.0000.yml (100%) rename {ida => scripts}/old/data_2023.05.31.0000.0000.yml (100%) rename {ida => scripts}/old/data_2023.06.14.0000.0000.yml (100%) rename {ida => scripts}/old/data_2023.07.06.0000.0000.yml (100%) rename {ida => scripts}/old/data_2023.07.26.0000.0000.yml (100%) rename {ida => scripts}/old/data_2023.09.28.0000.0000.yml (100%) rename {ida => scripts}/old/data_2023.10.23.0000.0000.yml (100%) rename {ida => scripts}/old/data_2023.11.09.0000.0000.yml (100%) rename {ida => scripts}/old/data_2024.01.06.0000.0000.yml (100%) rename {ida => scripts}/old/data_2024.02.05.0000.0000.yml (100%) rename {ida => scripts}/old/data_2024.03.08.0000.0000.yml (100%) rename {ida => scripts}/old/data_2024.03.27.0000.0000.yml (100%) rename {ida => scripts}/old/data_2024.04.23.0000.0000.yml (100%) rename {ida => scripts}/old/data_2024.06.18.0000.0000.yml (100%) rename {ida => scripts}/old/data_2024.07.06.0000.0000.yml (100%) rename {ida => scripts}/old/data_2024.07.10.0001.0000.yml (100%) rename {ida => scripts}/old/data_2024.07.24.0000.0000.yml (100%) rename {ida => scripts}/old/data_2024.08.02.0000.0000.yml (100%) rename {ida => scripts}/old/data_2024.08.21.0000.0000.yml (100%) rename {ida => scripts}/old/data_2024.11.06.0000.0000.yml (100%) rename {ida => scripts}/old/data_2024.11.20.0000.0000.yml (100%) rename {ida => scripts}/old/data_2024.12.07.0000.0000.yml (100%) rename {ida => scripts}/old/data_2025.01.14.0000.0000.yml (100%) rename {ida => scripts}/old/data_2025.01.28.0000.0000.yml (100%) rename {ida => scripts}/old/data_2025.02.19.0000.0000.yml (100%) rename {ida => scripts}/old/data_2025.02.27.0000.0000.yml (100%) rename {ida => scripts}/old/data_2025.03.18.0000.0000.yml (100%) rename {ida => scripts}/old/data_2025.03.27.0000.0000.yml (100%) rename {ida => scripts}/old/data_2025.04.16.0000.0000.yml (100%) rename {ida => scripts}/old/data_2025.05.17.0000.0000.yml (100%) rename {ida => scripts}/old/data_2025.06.10.0000.0000.yml (100%) rename {ida => scripts}/old/data_2025.06.19.0000.0000.yml (100%) rename {ida => scripts}/old/data_2025.06.28.0000.0000.yml (100%) rename {ida => scripts}/old/data_2025.07.30.0000.0000.yml (100%) rename {ida => scripts}/old/data_2025.08.07.0000.0000.yml (100%) rename {ida => scripts}/old/data_2025.08.22.0000.0000.yml (100%) rename {ida => scripts}/old/data_2025.09.04.0000.0000.yml (100%) rename {ida => scripts}/old/data_2025.09.30.0000.0000.yml (100%) rename {ida => scripts}/old/data_2025.10.13.0000.0000.yml (100%) rename {ida => scripts}/old/data_2025.10.30.0000.0000.yml (100%) rename {ida => scripts}/old/data_2025.12.09.0000.0000.yml (100%) rename {ida => scripts}/old/data_2025.12.18.0000.0000.yml (100%) rename {ida => scripts}/old/data_2025.12.23.0000.0000.yml (100%) rename {ida => scripts}/old/data_2026.01.21.0000.0000.yml (100%) rename {ida => scripts}/old/data_2026.01.30.0000.0000.yml (100%) rename {ida => scripts}/old/data_2026.02.20.0000.0000.yml (100%) rename {ida => scripts}/old/data_2026.03.07.0000.0000.yml (100%) rename {ida => scripts}/old/data_2026.03.17.0000.0000.yml (100%) rename {ida => scripts}/old/data_2026.04.21.0000.0000.yml (100%) rename {ida => scripts}/old/data_2026.05.01.0000.0000.yml (100%) rename {ida => scripts}/old/data_2026.05.25.0000.0000.yml (100%) rename {ida => scripts}/old/data_2026.06.10.0000.0000.yml (100%) rename {ida => scripts}/old/data_2026.06.18.0000.0000.yml (100%) rename {ida => scripts}/old/data_2026.07.16.0001.0000.yml (100%) rename {ida => scripts}/old/data_2026.08.05.0000.0000.yml (100%) rename {ida => scripts}/old/data_2026.08.11.0000.0000.yml (100%) rename {ida => scripts}/old/data_2026.09.01.0000.0000.yml (100%) rename {ida => scripts}/old/data_2026.09.15.0000.0000.yml (100%) rename {ida => scripts}/old/data_2026.10.06.0000.0000.yml (100%) rename {ida => scripts}/old/ffxiv_idarename_2020.10.21.0001.0000.py (100%) rename {ida => scripts}/old/ffxiv_idarename_2020.12.02.0000.0000.py (100%) rename {ida => scripts}/old/ffxiv_idarename_2020.12.15.0000.0000.py (100%) rename {ida => scripts}/poetry.lock (100%) rename {ida => scripts}/pyproject.toml (100%) create mode 100644 scripts/structs_schema.py diff --git a/CExporter/Program.cs b/CExporter/Program.cs index 4976271a72..e60adc05a8 100644 --- a/CExporter/Program.cs +++ b/CExporter/Program.cs @@ -13,11 +13,11 @@ public static void Main(string[] args) { var noWrite = args.Contains("--no-write"); var timeStart = DateTime.Now; var dir = new DirectoryInfo(Environment.CurrentDirectory); - while (dir.FullName.Contains("ida") && !dir.FullName.EndsWith("ida")) { + while (dir.FullName.Contains("scripts") && !dir.FullName.EndsWith("scripts")) { dir = dir.Parent!; } - while (!dir.FullName.Contains("ida") && !dir.FullName.EndsWith("ida")) { - dir = dir.GetDirectories("ida", SearchOption.AllDirectories).FirstOrDefault() ?? dir.Parent!; + while (!dir.FullName.Contains("scripts") && !dir.FullName.EndsWith("scripts")) { + dir = dir.GetDirectories("scripts", SearchOption.AllDirectories).FirstOrDefault() ?? dir.Parent!; } Exporter.ProcessTypes(quiet); diff --git a/ida/README.md b/scripts/README.md similarity index 100% rename from ida/README.md rename to scripts/README.md diff --git a/ida/classinformer.csv b/scripts/classinformer.csv similarity index 100% rename from ida/classinformer.csv rename to scripts/classinformer.csv diff --git a/ida/classinformer_lysander.csv b/scripts/classinformer_lysander.csv similarity index 100% rename from ida/classinformer_lysander.csv rename to scripts/classinformer_lysander.csv diff --git a/ida/data-validator.js b/scripts/data-validator.js similarity index 100% rename from ida/data-validator.js rename to scripts/data-validator.js diff --git a/ida/data.yml b/scripts/data.yml similarity index 100% rename from ida/data.yml rename to scripts/data.yml diff --git a/ida/data_schema.py b/scripts/data_schema.py similarity index 100% rename from ida/data_schema.py rename to scripts/data_schema.py diff --git a/ida/luminapie/__init__.py b/scripts/errors.txt similarity index 100% rename from ida/luminapie/__init__.py rename to scripts/errors.txt diff --git a/ida/ffxiv_addonmagic.py b/scripts/ffxiv_addonmagic.py similarity index 100% rename from ida/ffxiv_addonmagic.py rename to scripts/ffxiv_addonmagic.py diff --git a/ida/ffxiv_data_struct_matcher.py b/scripts/ffxiv_data_struct_matcher.py similarity index 100% rename from ida/ffxiv_data_struct_matcher.py rename to scripts/ffxiv_data_struct_matcher.py diff --git a/ida/ffxiv_exdgetters.py b/scripts/ffxiv_exdgetters.py similarity index 100% rename from ida/ffxiv_exdgetters.py rename to scripts/ffxiv_exdgetters.py diff --git a/ida/ffxiv_extract_uld_names.py b/scripts/ffxiv_extract_uld_names.py similarity index 100% rename from ida/ffxiv_extract_uld_names.py rename to scripts/ffxiv_extract_uld_names.py diff --git a/ida/ffxiv_fullrun_ida.py b/scripts/ffxiv_fullrun_ida.py similarity index 100% rename from ida/ffxiv_fullrun_ida.py rename to scripts/ffxiv_fullrun_ida.py diff --git a/ida/ffxiv_idarename.py b/scripts/ffxiv_idarename.py similarity index 100% rename from ida/ffxiv_idarename.py rename to scripts/ffxiv_idarename.py diff --git a/ida/ffxiv_luafuncs_ida.py b/scripts/ffxiv_luafuncs_ida.py similarity index 100% rename from ida/ffxiv_luafuncs_ida.py rename to scripts/ffxiv_luafuncs_ida.py diff --git a/ida/ffxiv_sigmaker.py b/scripts/ffxiv_sigmaker.py similarity index 100% rename from ida/ffxiv_sigmaker.py rename to scripts/ffxiv_sigmaker.py diff --git a/ida/ffxiv_structimporter.py b/scripts/ffxiv_structimporter.py similarity index 99% rename from ida/ffxiv_structimporter.py rename to scripts/ffxiv_structimporter.py index 086c2abfb5..b78e989ef7 100644 --- a/ida/ffxiv_structimporter.py +++ b/scripts/ffxiv_structimporter.py @@ -15,6 +15,23 @@ from structs_schema import * +predefine = """ +typedef char size8_st; +typedef unsigned char size8_t; +typedef short size16_st; +typedef unsigned short size16_t; +typedef int size32_st; +typedef unsigned int size32_t; +typedef long long size64_st; +typedef unsigned long long size64_t; + +#define DTOR(type) type *(__fastcall *Dtor)(type *__hidden self, size8_t freeFlags); +""" + +struct_string_forward = "" +struct_string_define = "" + + class BaseApi: @abstractmethod def can_run(self): @@ -257,6 +274,7 @@ def get_yaml(self): struct["union"], static_member_functions, static_members, + struct["template_types"] ) ) return DefinedStructExport(enums, structs) @@ -500,6 +518,8 @@ def delete_struct(self, struct): def create_struct(self, struct): # type: (DefinedStruct) -> None + + # TODO: change this over to ida_srclang fullname = self.clean_struct_name(struct.type) if self.srclang_importer: diff --git a/ida/ffxiv_structs.yml b/scripts/ffxiv_structs.yml similarity index 100% rename from ida/ffxiv_structs.yml rename to scripts/ffxiv_structs.yml diff --git a/ida/ida_wrapper.py b/scripts/ida_wrapper.py similarity index 82% rename from ida/ida_wrapper.py rename to scripts/ida_wrapper.py index b1f0795eb3..a2eb68dfc8 100644 --- a/ida/ida_wrapper.py +++ b/scripts/ida_wrapper.py @@ -9,6 +9,7 @@ import ida_hexrays import ida_name import ida_funcs +import ida_srclang from structs_schema import * from abc import abstractmethod @@ -16,28 +17,53 @@ # For more information about TERR_ constants, see: # https://docs.hex-rays.com/developer-guide/idapython/idapython-porting-guide-ida-9#type-information-error-codes -class BaseIdaInterface(object): - @abstractmethod - def get_struct_id(self, name): - pass +def _get_ida_major() -> int: + return idaapi.IDA_SDK_VERSION // 100 - @abstractmethod - def get_enum_id(self, name): - pass - - @abstractmethod - def delete_enum_members(self, eid: int): - """Remove all enum members +class SrcInterface(object): + def mark_as_vtable(vtbl_name: str) -> str | None: + """ + Mark a struct as a vtable (TAUDT_VFTABLE). - Args: - eid (int): The id of the enum + Returns: + An error string on failure, None on success. """ - pass + tif = ida_typeinf.tinfo_t() - def enum_exists(self, name: str): - return self.get_enum_id(name) != idaapi.BADADDR + if not tif.get_named_type(None, vtbl_name): + return f"Type '{vtbl_name}' not found in type library" + + if not tif.is_struct(): + actual = "union" if tif.is_union() else "enum" if tif.is_enum() else "unknown" + return f"'{vtbl_name}' is not a struct (got {actual})" + + udt = ida_typeinf.udt_type_data_t() + if not tif.get_udt_details(udt): + return f"Failed to retrieve UDT details for '{vtbl_name}'" + + if udt.taudt_bits & ida_typeinf.TAUDT_VFTABLE: + return f"'{vtbl_name}' is already marked as a vtable" + + udt.taudt_bits |= ida_typeinf.TAUDT_VFTABLE + + if not tif.create_udt(udt, ida_typeinf.BTF_STRUCT): + return f"Failed to reconstruct UDT for '{vtbl_name}' after setting vtable flag" - def get_idc_type_from_ida_type(self, type: str): + if _get_ida_major() >= 9: + result = tif.set_named_type(None, vtbl_name) + else: + result = tif.set_named_type( + ida_typeinf.get_idati(), + vtbl_name, + ida_typeinf.NTF_REPLACE + ) + + if not result: + return f"Failed to write back '{vtbl_name}' to type library" + + return None + + def get_idc_type_from_ida_type(self, type: str) -> int: """Retrieve the idc type from the ida type. Args: @@ -95,7 +121,7 @@ def get_idc_type_from_ida_type(self, type: str): else: return ida_bytes.stru_flag() - def get_idc_type_from_size(self, size: int, offset=0): + def get_idc_type_from_size(self, size: int, offset=0) -> int: if offset == 0: offset = size if offset % 8 == 0 and size >= 8: @@ -107,7 +133,7 @@ def get_idc_type_from_size(self, size: int, offset=0): else: return ida_bytes.byte_flag() - def get_size_from_idc_type(self, type: int): + def get_size_from_idc_type(self, type: int) -> int: if type == ida_bytes.byte_flag(): return 1 elif type == ida_bytes.word_flag(): @@ -122,8 +148,36 @@ def get_size_from_idc_type(self, type: int): return 8 else: return 0 + + def get_string_from_idc_type_and_sign(self, type: int, signed: bool = False) -> str: + if type == ida_bytes.byte_flag(): + if signed: + return "size8_st" + else: + return "size8_t" + elif type == ida_bytes.word_flag(): + if signed: + return "size16_st" + else: + return "size16_t" + elif type == ida_bytes.dword_flag(): + if signed: + return "size32_st" + else: + return "size32_t" + elif type == ida_bytes.qword_flag(): + if signed: + return "size32_st" + else: + return "size64_t" + elif type == ida_bytes.float_flag(): + return "float" + elif type == ida_bytes.double_flag(): + return "double" + else: + return "" - def is_signed(self, type: str): + def is_signed(self, type: str) -> bool: if ( type == "__int8" or type == "__int16" @@ -135,9 +189,167 @@ def is_signed(self, type: str): else: return False - def get_size_from_ida_type(self, type: str): + def get_size_from_ida_type(self, type: str) -> int: return self.get_size_from_idc_type(self.get_idc_type_from_ida_type(type)) + def select_parser(self): + """Sets the parser used for building struct data. + + Args: + lang (srclang_t): The language the parser should use one of (SRCLANG_C, SRCLANG_CPP, SRCLANG_OBJC, SRCLANG_SWIFT, SRCLANG_GO) + """ + ida_srclang.set_parser_argv("clang", "-x c++ -target x86_64-pc-win32") + + def build_struct_string(self, struct: DefinedStruct, struct_lookup: dict[str, int], enum_lookup: dict[str, str]) -> tuple[str, str, list[str]]: + """Builds the srclang definition required to parse the object. + + Args: + struct (DefinedStruct): The struct data to build""" + struct_string_forward: str = "" + struct_string_define: str = "" + struct_string_vtables: list[str] = [] + struct_name = struct.type + + struct_string_forward += f"struct {struct_name};\n" + if (struct.virtual_functions): + struct_string_forward += f"struct {struct_name}_vtbl;\n" + struct_string_vtables.append(f"{struct_name}_vtbl") + + struct_string_define += f"struct {struct_name}" + + prev_size = 0 + contiguous_fields = True + for field in struct.fields: + offset = field.offset + + while offset > prev_size: + contiguous_fields = False + fill_size = offset - prev_size + if self.full_padding: + flag = self.get_idc_type_from_size(prev_size) + size = self.get_size_from_idc_type(flag) + if size > fill_size: + flag = self.get_idc_type_from_size( + fill_size, prev_size + ) + size = self.get_size_from_idc_type(flag) + + struct_string_define += f" {self.get_string_from_idc_type_and_sign(flag)} field_{prev_size:X};" + else: + struct_string_define += f" char field_{prev_size:X}[{fill_size}];" + + prev_size += size + + field_is_base = field.base and contiguous_fields + field_name = ( + field.name + if not field_is_base + else "baseclass_{0:X}".format(offset) + ) + field_type = field.type + if field_type == "__fastcall": + struct_string_define += f" {field.return_type} ({field_type} *{field_name})(" + for param in field.parameters: + struct_string_define += param.type + " " + param.name + ", " + struct_string_define = struct_string_define[:-2] + ");" + prev_size += 8 + elif (field_type in struct_lookup): + struct_string_define += f" {field_type} {field_name};" + prev_size += struct_lookup[field_type] + elif (field_type in enum_lookup): + struct_string_define += f" {field_type} {field_name};" + prev_size += self.get_size_from_ida_type(enum_lookup[field_type]) + else: + struct_string_define += f" {field_type} {field_name};" + + if struct.size is not None and struct.size != 0: + while struct.size > prev_size: + fill_size = struct.size - prev_size + if self.full_padding: + flag = self.get_idc_type_from_size(prev_size) + size = self.get_size_from_idc_type(flag) + if size > fill_size: + flag = self.get_idc_type_from_size( + fill_size, prev_size + ) + size = self.get_size_from_idc_type(flag) + + struct_string_define += f" {self.get_string_from_idc_type_and_sign(flag)} field_{prev_size:X};" + else: + struct_string_define += f" char field_{prev_size:X}[{fill_size}];" + + prev_size += size + + return (struct_string_forward, struct_string_define, struct_string_vtables) + + def parse_string(self, decl: str, vtables: list[str]): + """""" + ida_srclang.parse_decls_with_parser("clang", None, decl, False) + for vtable in vtables: + self.mark_as_vtable(vtable) + +class BaseIdaInterface(SrcInterface): + @abstractmethod + def get_struct_id(self, name): + pass + + @abstractmethod + def get_enum_id(self, name): + pass + + @abstractmethod + def delete_enum_members(self, eid: int): + """Remove all enum members + + Args: + eid (int): The id of the enum + """ + pass + + def mark_as_vtable(vtbl_name: str) -> str | None: + """ + Mark a struct as a vtable (TAUDT_VFTABLE). + Compatible with IDA 8.x and 9.x. + Returns an error string on failure, None on success. + """ + tif = ida_typeinf.tinfo_t() + + if not tif.get_named_type(None, vtbl_name): + return f"Type '{vtbl_name}' not found in type library" + + if not tif.is_struct(): + actual = "union" if tif.is_union() else "enum" if tif.is_enum() else "unknown" + return f"'{vtbl_name}' is not a struct (got {actual})" + + udt = ida_typeinf.udt_type_data_t() + if not tif.get_udt_details(udt): + return f"Failed to retrieve UDT details for '{vtbl_name}'" + + if udt.taudt_bits & ida_typeinf.TAUDT_VFTABLE: + return f"'{vtbl_name}' is already marked as a vtable" + + udt.taudt_bits |= ida_typeinf.TAUDT_VFTABLE + + if not tif.create_udt(udt, ida_typeinf.BTF_STRUCT): + return f"Failed to reconstruct UDT for '{vtbl_name}' after setting vtable flag" + + if _get_ida_major() >= 9: + result = tif.set_named_type(None, vtbl_name) + else: + result = tif.set_named_type( + ida_typeinf.get_idati(), + vtbl_name, + ida_typeinf.NTF_REPLACE + ) + + if not result: + return f"Failed to write back '{vtbl_name}' to type library" + + return None + + def enum_exists(self, name: str): + return self.get_enum_id(name) != idaapi.BADADDR + def clean_name(self, name: str): """Clean a name @@ -238,6 +450,7 @@ def get_tinfo_from_type(self, raw_type: str, array_size=0): ptr_tinfo = array_tinfo return ptr_tinfo + def rename_struct(self, sid: int, new_name: str): """Rename a struct in IDA. @@ -249,7 +462,7 @@ def rename_struct(self, sid: int, new_name: str): idc.set_struc_name(sid, new_name) -if idaapi.IDA_SDK_VERSION < 900: +if _get_ida_major() < 9: import ida_struct # pyright: ignore[reportMissingImports] import ida_enum # pyright: ignore[reportMissingImports] else: @@ -257,7 +470,7 @@ def rename_struct(self, sid: int, new_name: str): class IdaInterface(BaseIdaInterface): # This is only for IDA 7 and 8 due to a change in the API for IDA 9 - if idaapi.IDA_SDK_VERSION < 900 and idaapi.IDA_SDK_VERSION >= 700: + if 7 <= _get_ida_major() < 9: def get_tinfo_from_func_data(self, data: DefinedStructFuncField): """Retrieve a tinfo_t from a raw function data. @@ -732,7 +945,7 @@ def get_enum_flag(self): """ return ida_bytes.enum_flag() - elif idaapi.IDA_SDK_VERSION >= 900: + elif _get_ida_major() >= 900: def get_tinfo_from_func_data(self, data: DefinedStructFuncField): """Retrieve a tinfo_t from a raw function data. diff --git a/ida/idauser.cfg b/scripts/idauser.cfg similarity index 100% rename from ida/idauser.cfg rename to scripts/idauser.cfg diff --git a/scripts/luminapie/__init__.py b/scripts/luminapie/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ida/luminapie/definitions.py b/scripts/luminapie/definitions.py similarity index 100% rename from ida/luminapie/definitions.py rename to scripts/luminapie/definitions.py diff --git a/ida/luminapie/enums.py b/scripts/luminapie/enums.py similarity index 100% rename from ida/luminapie/enums.py rename to scripts/luminapie/enums.py diff --git a/ida/luminapie/excel.py b/scripts/luminapie/excel.py similarity index 100% rename from ida/luminapie/excel.py rename to scripts/luminapie/excel.py diff --git a/ida/luminapie/exdschema.py b/scripts/luminapie/exdschema.py similarity index 100% rename from ida/luminapie/exdschema.py rename to scripts/luminapie/exdschema.py diff --git a/ida/luminapie/file_handlers.py b/scripts/luminapie/file_handlers.py similarity index 100% rename from ida/luminapie/file_handlers.py rename to scripts/luminapie/file_handlers.py diff --git a/ida/luminapie/game_data.py b/scripts/luminapie/game_data.py similarity index 100% rename from ida/luminapie/game_data.py rename to scripts/luminapie/game_data.py diff --git a/ida/luminapie/se_crc.py b/scripts/luminapie/se_crc.py similarity index 100% rename from ida/luminapie/se_crc.py rename to scripts/luminapie/se_crc.py diff --git a/ida/luminapie/sqpack.py b/scripts/luminapie/sqpack.py similarity index 100% rename from ida/luminapie/sqpack.py rename to scripts/luminapie/sqpack.py diff --git a/ida/old/data_2020.12.29.0000.0000.yml b/scripts/old/data_2020.12.29.0000.0000.yml similarity index 100% rename from ida/old/data_2020.12.29.0000.0000.yml rename to scripts/old/data_2020.12.29.0000.0000.yml diff --git a/ida/old/data_2021.01.28.0000.0000.yml b/scripts/old/data_2021.01.28.0000.0000.yml similarity index 100% rename from ida/old/data_2021.01.28.0000.0000.yml rename to scripts/old/data_2021.01.28.0000.0000.yml diff --git a/ida/old/data_2021.02.10.0000.0000.yml b/scripts/old/data_2021.02.10.0000.0000.yml similarity index 100% rename from ida/old/data_2021.02.10.0000.0000.yml rename to scripts/old/data_2021.02.10.0000.0000.yml diff --git a/ida/old/data_2021.04.07.0000.0000.yml b/scripts/old/data_2021.04.07.0000.0000.yml similarity index 100% rename from ida/old/data_2021.04.07.0000.0000.yml rename to scripts/old/data_2021.04.07.0000.0000.yml diff --git a/ida/old/data_2021.04.21.0000.0000.yml b/scripts/old/data_2021.04.21.0000.0000.yml similarity index 100% rename from ida/old/data_2021.04.21.0000.0000.yml rename to scripts/old/data_2021.04.21.0000.0000.yml diff --git a/ida/old/data_2021.05.18.0000.0000.yml b/scripts/old/data_2021.05.18.0000.0000.yml similarity index 100% rename from ida/old/data_2021.05.18.0000.0000.yml rename to scripts/old/data_2021.05.18.0000.0000.yml diff --git a/ida/old/data_2021.05.29.0000.0000.yml b/scripts/old/data_2021.05.29.0000.0000.yml similarity index 100% rename from ida/old/data_2021.05.29.0000.0000.yml rename to scripts/old/data_2021.05.29.0000.0000.yml diff --git a/ida/old/data_2021.06.16.0000.0000.yml b/scripts/old/data_2021.06.16.0000.0000.yml similarity index 100% rename from ida/old/data_2021.06.16.0000.0000.yml rename to scripts/old/data_2021.06.16.0000.0000.yml diff --git a/ida/old/data_2021.07.15.0000.0000.yml b/scripts/old/data_2021.07.15.0000.0000.yml similarity index 100% rename from ida/old/data_2021.07.15.0000.0000.yml rename to scripts/old/data_2021.07.15.0000.0000.yml diff --git a/ida/old/data_2021.08.17.0000.0000.yml b/scripts/old/data_2021.08.17.0000.0000.yml similarity index 100% rename from ida/old/data_2021.08.17.0000.0000.yml rename to scripts/old/data_2021.08.17.0000.0000.yml diff --git a/ida/old/data_2021.11.28.0000.0000.yml b/scripts/old/data_2021.11.28.0000.0000.yml similarity index 100% rename from ida/old/data_2021.11.28.0000.0000.yml rename to scripts/old/data_2021.11.28.0000.0000.yml diff --git a/ida/old/data_2021.12.16.0000.0000.yml b/scripts/old/data_2021.12.16.0000.0000.yml similarity index 100% rename from ida/old/data_2021.12.16.0000.0000.yml rename to scripts/old/data_2021.12.16.0000.0000.yml diff --git a/ida/old/data_2021.12.24.0000.0000.yml b/scripts/old/data_2021.12.24.0000.0000.yml similarity index 100% rename from ida/old/data_2021.12.24.0000.0000.yml rename to scripts/old/data_2021.12.24.0000.0000.yml diff --git a/ida/old/data_2022.01.25.0000.0000.yml b/scripts/old/data_2022.01.25.0000.0000.yml similarity index 100% rename from ida/old/data_2022.01.25.0000.0000.yml rename to scripts/old/data_2022.01.25.0000.0000.yml diff --git a/ida/old/data_2022.03.01.0000.0000.yml b/scripts/old/data_2022.03.01.0000.0000.yml similarity index 100% rename from ida/old/data_2022.03.01.0000.0000.yml rename to scripts/old/data_2022.03.01.0000.0000.yml diff --git a/ida/old/data_2022.04.07.0000.0000.yml b/scripts/old/data_2022.04.07.0000.0000.yml similarity index 100% rename from ida/old/data_2022.04.07.0000.0000.yml rename to scripts/old/data_2022.04.07.0000.0000.yml diff --git a/ida/old/data_2022.04.20.0000.0000.yml b/scripts/old/data_2022.04.20.0000.0000.yml similarity index 100% rename from ida/old/data_2022.04.20.0000.0000.yml rename to scripts/old/data_2022.04.20.0000.0000.yml diff --git a/ida/old/data_2022.05.19.0000.0000.yml b/scripts/old/data_2022.05.19.0000.0000.yml similarity index 100% rename from ida/old/data_2022.05.19.0000.0000.yml rename to scripts/old/data_2022.05.19.0000.0000.yml diff --git a/ida/old/data_2022.05.27.0000.0000.yml b/scripts/old/data_2022.05.27.0000.0000.yml similarity index 100% rename from ida/old/data_2022.05.27.0000.0000.yml rename to scripts/old/data_2022.05.27.0000.0000.yml diff --git a/ida/old/data_2022.06.21.0000.0000.yml b/scripts/old/data_2022.06.21.0000.0000.yml similarity index 100% rename from ida/old/data_2022.06.21.0000.0000.yml rename to scripts/old/data_2022.06.21.0000.0000.yml diff --git a/ida/old/data_2022.07.08.0000.0000.yml b/scripts/old/data_2022.07.08.0000.0000.yml similarity index 100% rename from ida/old/data_2022.07.08.0000.0000.yml rename to scripts/old/data_2022.07.08.0000.0000.yml diff --git a/ida/old/data_2022.08.17.0000.0000.yml b/scripts/old/data_2022.08.17.0000.0000.yml similarity index 100% rename from ida/old/data_2022.08.17.0000.0000.yml rename to scripts/old/data_2022.08.17.0000.0000.yml diff --git a/ida/old/data_2022.08.25.0000.0000.yml b/scripts/old/data_2022.08.25.0000.0000.yml similarity index 100% rename from ida/old/data_2022.08.25.0000.0000.yml rename to scripts/old/data_2022.08.25.0000.0000.yml diff --git a/ida/old/data_2022.09.07.0000.0000.yml b/scripts/old/data_2022.09.07.0000.0000.yml similarity index 100% rename from ida/old/data_2022.09.07.0000.0000.yml rename to scripts/old/data_2022.09.07.0000.0000.yml diff --git a/ida/old/data_2022.10.05.0000.0000.yml b/scripts/old/data_2022.10.05.0000.0000.yml similarity index 100% rename from ida/old/data_2022.10.05.0000.0000.yml rename to scripts/old/data_2022.10.05.0000.0000.yml diff --git a/ida/old/data_2022.10.26.0000.0000.yml b/scripts/old/data_2022.10.26.0000.0000.yml similarity index 100% rename from ida/old/data_2022.10.26.0000.0000.yml rename to scripts/old/data_2022.10.26.0000.0000.yml diff --git a/ida/old/data_2022.11.09.0000.0000.yml b/scripts/old/data_2022.11.09.0000.0000.yml similarity index 100% rename from ida/old/data_2022.11.09.0000.0000.yml rename to scripts/old/data_2022.11.09.0000.0000.yml diff --git a/ida/old/data_2022.12.23.0000.0000.yml b/scripts/old/data_2022.12.23.0000.0000.yml similarity index 100% rename from ida/old/data_2022.12.23.0000.0000.yml rename to scripts/old/data_2022.12.23.0000.0000.yml diff --git a/ida/old/data_2023.01.11.0000.0000.yml b/scripts/old/data_2023.01.11.0000.0000.yml similarity index 100% rename from ida/old/data_2023.01.11.0000.0000.yml rename to scripts/old/data_2023.01.11.0000.0000.yml diff --git a/ida/old/data_2023.01.17.0000.0000.yml b/scripts/old/data_2023.01.17.0000.0000.yml similarity index 100% rename from ida/old/data_2023.01.17.0000.0000.yml rename to scripts/old/data_2023.01.17.0000.0000.yml diff --git a/ida/old/data_2023.02.03.0000.0000.yml b/scripts/old/data_2023.02.03.0000.0000.yml similarity index 100% rename from ida/old/data_2023.02.03.0000.0000.yml rename to scripts/old/data_2023.02.03.0000.0000.yml diff --git a/ida/old/data_2023.02.28.0000.0000.yml b/scripts/old/data_2023.02.28.0000.0000.yml similarity index 100% rename from ida/old/data_2023.02.28.0000.0000.yml rename to scripts/old/data_2023.02.28.0000.0000.yml diff --git a/ida/old/data_2023.03.24.0000.0000.yml b/scripts/old/data_2023.03.24.0000.0000.yml similarity index 100% rename from ida/old/data_2023.03.24.0000.0000.yml rename to scripts/old/data_2023.03.24.0000.0000.yml diff --git a/ida/old/data_2023.05.18.0000.0000.yml b/scripts/old/data_2023.05.18.0000.0000.yml similarity index 100% rename from ida/old/data_2023.05.18.0000.0000.yml rename to scripts/old/data_2023.05.18.0000.0000.yml diff --git a/ida/old/data_2023.05.31.0000.0000.yml b/scripts/old/data_2023.05.31.0000.0000.yml similarity index 100% rename from ida/old/data_2023.05.31.0000.0000.yml rename to scripts/old/data_2023.05.31.0000.0000.yml diff --git a/ida/old/data_2023.06.14.0000.0000.yml b/scripts/old/data_2023.06.14.0000.0000.yml similarity index 100% rename from ida/old/data_2023.06.14.0000.0000.yml rename to scripts/old/data_2023.06.14.0000.0000.yml diff --git a/ida/old/data_2023.07.06.0000.0000.yml b/scripts/old/data_2023.07.06.0000.0000.yml similarity index 100% rename from ida/old/data_2023.07.06.0000.0000.yml rename to scripts/old/data_2023.07.06.0000.0000.yml diff --git a/ida/old/data_2023.07.26.0000.0000.yml b/scripts/old/data_2023.07.26.0000.0000.yml similarity index 100% rename from ida/old/data_2023.07.26.0000.0000.yml rename to scripts/old/data_2023.07.26.0000.0000.yml diff --git a/ida/old/data_2023.09.28.0000.0000.yml b/scripts/old/data_2023.09.28.0000.0000.yml similarity index 100% rename from ida/old/data_2023.09.28.0000.0000.yml rename to scripts/old/data_2023.09.28.0000.0000.yml diff --git a/ida/old/data_2023.10.23.0000.0000.yml b/scripts/old/data_2023.10.23.0000.0000.yml similarity index 100% rename from ida/old/data_2023.10.23.0000.0000.yml rename to scripts/old/data_2023.10.23.0000.0000.yml diff --git a/ida/old/data_2023.11.09.0000.0000.yml b/scripts/old/data_2023.11.09.0000.0000.yml similarity index 100% rename from ida/old/data_2023.11.09.0000.0000.yml rename to scripts/old/data_2023.11.09.0000.0000.yml diff --git a/ida/old/data_2024.01.06.0000.0000.yml b/scripts/old/data_2024.01.06.0000.0000.yml similarity index 100% rename from ida/old/data_2024.01.06.0000.0000.yml rename to scripts/old/data_2024.01.06.0000.0000.yml diff --git a/ida/old/data_2024.02.05.0000.0000.yml b/scripts/old/data_2024.02.05.0000.0000.yml similarity index 100% rename from ida/old/data_2024.02.05.0000.0000.yml rename to scripts/old/data_2024.02.05.0000.0000.yml diff --git a/ida/old/data_2024.03.08.0000.0000.yml b/scripts/old/data_2024.03.08.0000.0000.yml similarity index 100% rename from ida/old/data_2024.03.08.0000.0000.yml rename to scripts/old/data_2024.03.08.0000.0000.yml diff --git a/ida/old/data_2024.03.27.0000.0000.yml b/scripts/old/data_2024.03.27.0000.0000.yml similarity index 100% rename from ida/old/data_2024.03.27.0000.0000.yml rename to scripts/old/data_2024.03.27.0000.0000.yml diff --git a/ida/old/data_2024.04.23.0000.0000.yml b/scripts/old/data_2024.04.23.0000.0000.yml similarity index 100% rename from ida/old/data_2024.04.23.0000.0000.yml rename to scripts/old/data_2024.04.23.0000.0000.yml diff --git a/ida/old/data_2024.06.18.0000.0000.yml b/scripts/old/data_2024.06.18.0000.0000.yml similarity index 100% rename from ida/old/data_2024.06.18.0000.0000.yml rename to scripts/old/data_2024.06.18.0000.0000.yml diff --git a/ida/old/data_2024.07.06.0000.0000.yml b/scripts/old/data_2024.07.06.0000.0000.yml similarity index 100% rename from ida/old/data_2024.07.06.0000.0000.yml rename to scripts/old/data_2024.07.06.0000.0000.yml diff --git a/ida/old/data_2024.07.10.0001.0000.yml b/scripts/old/data_2024.07.10.0001.0000.yml similarity index 100% rename from ida/old/data_2024.07.10.0001.0000.yml rename to scripts/old/data_2024.07.10.0001.0000.yml diff --git a/ida/old/data_2024.07.24.0000.0000.yml b/scripts/old/data_2024.07.24.0000.0000.yml similarity index 100% rename from ida/old/data_2024.07.24.0000.0000.yml rename to scripts/old/data_2024.07.24.0000.0000.yml diff --git a/ida/old/data_2024.08.02.0000.0000.yml b/scripts/old/data_2024.08.02.0000.0000.yml similarity index 100% rename from ida/old/data_2024.08.02.0000.0000.yml rename to scripts/old/data_2024.08.02.0000.0000.yml diff --git a/ida/old/data_2024.08.21.0000.0000.yml b/scripts/old/data_2024.08.21.0000.0000.yml similarity index 100% rename from ida/old/data_2024.08.21.0000.0000.yml rename to scripts/old/data_2024.08.21.0000.0000.yml diff --git a/ida/old/data_2024.11.06.0000.0000.yml b/scripts/old/data_2024.11.06.0000.0000.yml similarity index 100% rename from ida/old/data_2024.11.06.0000.0000.yml rename to scripts/old/data_2024.11.06.0000.0000.yml diff --git a/ida/old/data_2024.11.20.0000.0000.yml b/scripts/old/data_2024.11.20.0000.0000.yml similarity index 100% rename from ida/old/data_2024.11.20.0000.0000.yml rename to scripts/old/data_2024.11.20.0000.0000.yml diff --git a/ida/old/data_2024.12.07.0000.0000.yml b/scripts/old/data_2024.12.07.0000.0000.yml similarity index 100% rename from ida/old/data_2024.12.07.0000.0000.yml rename to scripts/old/data_2024.12.07.0000.0000.yml diff --git a/ida/old/data_2025.01.14.0000.0000.yml b/scripts/old/data_2025.01.14.0000.0000.yml similarity index 100% rename from ida/old/data_2025.01.14.0000.0000.yml rename to scripts/old/data_2025.01.14.0000.0000.yml diff --git a/ida/old/data_2025.01.28.0000.0000.yml b/scripts/old/data_2025.01.28.0000.0000.yml similarity index 100% rename from ida/old/data_2025.01.28.0000.0000.yml rename to scripts/old/data_2025.01.28.0000.0000.yml diff --git a/ida/old/data_2025.02.19.0000.0000.yml b/scripts/old/data_2025.02.19.0000.0000.yml similarity index 100% rename from ida/old/data_2025.02.19.0000.0000.yml rename to scripts/old/data_2025.02.19.0000.0000.yml diff --git a/ida/old/data_2025.02.27.0000.0000.yml b/scripts/old/data_2025.02.27.0000.0000.yml similarity index 100% rename from ida/old/data_2025.02.27.0000.0000.yml rename to scripts/old/data_2025.02.27.0000.0000.yml diff --git a/ida/old/data_2025.03.18.0000.0000.yml b/scripts/old/data_2025.03.18.0000.0000.yml similarity index 100% rename from ida/old/data_2025.03.18.0000.0000.yml rename to scripts/old/data_2025.03.18.0000.0000.yml diff --git a/ida/old/data_2025.03.27.0000.0000.yml b/scripts/old/data_2025.03.27.0000.0000.yml similarity index 100% rename from ida/old/data_2025.03.27.0000.0000.yml rename to scripts/old/data_2025.03.27.0000.0000.yml diff --git a/ida/old/data_2025.04.16.0000.0000.yml b/scripts/old/data_2025.04.16.0000.0000.yml similarity index 100% rename from ida/old/data_2025.04.16.0000.0000.yml rename to scripts/old/data_2025.04.16.0000.0000.yml diff --git a/ida/old/data_2025.05.17.0000.0000.yml b/scripts/old/data_2025.05.17.0000.0000.yml similarity index 100% rename from ida/old/data_2025.05.17.0000.0000.yml rename to scripts/old/data_2025.05.17.0000.0000.yml diff --git a/ida/old/data_2025.06.10.0000.0000.yml b/scripts/old/data_2025.06.10.0000.0000.yml similarity index 100% rename from ida/old/data_2025.06.10.0000.0000.yml rename to scripts/old/data_2025.06.10.0000.0000.yml diff --git a/ida/old/data_2025.06.19.0000.0000.yml b/scripts/old/data_2025.06.19.0000.0000.yml similarity index 100% rename from ida/old/data_2025.06.19.0000.0000.yml rename to scripts/old/data_2025.06.19.0000.0000.yml diff --git a/ida/old/data_2025.06.28.0000.0000.yml b/scripts/old/data_2025.06.28.0000.0000.yml similarity index 100% rename from ida/old/data_2025.06.28.0000.0000.yml rename to scripts/old/data_2025.06.28.0000.0000.yml diff --git a/ida/old/data_2025.07.30.0000.0000.yml b/scripts/old/data_2025.07.30.0000.0000.yml similarity index 100% rename from ida/old/data_2025.07.30.0000.0000.yml rename to scripts/old/data_2025.07.30.0000.0000.yml diff --git a/ida/old/data_2025.08.07.0000.0000.yml b/scripts/old/data_2025.08.07.0000.0000.yml similarity index 100% rename from ida/old/data_2025.08.07.0000.0000.yml rename to scripts/old/data_2025.08.07.0000.0000.yml diff --git a/ida/old/data_2025.08.22.0000.0000.yml b/scripts/old/data_2025.08.22.0000.0000.yml similarity index 100% rename from ida/old/data_2025.08.22.0000.0000.yml rename to scripts/old/data_2025.08.22.0000.0000.yml diff --git a/ida/old/data_2025.09.04.0000.0000.yml b/scripts/old/data_2025.09.04.0000.0000.yml similarity index 100% rename from ida/old/data_2025.09.04.0000.0000.yml rename to scripts/old/data_2025.09.04.0000.0000.yml diff --git a/ida/old/data_2025.09.30.0000.0000.yml b/scripts/old/data_2025.09.30.0000.0000.yml similarity index 100% rename from ida/old/data_2025.09.30.0000.0000.yml rename to scripts/old/data_2025.09.30.0000.0000.yml diff --git a/ida/old/data_2025.10.13.0000.0000.yml b/scripts/old/data_2025.10.13.0000.0000.yml similarity index 100% rename from ida/old/data_2025.10.13.0000.0000.yml rename to scripts/old/data_2025.10.13.0000.0000.yml diff --git a/ida/old/data_2025.10.30.0000.0000.yml b/scripts/old/data_2025.10.30.0000.0000.yml similarity index 100% rename from ida/old/data_2025.10.30.0000.0000.yml rename to scripts/old/data_2025.10.30.0000.0000.yml diff --git a/ida/old/data_2025.12.09.0000.0000.yml b/scripts/old/data_2025.12.09.0000.0000.yml similarity index 100% rename from ida/old/data_2025.12.09.0000.0000.yml rename to scripts/old/data_2025.12.09.0000.0000.yml diff --git a/ida/old/data_2025.12.18.0000.0000.yml b/scripts/old/data_2025.12.18.0000.0000.yml similarity index 100% rename from ida/old/data_2025.12.18.0000.0000.yml rename to scripts/old/data_2025.12.18.0000.0000.yml diff --git a/ida/old/data_2025.12.23.0000.0000.yml b/scripts/old/data_2025.12.23.0000.0000.yml similarity index 100% rename from ida/old/data_2025.12.23.0000.0000.yml rename to scripts/old/data_2025.12.23.0000.0000.yml diff --git a/ida/old/data_2026.01.21.0000.0000.yml b/scripts/old/data_2026.01.21.0000.0000.yml similarity index 100% rename from ida/old/data_2026.01.21.0000.0000.yml rename to scripts/old/data_2026.01.21.0000.0000.yml diff --git a/ida/old/data_2026.01.30.0000.0000.yml b/scripts/old/data_2026.01.30.0000.0000.yml similarity index 100% rename from ida/old/data_2026.01.30.0000.0000.yml rename to scripts/old/data_2026.01.30.0000.0000.yml diff --git a/ida/old/data_2026.02.20.0000.0000.yml b/scripts/old/data_2026.02.20.0000.0000.yml similarity index 100% rename from ida/old/data_2026.02.20.0000.0000.yml rename to scripts/old/data_2026.02.20.0000.0000.yml diff --git a/ida/old/data_2026.03.07.0000.0000.yml b/scripts/old/data_2026.03.07.0000.0000.yml similarity index 100% rename from ida/old/data_2026.03.07.0000.0000.yml rename to scripts/old/data_2026.03.07.0000.0000.yml diff --git a/ida/old/data_2026.03.17.0000.0000.yml b/scripts/old/data_2026.03.17.0000.0000.yml similarity index 100% rename from ida/old/data_2026.03.17.0000.0000.yml rename to scripts/old/data_2026.03.17.0000.0000.yml diff --git a/ida/old/data_2026.04.21.0000.0000.yml b/scripts/old/data_2026.04.21.0000.0000.yml similarity index 100% rename from ida/old/data_2026.04.21.0000.0000.yml rename to scripts/old/data_2026.04.21.0000.0000.yml diff --git a/ida/old/data_2026.05.01.0000.0000.yml b/scripts/old/data_2026.05.01.0000.0000.yml similarity index 100% rename from ida/old/data_2026.05.01.0000.0000.yml rename to scripts/old/data_2026.05.01.0000.0000.yml diff --git a/ida/old/data_2026.05.25.0000.0000.yml b/scripts/old/data_2026.05.25.0000.0000.yml similarity index 100% rename from ida/old/data_2026.05.25.0000.0000.yml rename to scripts/old/data_2026.05.25.0000.0000.yml diff --git a/ida/old/data_2026.06.10.0000.0000.yml b/scripts/old/data_2026.06.10.0000.0000.yml similarity index 100% rename from ida/old/data_2026.06.10.0000.0000.yml rename to scripts/old/data_2026.06.10.0000.0000.yml diff --git a/ida/old/data_2026.06.18.0000.0000.yml b/scripts/old/data_2026.06.18.0000.0000.yml similarity index 100% rename from ida/old/data_2026.06.18.0000.0000.yml rename to scripts/old/data_2026.06.18.0000.0000.yml diff --git a/ida/old/data_2026.07.16.0001.0000.yml b/scripts/old/data_2026.07.16.0001.0000.yml similarity index 100% rename from ida/old/data_2026.07.16.0001.0000.yml rename to scripts/old/data_2026.07.16.0001.0000.yml diff --git a/ida/old/data_2026.08.05.0000.0000.yml b/scripts/old/data_2026.08.05.0000.0000.yml similarity index 100% rename from ida/old/data_2026.08.05.0000.0000.yml rename to scripts/old/data_2026.08.05.0000.0000.yml diff --git a/ida/old/data_2026.08.11.0000.0000.yml b/scripts/old/data_2026.08.11.0000.0000.yml similarity index 100% rename from ida/old/data_2026.08.11.0000.0000.yml rename to scripts/old/data_2026.08.11.0000.0000.yml diff --git a/ida/old/data_2026.09.01.0000.0000.yml b/scripts/old/data_2026.09.01.0000.0000.yml similarity index 100% rename from ida/old/data_2026.09.01.0000.0000.yml rename to scripts/old/data_2026.09.01.0000.0000.yml diff --git a/ida/old/data_2026.09.15.0000.0000.yml b/scripts/old/data_2026.09.15.0000.0000.yml similarity index 100% rename from ida/old/data_2026.09.15.0000.0000.yml rename to scripts/old/data_2026.09.15.0000.0000.yml diff --git a/ida/old/data_2026.10.06.0000.0000.yml b/scripts/old/data_2026.10.06.0000.0000.yml similarity index 100% rename from ida/old/data_2026.10.06.0000.0000.yml rename to scripts/old/data_2026.10.06.0000.0000.yml diff --git a/ida/old/ffxiv_idarename_2020.10.21.0001.0000.py b/scripts/old/ffxiv_idarename_2020.10.21.0001.0000.py similarity index 100% rename from ida/old/ffxiv_idarename_2020.10.21.0001.0000.py rename to scripts/old/ffxiv_idarename_2020.10.21.0001.0000.py diff --git a/ida/old/ffxiv_idarename_2020.12.02.0000.0000.py b/scripts/old/ffxiv_idarename_2020.12.02.0000.0000.py similarity index 100% rename from ida/old/ffxiv_idarename_2020.12.02.0000.0000.py rename to scripts/old/ffxiv_idarename_2020.12.02.0000.0000.py diff --git a/ida/old/ffxiv_idarename_2020.12.15.0000.0000.py b/scripts/old/ffxiv_idarename_2020.12.15.0000.0000.py similarity index 100% rename from ida/old/ffxiv_idarename_2020.12.15.0000.0000.py rename to scripts/old/ffxiv_idarename_2020.12.15.0000.0000.py diff --git a/ida/poetry.lock b/scripts/poetry.lock similarity index 100% rename from ida/poetry.lock rename to scripts/poetry.lock diff --git a/ida/pyproject.toml b/scripts/pyproject.toml similarity index 100% rename from ida/pyproject.toml rename to scripts/pyproject.toml diff --git a/scripts/structs_schema.py b/scripts/structs_schema.py new file mode 100644 index 0000000000..9b8e87a76e --- /dev/null +++ b/scripts/structs_schema.py @@ -0,0 +1,81 @@ +from dataclasses import dataclass + + +@dataclass +class DefinedStructBase: + name: str + type: str + namespace: str + + +@dataclass +class DefinedStructEnum(DefinedStructBase): + underlying: str + flags: bool + values: dict[str, int] + + +@dataclass +class DefinedStructFuncParam: + name: str + type: str + + +@dataclass +class DefinedStructVFunc: + name: str + return_type: str | None + offset: int + parameters: list[DefinedStructFuncParam] | None + + +@dataclass +class DefinedStructMemFunc: + signature: str + return_type: str + parameters: list[DefinedStructFuncParam] + name: str + + +@dataclass +class DefinedStructField(DefinedStructFuncParam): + offset: int + base: bool + + +@dataclass +class DefinedStructFuncField(DefinedStructField): + return_type: str | None + parameters: list[DefinedStructFuncParam] | None + + +@dataclass +class DefinedStructStaticMember: + signature: str + relative_offsets: list[int] + return_type: str + is_pointer: bool + + +@dataclass +class DefinedStructFixedField(DefinedStructField): + size: int + is_string: bool + +@dataclass +class DefinedStruct(DefinedStructBase): + fields: list[DefinedStructField] + size: int | None + vtable_size: int | None + virtual_functions: list[DefinedStructVFunc] | None + member_functions: list[DefinedStructMemFunc] + union: bool + static_member_functions: list[DefinedStructMemFunc] | None + static_members: list[DefinedStructStaticMember] | None + template_types: list[str] + + +@dataclass +class DefinedStructExport: + enums: list[DefinedStructEnum] + structs: list[DefinedStruct] From f3b0e34549c65b0a8264e3741a21115fec04b2eb Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Sat, 2 May 2026 12:29:10 +0200 Subject: [PATCH 03/95] add new template_types but only as empty array for now --- CExporter/Exporter.cs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/CExporter/Exporter.cs b/CExporter/Exporter.cs index 248fbc8c3d..d84f777457 100644 --- a/CExporter/Exporter.cs +++ b/CExporter/Exporter.cs @@ -675,7 +675,8 @@ .. memberFunctionParameters.Select(ProcessParameter).ToArray()] ProcessField(unionField, unionStartField.GetFieldOffset()) ], MemberFunctions = [], - StructTypeOverride = type.FixTypeName() + $"{ExporterStatics.Separator}{attr.Union}{(attr.IsStruct ? $"{ExporterStatics.Separator}{attr.Struct}" : "")}" + StructTypeOverride = type.FixTypeName() + $"{ExporterStatics.Separator}{attr.Union}{(attr.IsStruct ? $"{ExporterStatics.Separator}{attr.Struct}" : "")}", + TemplateTypes = [] }); } else unions[index].Fields = [ @@ -720,7 +721,8 @@ .. memberFunctionParameters.Select(ProcessParameter).ToArray()] Fields = ProcessFields(fields), VirtualFunctions = virtualFunctions, MemberFunctions = memberFunctionsArray, - StructTypeOverride = overrideType + StructTypeOverride = overrideType, + TemplateTypes = [] }; foreach (var (unionAttr, fieldInfo) in unionOffsets.Where(t => !t.Key.IsStruct)) { @@ -817,6 +819,7 @@ public class ProcessedStaticMembers { public required Type ReturnType; } +// TODO: Export if struct is generic with types public class ProcessedStruct { public string? StructTypeOverride; public bool IsUnion; @@ -830,6 +833,7 @@ public class ProcessedStruct { public required ProcessedMemberFunction[] MemberFunctions; public ProcessedMemberFunction[]? StaticMemberFunctions; public ProcessedStaticMembers[]? StaticMembers; + public required string[] TemplateTypes; [YamlIgnore] public required string StructTypeName; [YamlIgnore] @@ -1046,6 +1050,13 @@ public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializ } emitter.Emit(new SequenceEnd()); } + emitter.Emit(new Scalar("template_types")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Flow)); + foreach (var templateType in s.TemplateTypes) + { + emitter.Emit(new Scalar(templateType)); + } + emitter.Emit(new SequenceEnd()); emitter.Emit(new MappingEnd()); } From d50ba92586074ade653cd9475debcb2f9b7f409d Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Sat, 2 May 2026 12:29:35 +0200 Subject: [PATCH 04/95] shift how some types are exported prep work for new importer script --- CExporter/Extensions.cs | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/CExporter/Extensions.cs b/CExporter/Extensions.cs index 424ad7f471..9fc105fa8e 100644 --- a/CExporter/Extensions.cs +++ b/CExporter/Extensions.cs @@ -77,31 +77,20 @@ public static string FixTypeName(this Type type, bool shouldLower = true) { using var builderPooled = StringBuilderPool.Get(500); var builder = builderPooled.Builder; var name = type switch { - _ when type == typeof(void) => shouldLower ? type.Name.ToLower() : type.Name, - _ when type == typeof(byte) || type == typeof(byte*) || type == typeof(byte**) => shouldLower ? type.Name.ToLower() : type.Name, + _ when type == typeof(void) => "void", _ when type == typeof(char) => "wchar_t", - _ when type == typeof(bool) => "char", + _ when type == typeof(CStringPointer) => "char*", _ when type == typeof(float) => "float", _ when type == typeof(double) => "double", - _ when type == typeof(short) => "__int16", - _ when type == typeof(int) => "int", - _ when type == typeof(long) || type == typeof(nint) => "__int64", - _ when type == typeof(ushort) => "unsigned __int16", - _ when type == typeof(uint) => "unsigned int", - _ when type == typeof(ulong) || type == typeof(nuint) => "unsigned __int64", - _ when type == typeof(sbyte) => "__int8", - _ when type == typeof(char*) => "wchar_t*", - _ when type == typeof(short*) => "__int16*", - _ when type == typeof(ushort*) => "unsigned __int16*", - _ when type == typeof(int*) => "int*", - _ when type == typeof(uint*) => "unsigned int*", - _ when type == typeof(long*) || type == typeof(nint*) => "__int64*", - _ when type == typeof(ulong*) || type == typeof(nuint*) => "unsigned __int64*", - _ when type == typeof(float*) => "float*", - _ when type == typeof(Half) => "__int16", // Half is a struct that is 2 bytes long and does not exist in C so we just use __int16 - _ when type == typeof(void*) => "__int64", - _ when type == typeof(void**) => "__int64*", - _ when type == typeof(CStringPointer) => "char*", + _ when type == typeof(byte) || type == typeof(bool) => "size8_t", + _ when type == typeof(sbyte) => "size8_st", + _ when type == typeof(ushort) || type == typeof(Half) => "size16_t", + _ when type == typeof(short) => "size16_st", + _ when type == typeof(uint) => "size32_t", + _ when type == typeof(int) => "size32_st", + _ when type == typeof(long) || type == typeof(nint) => "size64_st", + _ when type == typeof(ulong) || type == typeof(nuint) => "size64_t", + _ when type.IsPointer() => $"{FixTypeName(type.GetElementType()!, shouldLower)}*", // able to handle infinite amount of pointers until stack size says no but this should never happen _ => type.SanitizeName() }; return builder.Append(name).Replace("+", ExporterStatics.Separator).Replace(".", ExporterStatics.Separator).ToString(); From 7d45320f342cb97696d9ccff69c791a4f14493cb Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Sat, 2 May 2026 23:40:39 +0200 Subject: [PATCH 05/95] fix wrong wrapper version detection for old scripts --- scripts/ida_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ida_wrapper.py b/scripts/ida_wrapper.py index a2eb68dfc8..bb753e1f50 100644 --- a/scripts/ida_wrapper.py +++ b/scripts/ida_wrapper.py @@ -945,7 +945,7 @@ def get_enum_flag(self): """ return ida_bytes.enum_flag() - elif _get_ida_major() >= 900: + elif _get_ida_major() >= 9: def get_tinfo_from_func_data(self, data: DefinedStructFuncField): """Retrieve a tinfo_t from a raw function data. From 1bcb95e08c9ba466dbf0dd677bd5790710f2d26b Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Sun, 3 May 2026 00:01:41 +0200 Subject: [PATCH 06/95] fix ordering due to dataclass --- scripts/ffxiv_structimporter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ffxiv_structimporter.py b/scripts/ffxiv_structimporter.py index b78e989ef7..f2d7cd680f 100644 --- a/scripts/ffxiv_structimporter.py +++ b/scripts/ffxiv_structimporter.py @@ -149,8 +149,8 @@ def get_yaml(self): DefinedStructEnum( enum["name"], enum["type"], - enum["underlying"], enum["namespace"], + enum["underlying"], enum["flags"], enum["values"], ) @@ -274,7 +274,7 @@ def get_yaml(self): struct["union"], static_member_functions, static_members, - struct["template_types"] + [] ) ) return DefinedStructExport(enums, structs) From b4cd6ef00ac44e9cc0aa7a69942c9ea99a22fd80 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Mon, 4 May 2026 00:09:49 +0200 Subject: [PATCH 07/95] improve linter type discovery --- scripts/structs_schema.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/structs_schema.py b/scripts/structs_schema.py index 9b8e87a76e..796bf84f17 100644 --- a/scripts/structs_schema.py +++ b/scripts/structs_schema.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +from typing import TypeAlias @dataclass @@ -62,10 +63,12 @@ class DefinedStructFixedField(DefinedStructField): size: int is_string: bool +Fields: TypeAlias = list[DefinedStructField | DefinedStructFixedField] + @dataclass class DefinedStruct(DefinedStructBase): - fields: list[DefinedStructField] - size: int | None + fields: Fields + size: int vtable_size: int | None virtual_functions: list[DefinedStructVFunc] | None member_functions: list[DefinedStructMemFunc] From 1ab1e1c7a566efae4f4cc43f879bfe7765398b74 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Mon, 4 May 2026 01:14:05 +0200 Subject: [PATCH 08/95] move wrapper to new location --- scripts/ffxiv_data_struct_matcher.py | 2 +- scripts/ffxiv_exdgetters.py | 2 +- scripts/ffxiv_structimporter.py | 2 +- scripts/{ => ida}/ida_wrapper.py | 16 ++++++++++++++-- 4 files changed, 17 insertions(+), 5 deletions(-) rename scripts/{ => ida}/ida_wrapper.py (98%) diff --git a/scripts/ffxiv_data_struct_matcher.py b/scripts/ffxiv_data_struct_matcher.py index ec204c7370..0e393b5bc7 100644 --- a/scripts/ffxiv_data_struct_matcher.py +++ b/scripts/ffxiv_data_struct_matcher.py @@ -8,7 +8,7 @@ import ida_kernwin import os import sys -from ida_wrapper import IdaInterface +from ida.ida_wrapper import IdaInterface from structs_schema import * from data_schema import * from yaml import load diff --git a/scripts/ffxiv_exdgetters.py b/scripts/ffxiv_exdgetters.py index 8c66eb37c8..e6ecef9545 100644 --- a/scripts/ffxiv_exdgetters.py +++ b/scripts/ffxiv_exdgetters.py @@ -45,7 +45,7 @@ def process_pattern(self, pattern): import ida_hexrays import ida_name import idc - from ida_wrapper import IdaInterface + from ida.ida_wrapper import IdaInterface except ImportError as e: print("Warning: Unable to load IDA failed with {0}".format(e)) else: diff --git a/scripts/ffxiv_structimporter.py b/scripts/ffxiv_structimporter.py index f2d7cd680f..eb5f64c01b 100644 --- a/scripts/ffxiv_structimporter.py +++ b/scripts/ffxiv_structimporter.py @@ -310,7 +310,7 @@ def load_data_yaml(self): import ida_srclang import hashlib import copy - from ida_wrapper import IdaInterface + from ida.ida_wrapper import IdaInterface except ImportError: print("Warning: Unable to load IDA") else: diff --git a/scripts/ida_wrapper.py b/scripts/ida/ida_wrapper.py similarity index 98% rename from scripts/ida_wrapper.py rename to scripts/ida/ida_wrapper.py index bb753e1f50..f8a3115751 100644 --- a/scripts/ida_wrapper.py +++ b/scripts/ida/ida_wrapper.py @@ -176,6 +176,20 @@ def get_string_from_idc_type_and_sign(self, type: int, signed: bool = False) -> return "double" else: return "" + + def get_size_from_string(self, type: str) -> int: + """ + Gets the size of a base type string. + """ + if type == "size8_st" or type == "size8_t": + return 1 + if type == "size16_st" or type == "size16_t": + return 2 + if type == "size32_st" or type == "size32_t" or type == "float": + return 4 + if type == "size64_st" or type == "size64_t" or type == "double" or type.endswith('*'): + return 8 + return 0 def is_signed(self, type: str) -> bool: if ( @@ -922,9 +936,7 @@ def add_enum_member(self, eid: int, name: str, value: int, mask: int = -1): if mask == -1 and self.get_enum_bf(eid): mask = self.get_enum_default_mask(eid) - # print(f"Adding value: {value} with name: {name} and mask: {mask}") ec = ida_enum.add_enum_member(eid, name, value, mask) - # print(f"Got error code: {ec}") if ec == ida_enum.ENUM_MEMBER_ERROR_MASK: ida_enum.add_enum_member(eid, name, value) From c2804d640f3f252c59c9e0088f046a7931da19d1 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Mon, 4 May 2026 01:24:37 +0200 Subject: [PATCH 09/95] create new wrapper files for less duplication code between IDA and Ghidra --- scripts/exdschema_wrapper.py | 244 +++++++++++++++++++++++++++++++++++ scripts/lumina_wrapper.py | 199 ++++++++++++++++++++++++++++ 2 files changed, 443 insertions(+) create mode 100644 scripts/exdschema_wrapper.py create mode 100644 scripts/lumina_wrapper.py diff --git a/scripts/exdschema_wrapper.py b/scripts/exdschema_wrapper.py new file mode 100644 index 0000000000..0a1cc5b40c --- /dev/null +++ b/scripts/exdschema_wrapper.py @@ -0,0 +1,244 @@ +from urllib.request import urlopen +from zipfile import ZipFile +from io import BytesIO +from json import load as jsonload +from dataclasses import dataclass +from yaml import load as yamlload +from typing import TypeAlias, Union +from lumina_wrapper import ExcelColumnDefinition, ExcelColumnDataType +from structs_schema import ( + DefinedStruct, + DefinedStructEnum, + DefinedStructFixedField, + DefinedStructField, + DefinedStructExport, + Fields, +) + +try: + from yaml import CSafeLoader as Loader +except ImportError: + from yaml import SafeLoader as Loader + +exd_namespace = "Component::Exd::Sheets" + +ExdSchemaYaml: TypeAlias = list[dict[str, Union[str, int, "ExdSchemaYaml"]]] + + +@dataclass +class Definition: + name: str + comment: str | None + + def size(self) -> int: + return 1 + + +@dataclass +class RepeatDefinition(Definition): + count: int + fields: "DefinitionFields" + + def size(self) -> int: + return len(self.fields) * self.count + + +DefinitionFields: TypeAlias = list[Union[Definition, RepeatDefinition]] + + +def get_exdschema_versions(): + with urlopen("https://api.github.com/repos/xivdev/exdschema/branches") as response: + vers: list[str] = [] + for branch in jsonload(response): + vers.append(branch["name"]) + return vers + + +def process_exdschema_definition(fields: ExdSchemaYaml): + definitions: DefinitionFields = [] + for field in fields: + if "type" in field and field["type"] == "array": + definitions.append( + RepeatDefinition( + field["name"] if "name" in field else "", + field["comment"] if "comment" in field else None, + field["count"], + ( + process_exdschema_definition(field["fields"]) + if "fields" in field + else [] + ), + ) + ) + elif "name" in field: + definitions.append( + Definition( + field["name"], field["comment"] if "comment" in field else None + ) + ) + return definitions + + +def get_exdschema_data(version: str): + with urlopen( + f"https://github.com/xivdev/EXDSchema/archive/refs/heads/{version}.zip" + ) as response: + with ZipFile(BytesIO(response.read())) as z: + exd_schema_map: dict[str, DefinitionFields] = {} + for file in z.namelist(): + if file.endswith(".yml") and ".github" not in file: + yaml_data = yamlload(z.read(file), Loader) + exd_schema_map[file.rsplit(".", 1)[0].rsplit("/")[1]] = ( + process_exdschema_definition(yaml_data["fields"]) + ) + return exd_schema_map + + +def get_size_from_string(type: str) -> int: + if type == "size8_st" or type == "size8_t": + return 1 + if type == "size16_st" or type == "size16_t": + return 2 + if type == "size32_st" or type == "size32_t" or type == "float": + return 4 + if type == "size64_st" or type == "size64_t" or type == "double": + return 8 + return 0 + + +def find_struct(structs: list[DefinedStruct], name: str): + for struct in structs: + if struct.name == name: + return struct + return None + + +def create_struct_from_header_and_schema( + header: list[ExcelColumnDefinition], + schema: DefinitionFields, + header_index_offset: int = 0, + schema_name: str = "", +): + fields: Fields = [] + enums: list[DefinedStructEnum] = [] + enum_field_map: dict[int, int] = {} + structs: list[DefinedStruct] = [] + for index in range(len(schema)): + header_index = sum(f.size() for f in schema[:index]) + header_index_offset + definition = header[header_index] + field = schema[index] + + if isinstance(field, RepeatDefinition): + exported = create_struct_from_header_and_schema( + header, field.fields, header_index, field.name + ) + if len(exported.structs) == 0: + base_type = definition.get_base_type_string() + fields.append( + DefinedStructFixedField( + field.name, + base_type, + definition.offset, + False, + field.count, + False, + ) + ) + pass + else: + fields.append( + DefinedStructFixedField( + field.name, + exported.structs[-1].name, + definition.offset, + False, + field.count, + False, + ) + ) + structs.extend(exported.structs) + enums.extend(exported.enums) + pass + else: + if definition.is_packed_bool(): + if definition.offset in enum_field_map: + enums[enum_field_map[definition.offset]].values[field.name] = ( + definition.type - ExcelColumnDataType.PackedBool0 + ) + pass + else: + enum_index = len(enums) + enum_field_map[definition.offset] = enum_index + enum_type = f"{schema_name}PackedBool{definition.offset:X}" + enum_name = f"{exd_namespace}::{enum_type}" + enums.append( + DefinedStructEnum( + enum_name, + enum_type, + exd_namespace, + definition.get_base_type_string(), + True, + { + field.name: definition.type + - ExcelColumnDataType.PackedBool0 + }, + ) + ) + fields.append( + DefinedStructField( + f"PackedBool{definition.offset:X}", + enum_name, + definition.offset, + False, + ) + ) + pass + else: + fields.append( + DefinedStructField( + field.name, + definition.get_base_type_string(), + definition.offset, + False, + ) + ) + pass + + size: int = 0 + for field in fields: + if isinstance(field, DefinedStructFixedField): + struct = find_struct(structs, field.type) + if struct == None: + field_size = get_size_from_string(field.type) + if field_size == 0: + field_size = 1 + size += field_size + else: + size += struct.size * field.size + else: + field_size = get_size_from_string(field.type) + if field_size == 0: + field_size = 1 + size += field_size + + size = (size + 3) & ~3 + + if len(fields) != 0: + structs.append( + DefinedStruct( + f"{exd_namespace}::{schema_name}", + schema_name, + exd_namespace, + fields, + size, + None, + None, + [], + False, + None, + None, + [], + ) + ) + + return DefinedStructExport(enums, structs) diff --git a/scripts/lumina_wrapper.py b/scripts/lumina_wrapper.py new file mode 100644 index 0000000000..1fafe5bdb2 --- /dev/null +++ b/scripts/lumina_wrapper.py @@ -0,0 +1,199 @@ +from urllib.request import urlopen +from zipfile import ZipFile +from io import BytesIO +from os.path import join, basename, abspath +from os import getenv, getcwd, makedirs, listdir +from sys import platform, path +from shutil import rmtree +from xml.dom import pulldom +from pythonnet import load as dotnetload +from json import load as jsonload +from enum import IntEnum +from dataclasses import dataclass + +dotnetload("coreclr") +import clr + +tempdirname = join(getcwd(), "luminatmp") +try: + rmtree(tempdirname) +except: + pass # ignore error +finally: + makedirs(tempdirname) + +if platform.startswith("linux"): + launcher_ini = join(getenv("HOME"), ".xlcore", "launcher.ini") + game_path = None + + f = open(launcher_ini, "r") + try: + for line in f: + if line.startswith("GamePath="): + game_path = line.split("=", 1)[1].strip() + break + finally: + f.close() + + if not game_path: + raise ValueError("GamePath key not found in {0}".format(launcher_ini)) +else: + f = open(join(getenv("APPDATA"), "XIVLauncher", "launcherConfigV3.json"), "r") + try: + config = jsonload(f) + finally: + f.close() + game_path = config["GamePath"] + + +class ExcelColumnDataType(IntEnum): + String = 0x0 + Bool = 0x1 + Int8 = 0x2 + UInt8 = 0x3 + Int16 = 0x4 + UInt16 = 0x5 + Int32 = 0x6 + UInt32 = 0x7 + # unused? + Unk = 0x8 + Float32 = 0x9 + Int64 = 0xA + UInt64 = 0xB + # unused? + Unk2 = 0xC + # 0 is read like data & 1, 1 is like data & 2, 2 = data & 4, etc... + PackedBool0 = 0x19 + PackedBool1 = 0x1A + PackedBool2 = 0x1B + PackedBool3 = 0x1C + PackedBool4 = 0x1D + PackedBool5 = 0x1E + PackedBool6 = 0x1F + PackedBool7 = 0x20 + + +@dataclass +class ExcelColumnDefinition: + type: ExcelColumnDataType + offset: int + + def is_packed_bool(self): + return self.type >= ExcelColumnDataType.PackedBool0 + + def get_base_type_string(self): + match self.type: + case ExcelColumnDataType.Bool | ExcelColumnDataType.UInt8: + return "size8_t" + case ExcelColumnDataType.Int8: + return "size8_st" + case ExcelColumnDataType.UInt16: + return "size16_t" + case ExcelColumnDataType.Int16: + return "size16_st" + case ExcelColumnDataType.String | ExcelColumnDataType.UInt32: + return "size32_t" + case ExcelColumnDataType.Int32: + return "size32_st" + case ExcelColumnDataType.Float32: + return "float" + case ExcelColumnDataType.Int64: + return "size64_st" + case ExcelColumnDataType.UInt64: + return "size64_t" + case _ if self.is_packed_bool(): + return "size8_t" + case _: + raise ValueError(f"Unhandled type: {self.type}") + + def __lt__(self, other): + return self.offset < other.offset + + +def get_nugpkg_and_extract(package: str, version: str = ""): + with urlopen( + f"https://www.nuget.org/api/v2/package/{package}/{version}" + ) as response: + nupkg = response.read() + with ZipFile(BytesIO(nupkg)) as z: + for file in z.namelist(): + file_path = join(tempdirname, basename(file)) + with open(file_path, "wb") as f: + f.write(z.read(file)) + + +def get_nugpkg_deps(nuspec_path: str) -> list[tuple[str, str]]: + deps = [] + stream = pulldom.parse(nuspec_path) + for event, node in stream: + if event == "START_ELEMENT" and node.nodeName == "dependency": + stream.expandNode(node) + deps.append((node.getAttribute("id"), node.getAttribute("version"))) + return deps + + +def get_nugpkg(package: str, version: str = ""): + ver_str = f"version {version}" if version != "" else "latest version" + print(f"Requesting nugpkg {package} with {ver_str}") + get_nugpkg_and_extract(package, version) + return get_nugpkg_deps(join(tempdirname, f"{package}.nuspec")) + + +def get_excel_header_files() -> dict[str, list[ExcelColumnDefinition]]: + deps = get_nugpkg("Lumina") + deps_gotten: dict[str, str] = {} + + while len(deps) > 0: + local_deps = [] + for package, version in deps: + if package not in deps_gotten or deps_gotten[package] != version: + local_deps.extend(get_nugpkg(package, version)) + deps_gotten[package] = version + deps = local_deps + + path.append(tempdirname) + + from System.Runtime.Loader import AssemblyLoadContext # type: ignore + from System import Activator # type: ignore + + ctx = AssemblyLoadContext("temp", isCollectible=True) + assemblies: dict = {} + + for file in listdir(tempdirname): + if file.endswith(".dll"): + assemblies[file] = ctx.LoadFromAssemblyPath( + abspath(join(tempdirname, file)) + ) + + assembly = assemblies["Lumina.dll"] + game_data_type = assembly.GetType("Lumina.GameData") + game_data_instance = Activator.CreateInstance( + game_data_type, [join(game_path, "game", "sqpack"), None] + ) + + excel_list_type = assembly.GetType("Lumina.Data.Files.Excel.ExcelListFile") + excel_header_type = assembly.GetType("Lumina.Data.Files.Excel.ExcelHeaderFile") + + root_exl = game_data_instance.GetFile[excel_list_type]("exd/root.exl") + + excel_files: dict[int, str] = {} + excel_header_files: dict[str, list[ExcelColumnDefinition]] = {} + + for kvp in root_exl.ExdMap: + key = kvp.Key + value = kvp.Value + if value != -1: + excel_files[value] = key + + if value != -1: + defs: list[ExcelColumnDefinition] = [] + header = game_data_instance.GetFile[excel_header_type](f"exd/{key}.exh") + for column_definition in header.ColumnDefinitions: + defs.append( + ExcelColumnDefinition( + int(column_definition.Type), column_definition.Offset + ) + ) + excel_header_files[key] = sorted(defs) + + return excel_header_files From 64497ccb249e94bd8fe6705a0a1a9e38b776b506 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Mon, 4 May 2026 01:26:31 +0200 Subject: [PATCH 10/95] add test file for now this will only exists while working on the new scripts --- scripts/test_execute.py | 207 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 207 insertions(+) create mode 100644 scripts/test_execute.py diff --git a/scripts/test_execute.py b/scripts/test_execute.py new file mode 100644 index 0000000000..ad60c3af51 --- /dev/null +++ b/scripts/test_execute.py @@ -0,0 +1,207 @@ +from lumina_wrapper import get_excel_header_files +from exdschema_wrapper import create_struct_from_header_and_schema, get_exdschema_data +from structs_schema import ( + DefinedStruct, + DefinedStructFixedField, + DefinedStructField, +) + + +class SrcInterface(object): + def get_idc_type_from_size(self, size: int, offset=0) -> int: + if offset == 0: + offset = size + if offset % 8 == 0 and size >= 8: + return 4 + elif offset % 4 == 0 and size >= 4: + return 3 + elif offset % 2 == 0 and size >= 2: + return 2 + else: + return 1 + + def get_size_from_idc_type(self, type: int) -> int: + if type == 1: + return 1 + elif type == 2: + return 2 + elif type == 3: + return 4 + elif type == 4: + return 8 + elif type == 5: + return 4 + elif type == 6: + return 8 + else: + return 0 + + def get_string_from_idc_type_and_sign(self, type: int, signed: bool = False) -> str: + if type == 1: + if signed: + return "size8_st" + else: + return "size8_t" + elif type == 2: + if signed: + return "size16_st" + else: + return "size16_t" + elif type == 3: + if signed: + return "size32_st" + else: + return "size32_t" + elif type == 4: + if signed: + return "size32_st" + else: + return "size64_t" + elif type == 5: + return "float" + elif type == 6: + return "double" + else: + return "" + + def get_size_from_string(self, type: str) -> int: + """ + Gets the size of a base type string. + """ + if type == "size8_st" or type == "size8_t": + return 1 + if type == "size16_st" or type == "size16_t": + return 2 + if type == "size32_st" or type == "size32_t" or type == "float": + return 4 + if ( + type == "size64_st" + or type == "size64_t" + or type == "double" + or type.endswith("*") + ): + return 8 + return 0 + + def get_string_define_and_size(self, field: DefinedStructField | DefinedStructFixedField, namespace: str, struct_lookup: dict[str, int], enum_lookup: dict[str, int]) -> tuple[str, int]: + field_type = field.type + if field_type.startswith(namespace): + field_type = field_type.removeprefix(f"{namespace}::") + field_name = field.name + if field.type in struct_lookup: + return (f" {field_type} {field_name};\n", struct_lookup[field.type]) + elif field.type in enum_lookup: + return (f" {field_type} {field_name};\n", enum_lookup[field.type]) + else: + return (f" {field_type} {field_name};\n", self.get_size_from_string(field_type)) + + + def build_struct_string( + self, + struct: DefinedStruct, + struct_lookup: dict[str, int], + enum_lookup: dict[str, int], + ) -> tuple[str, str, list[str]]: + """Builds the srclang definition required to parse the object. + + Args: + struct (DefinedStruct): The struct data to build""" + struct_string_forward: str = "" + struct_string_define: str = "" + struct_string_vtables: list[str] = [] + struct_name = struct.type + + struct_string_forward += f"struct {struct_name};\n" + if struct.virtual_functions: + struct_string_forward += f"struct {struct_name}_vtbl;\n" + struct_string_vtables.append(f"{struct_name}_vtbl") + + struct_string_define += f"struct {struct_name}" + + prev_size = 0 + contiguous_fields = True + for field in struct.fields: + offset = field.offset + + while offset > prev_size: + contiguous_fields = False + fill_size = offset - prev_size + if self.full_padding: + flag = self.get_idc_type_from_size(prev_size) + size = self.get_size_from_idc_type(flag) + if size > fill_size: + flag = self.get_idc_type_from_size(fill_size, prev_size) + size = self.get_size_from_idc_type(flag) + + struct_string_define += f" {self.get_string_from_idc_type_and_sign(flag)} field_{prev_size:X};\n" + else: + struct_string_define += f" char field_{prev_size:X}[{fill_size}];\n" + + prev_size += size + + field_is_base = field.base and contiguous_fields + if contiguous_fields != field_is_base: + struct_string_define += " {\n" + contiguous_fields = False + field_name = ( + field.name if not field_is_base else "baseclass_{0:X}".format(offset) + ) + field_type = field.type + if isinstance(field, DefinedStructFixedField): + field_define, field_define_size = self.get_string_define_and_size(field, struct.namespace, struct_lookup, enum_lookup) + struct_string_define += f"{field_define[:-2]}[{field.size}];\n" + prev_size += field_define_size * field.size + elif field_type == "__fastcall": + struct_string_define += ( + f" {field.return_type} ({field_type} *{field_name})(" + ) + for param in field.parameters: + struct_string_define += param.type + " " + param.name + ", " + struct_string_define = struct_string_define[:-2] + ");\n" + prev_size += 8 + else: + field_define, field_define_size = self.get_string_define_and_size(field, struct.namespace, struct_lookup, enum_lookup) + struct_string_define += field_define + prev_size += field_define_size + + if struct.size is not None and struct.size != 0: + while struct.size > prev_size: + fill_size = struct.size - prev_size + if self.full_padding: + flag = self.get_idc_type_from_size(prev_size) + size = self.get_size_from_idc_type(flag) + if size > fill_size: + flag = self.get_idc_type_from_size(fill_size, prev_size) + size = self.get_size_from_idc_type(flag) + + struct_string_define += f" {self.get_string_from_idc_type_and_sign(flag)} field_{prev_size:X};\n" + else: + struct_string_define += f" char field_{prev_size:X}[{fill_size}];\n" + + prev_size += size + + struct_string_define += "};\n" + + return (struct_string_forward, struct_string_define, struct_string_vtables) + +excel_header_files = get_excel_header_files() +exdschema = get_exdschema_data("latest") + +struct_export = create_struct_from_header_and_schema( + excel_header_files["GFATE"], exdschema["GFATE"], 0, "GFATE" +) + +src = SrcInterface() +src.full_padding = True + +struct_lookup = {struct.name: struct.size for struct in struct_export.structs} +enum_lookup = {enum.name: 1 for enum in struct_export.enums} + +forwards: list[str] = [] +defines: list[str] = [] +for struct in struct_export.structs: + forward, define, _ = src.build_struct_string(struct, struct_lookup, enum_lookup) + forwards.append(forward) + defines.append(define) + +print("".join(forwards + defines)) From 67a67947268b70b1ffbe2ade2e6317f6cbe73238 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Mon, 4 May 2026 01:27:15 +0200 Subject: [PATCH 11/95] update ignore file just so people don't accidentally commit runtime/testing files --- .gitignore | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index a0cda0c0db..3af96880f5 100644 --- a/.gitignore +++ b/.gitignore @@ -4,9 +4,6 @@ ida/uld_names.txt .vscode/ .idea/ -# CompatChecker test files -tests/ - ## Ignore Visual Studio temporary files, build results, and ## files generated by popular Visual Studio add-ons. ## @@ -435,3 +432,9 @@ FodyWeavers.xsd *.msix *.msm *.msp + + +# test folders +scripts/luminatmp/ +tests/ +luminatmp/ \ No newline at end of file From f0da0a1b9da9f786550def9565adfb2dbea104b5 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Thu, 7 May 2026 00:35:11 +0200 Subject: [PATCH 12/95] move src writer out from ida to general file --- scripts/ida/ida_wrapper.py | 291 +++++++++++++++++++++------------ scripts/indent_writer.py | 32 ++++ scripts/src_wrapper.py | 324 +++++++++++++++++++++++++++++++++++++ 3 files changed, 543 insertions(+), 104 deletions(-) create mode 100644 scripts/indent_writer.py create mode 100644 scripts/src_wrapper.py diff --git a/scripts/ida/ida_wrapper.py b/scripts/ida/ida_wrapper.py index f8a3115751..95eb49c41c 100644 --- a/scripts/ida/ida_wrapper.py +++ b/scripts/ida/ida_wrapper.py @@ -2,11 +2,9 @@ import idc import ida_bytes import ida_nalt -import ida_kernwin import ida_search import ida_ida import ida_typeinf -import ida_hexrays import ida_name import ida_funcs import ida_srclang @@ -17,10 +15,144 @@ # For more information about TERR_ constants, see: # https://docs.hex-rays.com/developer-guide/idapython/idapython-porting-guide-ida-9#type-information-error-codes + def _get_ida_major() -> int: return idaapi.IDA_SDK_VERSION // 100 -class SrcInterface(object): +class BaseIdaInterface(object): + @abstractmethod + def get_struct_id(self, name): + pass + + @abstractmethod + def get_enum_id(self, name): + pass + + @abstractmethod + def delete_enum_members(self, eid: int): + """Remove all enum members + + Args: + eid (int): The id of the enum + """ + pass + + def enum_exists(self, name: str): + return self.get_enum_id(name) != idaapi.BADADDR + + def clean_name(self, name: str): + """Clean a name + + Args: + name (str): The name + + Returns: + str: The cleaned name + """ + return name + + def clean_struct_name(self, name: str): + """Clean a struct name + + Args: + name (str): The struct name + + Returns: + str: The cleaned struct name + """ + + if name == "Tm": + return "tm" # tm is a keyword in IDA for the time struct but C# exports it as Tm + return ( + name.replace(" ", "") + .replace("unsigned", "u") + .replace("__int64", "long") + .replace("__int32", "int") + .replace("__int16", "short") + .replace("__int8", "char") + ) + + def get_named_type(self, name: str): + """Retrieve a tinfo_t from the named type. + + Args: + name (str): Name of the type. + """ + + tinfo = ida_typeinf.tinfo_t() + clean_name = self.clean_struct_name(name) + if ( + self.get_struct_id(clean_name) != idaapi.BADADDR + or self.get_enum_id(clean_name) != idaapi.BADADDR + ): + if not tinfo.get_named_type(idaapi.get_idati(), clean_name): + raise ValueError("{0} not found in IDA database".format(clean_name)) + return tinfo + + if name == "void": + idaapi.parse_decl( + tinfo, idaapi.get_idati(), "void (__fastcall)();", idaapi.PT_SIL + ) + return tinfo.get_rettype() + + terminated = name + ";" + idaapi.parse_decl(tinfo, idaapi.get_idati(), terminated, idaapi.PT_SIL) + + tinfo_str = tinfo.dstr() + if tinfo_str == name or tinfo_str == clean_name: + return tinfo + + terminated = clean_name + ";" + idaapi.parse_decl(tinfo, idaapi.get_idati(), terminated, idaapi.PT_SIL) + + return tinfo + + def get_tinfo_from_type(self, raw_type: str, array_size=0): + """Retrieve a tinfo_t from a raw type string. + + Args: + raw_type (str): Raw type string. + array_size (int, optional): Size of the array. Defaults to 0. + """ + + type = raw_type.rstrip("*") + ptr_count = len(raw_type) - len(type) + + type_tinfo = self.get_named_type(type) + + ptr_tinfo = None + if ptr_count > 0: + for i in range(ptr_count): + ptr_tinfo = idaapi.tinfo_t() + if not ptr_tinfo.create_ptr(type_tinfo): + print("! failed to create pointer") + return None + type_tinfo = ptr_tinfo + else: + ptr_tinfo = type_tinfo + + if array_size > 0: + array_tinfo = idaapi.tinfo_t() + if not array_tinfo.create_array(ptr_tinfo, array_size): + print("! failed to create array") + return None + + ptr_tinfo = array_tinfo + + return ptr_tinfo + + def select_parser(self): + """Sets the parser used for building struct data.""" + ida_srclang.set_parser_argv( + "clang", + "-x c++ -target x86_64-pc-windows-msvc -fms-compatibility -fms-extensions -fdelayed-template-parsing", + ) + + def parse_string(self, decl: str, vtables: list[str]): + """""" + ida_srclang.parse_decls_with_parser("clang", None, decl, False) + for vtable in vtables: + self.mark_as_vtable(vtable) def mark_as_vtable(vtbl_name: str) -> str | None: """ Mark a struct as a vtable (TAUDT_VFTABLE). @@ -34,7 +166,9 @@ def mark_as_vtable(vtbl_name: str) -> str | None: return f"Type '{vtbl_name}' not found in type library" if not tif.is_struct(): - actual = "union" if tif.is_union() else "enum" if tif.is_enum() else "unknown" + actual = ( + "union" if tif.is_union() else "enum" if tif.is_enum() else "unknown" + ) return f"'{vtbl_name}' is not a struct (got {actual})" udt = ida_typeinf.udt_type_data_t() @@ -47,15 +181,15 @@ def mark_as_vtable(vtbl_name: str) -> str | None: udt.taudt_bits |= ida_typeinf.TAUDT_VFTABLE if not tif.create_udt(udt, ida_typeinf.BTF_STRUCT): - return f"Failed to reconstruct UDT for '{vtbl_name}' after setting vtable flag" + return ( + f"Failed to reconstruct UDT for '{vtbl_name}' after setting vtable flag" + ) if _get_ida_major() >= 9: result = tif.set_named_type(None, vtbl_name) else: result = tif.set_named_type( - ida_typeinf.get_idati(), - vtbl_name, - ida_typeinf.NTF_REPLACE + ida_typeinf.get_idati(), vtbl_name, ida_typeinf.NTF_REPLACE ) if not result: @@ -148,71 +282,14 @@ def get_size_from_idc_type(self, type: int) -> int: return 8 else: return 0 - - def get_string_from_idc_type_and_sign(self, type: int, signed: bool = False) -> str: - if type == ida_bytes.byte_flag(): - if signed: - return "size8_st" - else: - return "size8_t" - elif type == ida_bytes.word_flag(): - if signed: - return "size16_st" - else: - return "size16_t" - elif type == ida_bytes.dword_flag(): - if signed: - return "size32_st" - else: - return "size32_t" - elif type == ida_bytes.qword_flag(): - if signed: - return "size32_st" - else: - return "size64_t" - elif type == ida_bytes.float_flag(): - return "float" - elif type == ida_bytes.double_flag(): - return "double" - else: - return "" - - def get_size_from_string(self, type: str) -> int: - """ - Gets the size of a base type string. - """ - if type == "size8_st" or type == "size8_t": - return 1 - if type == "size16_st" or type == "size16_t": - return 2 - if type == "size32_st" or type == "size32_t" or type == "float": - return 4 - if type == "size64_st" or type == "size64_t" or type == "double" or type.endswith('*'): - return 8 - return 0 - - def is_signed(self, type: str) -> bool: - if ( - type == "__int8" - or type == "__int16" - or type == "__int32" - or type == "__int64" - or type == "int" - ): - return True - else: - return False def get_size_from_ida_type(self, type: str) -> int: return self.get_size_from_idc_type(self.get_idc_type_from_ida_type(type)) - def select_parser(self): - """Sets the parser used for building struct data. + def set_func_name(self, ea, name, cmt): + ida_name.set_name(ea, name) + ida_bytes.set_cmt(ea, cmt, 0) - Args: - lang (srclang_t): The language the parser should use one of (SRCLANG_C, SRCLANG_CPP, SRCLANG_OBJC, SRCLANG_SWIFT, SRCLANG_GO) - """ - ida_srclang.set_parser_argv("clang", "-x c++ -target x86_64-pc-win32") def build_struct_string(self, struct: DefinedStruct, struct_lookup: dict[str, int], enum_lookup: dict[str, str]) -> tuple[str, str, list[str]]: """Builds the srclang definition required to parse the object. @@ -477,11 +554,12 @@ def rename_struct(self, sid: int, new_name: str): if _get_ida_major() < 9: - import ida_struct # pyright: ignore[reportMissingImports] - import ida_enum # pyright: ignore[reportMissingImports] + import ida_struct # pyright: ignore[reportMissingImports] + import ida_enum # pyright: ignore[reportMissingImports] else: print("Using IDA 9+ API") + class IdaInterface(BaseIdaInterface): # This is only for IDA 7 and 8 due to a change in the API for IDA 9 if 7 <= _get_ida_major() < 9: @@ -765,7 +843,7 @@ def set_struct_member_info( offset: int, tif: ida_typeinf.tinfo_t, flag: int = 0, - is_string: bool = False + is_string: bool = False, ): """Set the info of a struct member @@ -838,7 +916,7 @@ def delete_enum_members(self, eid: int): eid (int): The id of the enum """ - masks = [ -1 ] + masks = [-1] if idc.is_bf(eid): mask = idc.get_first_bmask(eid) while mask != idaapi.DEFMASK: @@ -858,9 +936,9 @@ def delete_enum_members(self, eid: int): eid, ida_enum.get_enum_member_value(cid), ida_enum.get_enum_member_serial(cid), - ida_enum.get_enum_member_bmask(cid) + ida_enum.get_enum_member_bmask(cid), ) - + def create_enum(self, name: str) -> int: """Create an enum by its name @@ -904,12 +982,12 @@ def get_enum_default_mask(self, eid: int): if mask == ida_enum.DEFMASK: mask >>= 1 - + return mask - + def get_enum_name(self, eid: int): return idc.get_enum_name(eid) - + def get_enum_bf(self, eid: int): return ida_enum.is_bf(eid) @@ -937,7 +1015,7 @@ def add_enum_member(self, eid: int, name: str, value: int, mask: int = -1): mask = self.get_enum_default_mask(eid) ec = ida_enum.add_enum_member(eid, name, value, mask) - + if ec == ida_enum.ENUM_MEMBER_ERROR_MASK: ida_enum.add_enum_member(eid, name, value) @@ -992,7 +1070,7 @@ def get_struct_opinfo_from_type(self, raw_type: str): """ opinf = ida_nalt.opinfo_t() opinf.tid = ida_typeinf.get_named_type_tid(raw_type) - + return opinf def get_enum_opinfo_from_type(self, raw_type: str): @@ -1006,7 +1084,7 @@ def get_enum_opinfo_from_type(self, raw_type: str): """ opinf = ida_nalt.opinfo_t() opinf.ec.tid = idc.get_enum(raw_type) - + return opinf def search_binary(self, ea: int, pattern: str, flag: int): @@ -1122,10 +1200,10 @@ def create_struct_type(self, name: str, union: bool = False) -> int: tinfo = ida_typeinf.tinfo_t() if not tinfo.create_udt(udt): return idc.BADADDR - + if tinfo.set_named_type(None, name) != ida_typeinf.TERR_OK: return idc.BADADDR - + return tinfo.get_tid() def get_struct_id(self, name: str) -> int: @@ -1195,9 +1273,9 @@ def create_struct_member( nbytes = tinfo.get_size() else: raise ValueError("Cannot find type with tid {0}".format(typeid.tid)) - + typeid = typeid.tid - + ec = idc.add_struc_member(sid.get_tid(), name, offset, flag, typeid, nbytes) if ec != ida_typeinf.TERR_OK: return False @@ -1219,7 +1297,7 @@ def remove_struct_member(self, sid: int, name: str) -> bool: idx = tinfo.find_udm(name=name) if idx == -1: return True - + return tinfo.del_udm(idx) except: return ida_typeinf.TERR_BAD_ARG @@ -1237,13 +1315,13 @@ def remove_struct_members(self, sid: int) -> int: tinfo = ida_typeinf.tinfo_t() if not tinfo.get_type_by_tid(sid): return -1 - + size = tinfo.get_udt_nmembers() tinfo.del_udms(0, size) return size except: return -1 - + def get_struct_member( self, sid: ida_typeinf.tinfo_t, offset: int ) -> ida_typeinf.udm_t: @@ -1281,7 +1359,7 @@ def set_struct_member_info( offset: int, tif: ida_typeinf.tinfo_t, flag: int = 0, - is_string: bool = False + is_string: bool = False, ): """Set the info of a struct member @@ -1296,8 +1374,10 @@ def set_struct_member_info( tinfo_code_t: See ida_typeinfo.TERR_ constants """ if offset != 0: - raise ValueError("IDA 9+ does not support offset != 0 in set_struct_member_info") - + raise ValueError( + "IDA 9+ does not support offset != 0 in set_struct_member_info" + ) + memberIdx, _ = sid.get_udm_by_offset(member.offset) return sid.set_udm_type(index=memberIdx, tif=tif) @@ -1323,7 +1403,7 @@ def get_enum_id(self, name: str): Returns: int: The id of the enum """ - + return idc.get_enum(name) def remove_enum_member(self, eid: int, value: str, name: str): @@ -1352,23 +1432,23 @@ def delete_enum_members(self, eid: int): """ tif = ida_typeinf.tinfo_t() if not tif.get_type_by_tid(eid): - raise RuntimeError(f'Failed to get tinfo_t for enum {eid}') + raise RuntimeError(f"Failed to get tinfo_t for enum {eid}") eti = ida_typeinf.enum_type_data_t() if not tif.get_enum_details(eti): - raise RuntimeError(f'Failed to get enum details for enum id {eid}') - + raise RuntimeError(f"Failed to get enum details for enum id {eid}") + # must clear values before groups values = [] - for (idx, _, _) in eti.all_constants(): + for idx, _, _ in eti.all_constants(): values.append(eti[idx].name) for name in values: tif.del_edm(name) groups = [] - for (idx, _) in eti.all_groups(): + for idx, _ in eti.all_groups(): groups.append(eti[idx].name) for name in groups: @@ -1416,7 +1496,7 @@ def get_enum_default_mask(self, eid: int): # from the default group for 64-bit enums. if mask == ida_typeinf.DEFMASK64: mask >>= 1 - + return mask def set_enum_flag(self, eid: int, flag: int): @@ -1427,13 +1507,13 @@ def set_enum_flag(self, eid: int, flag: int): flag (int): The flag to set """ idc.set_enum_flag(eid, flag) - + def get_enum_bf(self, eid: int): return idc.is_bf(eid) - + def get_enum_name(self, eid: int): return idc.get_enum_name(eid) - + def get_enum_bitmask_field(self, eid: int): name = self.get_enum_name(eid) bmask = self.get_enum_default_mask(eid) @@ -1443,10 +1523,10 @@ def get_enum_bitmask_field(self, eid: int): def set_enum_as_bf(self, eid: int): if idc.is_bf(eid): return - + idc.set_enum_bf(eid, True) - (name, bmask) = self.get_enum_bitmask_field(eid) + name, bmask = self.get_enum_bitmask_field(eid) # this shouldn't happen under normal circumstances if idc.get_enum_member_by_name(f"{name}_Mask") != idaapi.BADADDR: @@ -1472,10 +1552,12 @@ def add_enum_member(self, eid: int, name: str, value: int, mask: int = -1): processedName = name while idc.get_enum_member_by_name(processedName) != idaapi.BADADDR: if retries >= 3: - raise RuntimeError(f"Error: too many duplicate enum member names for '{name}'") + raise RuntimeError( + f"Error: too many duplicate enum member names for '{name}'" + ) retries += 1 processedName += "_" - + # IDA 9.0 - 9.2 vary in how they handle errors. # IDA 9.0 will return a TERR_ code, while IDA 9.2 will raise an exception. # This will effectively convert the TERR_ to an equivalent ValueError for 9.0. @@ -1498,5 +1580,6 @@ def get_enum_flag(self): int: The flag for an enum data type """ return ida_bytes.enum_flag() + else: raise RuntimeError("Unsupported IDA version") diff --git a/scripts/indent_writer.py b/scripts/indent_writer.py new file mode 100644 index 0000000000..178b3b6cf2 --- /dev/null +++ b/scripts/indent_writer.py @@ -0,0 +1,32 @@ +from dataclasses import dataclass + +@dataclass +class IndentWriterValue: + indent_level: int + line: str + + def __str__(self): + return " " * (self.indent_level * 2) + self.line + +class IndentWriter(object): + def __init__(self): + self.indent_level: int = 0 + self.lines: list[IndentWriterValue] = [] + + def indent(self): + self.indent_level += 1 + + def unindent(self): + self.indent_level -= 1 + if (self.indent_level < 0): + self.indent_level = 0 + + def append(self, line: str): + self.lines.append(IndentWriterValue(self.indent_level, line)) + + def __iadd__(self, line: str): + self.append(line) + return self + + def __str__(self): + return "\n".join([str(line) for line in self.lines]) \ No newline at end of file diff --git a/scripts/src_wrapper.py b/scripts/src_wrapper.py new file mode 100644 index 0000000000..267d62d337 --- /dev/null +++ b/scripts/src_wrapper.py @@ -0,0 +1,324 @@ +from structs_schema import ( + DefinedStruct, + DefinedStructFixedField, + DefinedStructField, + DefinedStructExport, + DefinedStructEnum, + DefinedStructVFunc, +) +from indent_writer import IndentWriter + +class SrcInterface(object): + full_padding: bool = True + + def get_type_and_type_size_from_gap(self, gap: int, offset: int = 0) -> tuple[str, int]: + if offset == 0: + offset = gap + if offset % 8 == 0 and gap >= 8: + return ("uint64_t", 8) + elif offset % 4 == 0 and gap >= 4: + return ("uint32_t", 4) + elif offset % 2 == 0 and gap >= 2: + return ("uint16_t", 2) + else: + return ("uint8_t", 1) + + def get_size_from_string(self, type: str) -> int: + """ + Gets the int of a base type string. + """ + if type == "int8_t" or type == "uint8_t" or type == "bool": + return 1 + if type == "int16_t" or type == "uint16_t": + return 2 + if type == "int32_t" or type == "uint32_t" or type == "float": + return 4 + if ( + type == "int64_t" + or type == "uint64_t" + or type == "double" + or type.endswith("*") + ): + return 8 + return 0 + + def get_string_define_and_size(self, field: DefinedStructField | DefinedStructFixedField, namespace: str, struct_lookup: dict[str, int], enum_lookup: dict[str, int]) -> tuple[str, int]: + field_type = field.type + if field_type.startswith(namespace): + field_type = field_type.removeprefix(f"{namespace}::") + field_name = field.name + lookup_type = self.get_short_name(field.type) + if lookup_type in struct_lookup: + return (f"{field_type} {field_name};", struct_lookup[lookup_type]) + elif lookup_type in enum_lookup: + return (f"{field_type} {field_name};", enum_lookup[lookup_type]) + else: + return (f"{field_type} {field_name};", self.get_size_from_string(field_type)) + + def get_short_name(self, full_type: str) -> str: + """Extract the short name from a fully qualified type name.""" + return full_type.split("::")[-1] + + def build_vfunc_signature(self, vfunc: DefinedStructVFunc) -> str: + """Build a function pointer signature for a virtual function. + + Args: + vfunc: Virtual function definition + + Returns: + Function pointer signature string + """ + # Special case for Dtor + if vfunc.name == "Dtor": + # Find the struct name from context - this will be passed separately + return "DTOR" + + return_type = vfunc.return_type if vfunc.return_type else "void" + + # Build parameter list + param_list = "" + if vfunc.parameters: + params = [f"{param.type} {param.name}" for param in vfunc.parameters] + param_list = ", ".join(params) + + return f"{return_type} (__fastcall *{vfunc.name})({param_list});" + + def build_export_string( + self, + export: DefinedStructExport + ): + output = IndentWriter() + + struct_lookup = { + struct.type: struct.size for struct in export.structs if struct.size is not None + } + enum_lookup = {enum.type: self.get_size_from_string(enum.underlying) for enum in export.enums} + + namespace_enum_chunks: dict[str, list[IndentWriter]] = {} + namespace_forward_chunks: dict[str, list[IndentWriter]] = {} + namespace_define_chunks: dict[str, list[IndentWriter]] = {} + namespace_order: list[str] = [] + + def add_namespace(namespace: str): + if namespace not in namespace_enum_chunks: + namespace_enum_chunks[namespace] = [] + namespace_forward_chunks[namespace] = [] + namespace_define_chunks[namespace] = [] + namespace_order.append(namespace) + + for enum in export.enums: + add_namespace(enum.namespace) + namespace_enum_chunks[enum.namespace].append(self.build_enum_string(enum)) + + vtable_names: list[str] = [] + + for struct in export.structs: + add_namespace(struct.namespace) + forward, define, vtable, vtable_full_name = self.build_struct_string( + struct, struct_lookup, enum_lookup + ) + if vtable_full_name: + vtable_names.append(vtable_full_name) + namespace_forward_chunks[struct.namespace].append(forward) + namespace_define_chunks[struct.namespace].append(define) + namespace_define_chunks[struct.namespace].append(vtable) + + def append_chunk(chunk: IndentWriter | str): + chunk_text = str(chunk) + for line in chunk_text.split("\n"): + if not line.strip(): + continue + # preserve the chunk's indentation by measuring leading spaces + leading = len(line) - len(line.lstrip(" ")) + chunk_indent = leading // 2 + prev_indent = output.indent_level + output.indent_level = prev_indent + chunk_indent + output.append(line.lstrip(" ")) + output.indent_level = prev_indent + + def append_namespace_content(namespace: str): + ordered_chunks = [ + *namespace_enum_chunks[namespace], + *namespace_forward_chunks[namespace], + *namespace_define_chunks[namespace], + ] + for chunk in ordered_chunks: + append_chunk(chunk) + + # Build an ordered namespace tree so parent namespaces are emitted only once. + namespace_tree_children: dict[str, list[str]] = {"": []} + for namespace in namespace_order: + if namespace == "": + continue + + parent_path = "" + current_parts: list[str] = [] + for part in namespace.split("::"): + current_parts.append(part) + current_path = "::".join(current_parts) + if parent_path not in namespace_tree_children: + namespace_tree_children[parent_path] = [] + if current_path not in namespace_tree_children[parent_path]: + namespace_tree_children[parent_path].append(current_path) + if current_path not in namespace_tree_children: + namespace_tree_children[current_path] = [] + parent_path = current_path + + # Emit global namespace content first (if any), then recurse through namespace tree. + if "" in namespace_enum_chunks: + append_namespace_content("") + + def emit_namespace_subtree(parent_path: str): + for child_path in namespace_tree_children.get(parent_path, []): + child_name = child_path.split("::")[-1] + output.append(f"namespace {child_name} {{") + output.indent() + + if child_path in namespace_enum_chunks: + append_namespace_content(child_path) + + emit_namespace_subtree(child_path) + + output.unindent() + output.append("}") + + emit_namespace_subtree("") + + return (str(output), vtable_names) + + def build_enum_string( + self, + enum: DefinedStructEnum, + ): + output = IndentWriter() + + enum_name = self.get_short_name(enum.type) + output += f"enum {enum_name} : {enum.underlying} {{" + output.indent() + + for value_name, value in enum.values.items(): + output += f"{enum_name}_{value_name} = {value}," + + output.unindent() + output += "};" + + return output + + def get_struct_pad(self, fill_size: int, prev_size: int): + if self.full_padding: + type, size = self.get_type_and_type_size_from_gap(prev_size) + if size > fill_size: + type, size = self.get_type_and_type_size_from_gap(fill_size, prev_size) + + return (f"{type} field_{prev_size:X};", size) + else: + return (f"char field_{prev_size:X}[{fill_size}];", fill_size) + + def build_struct_string( + self, + struct: DefinedStruct, + struct_lookup: dict[str, int], + enum_lookup: dict[str, int], + ) -> tuple[IndentWriter, IndentWriter, IndentWriter, str | None]: + """Builds the C++ struct definition with namespace wrapping. + + Args: + struct (DefinedStruct): The struct data to build + + Returns: + tuple: (forward_declarations, struct_definition, vtable_definition, vtable_full_name or None) + """ + struct_string_forward: IndentWriter = IndentWriter() + struct_string_define: IndentWriter = IndentWriter() + struct_string_vtable: IndentWriter = IndentWriter() + + short_name = self.get_short_name(struct.type) + vtable_full_name: str | None = None + + # Forward declarations + struct_string_forward += f"struct __cppobj {short_name};" + if struct.virtual_functions: + struct_string_forward += f"struct __cppobj {short_name}_vtbl;" + + # Struct definition + struct_define_line = f"struct __cppobj {short_name}" + struct_extend_lines = [] + + base_fields = [field for field in struct.fields if field.base] + base_fields_used = [] + prev_size = 0 + for base_field in base_fields: + if base_field.offset == prev_size: + base_fields_used.append(base_field.type) + short_base_name = self.get_short_name(base_field.type) + struct_extend_lines.append(short_base_name) + prev_size += struct_lookup[base_field.type] + + struct_extend_line = "" + if len(struct_extend_lines) > 0: + struct_extend_line = " : " + ", ".join(struct_extend_lines) + + struct_string_define += f"{struct_define_line}{struct_extend_line} {{" + struct_string_define.indent() + + # Add __vtable member if this struct has virtual functions + if struct.virtual_functions: + struct_string_define += f"{short_name}_vtbl* __vtable;" + + for field in struct.fields: + if field.type in base_fields_used: + continue + + offset = field.offset + + while offset > prev_size: + fill_size = offset - prev_size + line, size = self.get_struct_pad(fill_size, prev_size) + struct_string_define += line + prev_size += size + + field_name = field.name + field_type = field.type + if isinstance(field, DefinedStructFixedField): + field_define, field_define_size = self.get_string_define_and_size(field, struct.namespace, struct_lookup, enum_lookup) + struct_string_define += f"{field_define[:-1]}[{field.size}];" + prev_size += field_define_size * field.size + elif field_type == "__fastcall": + struct_string_define += ( + f"{field.return_type} ({field_type} *{field_name})(" + ) + for param in field.parameters: + struct_string_define += param.type + " " + param.name + ", " + struct_string_define = struct_string_define[:-1] + ");" + prev_size += 8 + else: + field_define, field_define_size = self.get_string_define_and_size(field, struct.namespace, struct_lookup, enum_lookup) + struct_string_define += field_define + prev_size += field_define_size + + if struct.size is not None and struct.size != 0: + while struct.size > prev_size: + fill_size = offset - prev_size + line, size = self.get_struct_pad(fill_size, prev_size) + struct_string_define += line + prev_size += size + + struct_string_define.unindent() + struct_string_define += "};" + + # Virtual table struct definition + if struct.virtual_functions: + struct_string_vtable += f"struct __cppobj {short_name}_vtbl {{" + struct_string_vtable.indent() + for vfunc in struct.virtual_functions: + sig = self.build_vfunc_signature(vfunc) + # Special case for DTOR macro + if sig.startswith("DTOR"): + struct_string_vtable += f"DTOR({short_name})" + else: + struct_string_vtable += sig + struct_string_vtable.unindent() + struct_string_vtable += "};" + vtable_full_name = f"{struct.type}_vtbl" + + return (struct_string_forward, struct_string_define, struct_string_vtable, vtable_full_name) \ No newline at end of file From d4f4767a1da3a347f1e90a3a514c725a377c9a69 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Thu, 7 May 2026 00:38:11 +0200 Subject: [PATCH 13/95] update some schema definitions and functions --- scripts/exdschema_wrapper.py | 84 ++++++++++++++++--- scripts/lumina_wrapper.py | 29 ++++--- scripts/structs_schema.py | 152 ++++++++++++++++++++++++++++++++++- 3 files changed, 237 insertions(+), 28 deletions(-) diff --git a/scripts/exdschema_wrapper.py b/scripts/exdschema_wrapper.py index 0a1cc5b40c..be300eaec2 100644 --- a/scripts/exdschema_wrapper.py +++ b/scripts/exdschema_wrapper.py @@ -95,13 +95,21 @@ def get_exdschema_data(version: str): def get_size_from_string(type: str) -> int: - if type == "size8_st" or type == "size8_t": + """ + Gets the int of a base type string. + """ + if type == "int8_t" or type == "uint8_t" or type == "bool": return 1 - if type == "size16_st" or type == "size16_t": + if type == "int16_t" or type == "uint16_t": return 2 - if type == "size32_st" or type == "size32_t" or type == "float": + if type == "int32_t" or type == "uint32_t" or type == "float": return 4 - if type == "size64_st" or type == "size64_t" or type == "double": + if ( + type == "int64_t" + or type == "uint64_t" + or type == "double" + or type.endswith("*") + ): return 8 return 0 @@ -113,7 +121,7 @@ def find_struct(structs: list[DefinedStruct], name: str): return None -def create_struct_from_header_and_schema( +def map_header_and_schema( header: list[ExcelColumnDefinition], schema: DefinitionFields, header_index_offset: int = 0, @@ -129,10 +137,10 @@ def create_struct_from_header_and_schema( field = schema[index] if isinstance(field, RepeatDefinition): - exported = create_struct_from_header_and_schema( - header, field.fields, header_index, field.name + exported_enums, exported_structs = map_header_and_schema( + header, field.fields, header_index, schema_name + field.name ) - if len(exported.structs) == 0: + if len(exported_structs) == 0: base_type = definition.get_base_type_string() fields.append( DefinedStructFixedField( @@ -149,15 +157,15 @@ def create_struct_from_header_and_schema( fields.append( DefinedStructFixedField( field.name, - exported.structs[-1].name, + exported_structs[-1].name, definition.offset, False, field.count, False, ) ) - structs.extend(exported.structs) - enums.extend(exported.enums) + structs.extend(exported_structs) + enums.extend(exported_enums) pass else: if definition.is_packed_bool(): @@ -241,4 +249,56 @@ def create_struct_from_header_and_schema( ) ) - return DefinedStructExport(enums, structs) + return (enums, structs) + + +def create_struct_from_header_and_schema( + headers: dict[str, tuple[int, list[ExcelColumnDefinition]]], + schemas: dict[str, DefinitionFields], +) -> tuple[DefinedStructExport, dict[str, int]]: + struct_export = DefinedStructExport([], []) + excel_map: dict[str, int] = {} + for excel_header_name in headers: + index, header = headers[excel_header_name] + schema = schemas[excel_header_name] + excel_map[excel_header_name] = index + enums, structs = map_header_and_schema(header, schema, 0, excel_header_name) + struct_export.enums.extend(enums) + struct_export.structs.extend(structs) + + struct_export.enums.append(DefinedStructEnum( + "SheetsEnum", + "Component::Exd::SheetsEnum", + "Component::Exd", + "uint32_t", + False, + excel_map)) + + return (struct_export, excel_map) + +exd_func_patterns: dict[str, tuple[str, tuple[str | None, int]]] = { + "48 83 EC 28 48 8B 05 ? ? ? ? 44 8B C1 BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ("Row", ("48 8B 05 ? ? ? ? 44 8B C1 BA", 11)), + "48 83 EC 28 85 C9 74 20 48 8B 05 ? ? ? ? 44 8B C1 BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ("Row", ("48 8B 05 ? ? ? ? 44 8B C1 BA", 11)), + "48 83 EC 28 48 8B 05 ? ? ? ? BA ? ? ? ? 44 0F B6 C1 48 8B 88 ? ? ? ? E8": ("Row", ("48 8B 05 ? ? ? ? BA", 8)), + "48 83 EC 28 48 8B 05 ? ? ? ? BA ? ? ? ? 44 0F B7 C1 48 8B 88 ? ? ? ? E8": ("Row", ("48 8B 05 ? ? ? ? BA", 8)), + "48 83 EC 28 48 8B 05 ? ? ? ? BA ? ? ? ? 44 0F B7 81 ? ? ? ? 48 8B 88 ? ? ? ? E8": ("Row", ("48 8B 05 ? ? ? ? BA", 8)), + "40 53 48 83 EC 20 48 8B 05 ? ? ? ? 44 8B C1 8B DA BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ("Row", ("44 8B C1 8B DA BA", 6)), + "48 83 EC 28 48 8B 05 ? ? ? ? 44 8B C1 84 D2 BA ? ? ? ? 48 8B 88 ? ? ? ? 74 07 E8": ("Row", ("44 8B C1 84 D2 BA", 6)), + "48 83 EC 38 48 8B 05 ? ? ? ? 44 8B CA 44 8B C1 48 C7 44 24 ? ? ? ? ? BA ? ? ? ? 48 C7 44 24 ? ? ? ? ? 48 8B 88 ? ? ? ? E8": ("RowAndSubRowId", ("C1 48 C7 44 24 ? ? ? ? ? BA", 11)), + "48 83 EC 38 48 8B 05 ? ? ? ? 44 8B C1 44 0F ? CA BA ? ? ? ? 48 C7 44 24 28 ? ? ? ? 48 C7 44 24 20 ? ? ? ? 48 8B 88 ? ? ? ? E8": ("RowAndSubRowId", ("44 8B C1 44 0F ? CA BA", 8)), + "48 83 EC 28 48 8B 05 ? ? ? ? 44 8D 81 ? ? ? ? BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ("RowIndex", ("05 ? ? ? ? 44 8D 81 ? ? ? ? BA", 13)), + "48 83 EC 28 8D 41 ? 3D ? ? ? ? 77 20 48 8B 05 ? ? ? ? 44 8B C1 BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ("RowIndex", ("48 8B 05 ? ? ? ? 44 8B C1 BA", 11)), + "48 83 EC 28 48 8B 05 ? ? ? ? BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ("SheetIndex", ("48 8B 05 ? ? ? ? BA", 8)), + "48 8B 05 ? ? ? ? BA ? ? ? ? 48 8B 88 ? ? ? ? E9 ? ? ? ?": ("RowCount", (None, 8)) +} + +exd_comment_patterns: dict[str, int] = { + "48 8B 05 ? ? ? ? BA ? ? ? ? 48 8B 88 40 2B 00 00": 8, + "48 8B 05 ? ? ? ? BA ? ? ? ? 44 0F": 8, + "48 8B 05 ? ? ? ? 44 8B C1 8B DA BA": 13, + "48 8B 05 ? ? ? ? 44 8B C1 BA": 11, + "48 8B 05 ? ? ? ? 44 8B CA 44 8B C1 48 C7 44 24 ? ? ? ? ? BA": 23, + "48 8B 05 ? ? ? ? 44 8B C1 84 D2 BA": 13, + "48 8B 05 ? ? ? ? 44 8B C1 44 0F ? CA BA": 15, + "48 8B 05 ? ? ? ? 44 8D 81 ? ? ? ? BA": 15 +} \ No newline at end of file diff --git a/scripts/lumina_wrapper.py b/scripts/lumina_wrapper.py index 1fafe5bdb2..6353fe470c 100644 --- a/scripts/lumina_wrapper.py +++ b/scripts/lumina_wrapper.py @@ -83,26 +83,28 @@ def is_packed_bool(self): def get_base_type_string(self): match self.type: - case ExcelColumnDataType.Bool | ExcelColumnDataType.UInt8: - return "size8_t" + case ExcelColumnDataType.Bool: + return "bool" + case ExcelColumnDataType.UInt8: + return "uint8_t" case ExcelColumnDataType.Int8: - return "size8_st" + return "int8_t" case ExcelColumnDataType.UInt16: - return "size16_t" + return "uint16_t" case ExcelColumnDataType.Int16: - return "size16_st" + return "int16_t" case ExcelColumnDataType.String | ExcelColumnDataType.UInt32: - return "size32_t" + return "uint32_t" case ExcelColumnDataType.Int32: - return "size32_st" + return "int32_t" case ExcelColumnDataType.Float32: return "float" case ExcelColumnDataType.Int64: - return "size64_st" + return "int64_t" case ExcelColumnDataType.UInt64: - return "size64_t" + return "uint64_t" case _ if self.is_packed_bool(): - return "size8_t" + return "uint8_t" case _: raise ValueError(f"Unhandled type: {self.type}") @@ -176,14 +178,11 @@ def get_excel_header_files() -> dict[str, list[ExcelColumnDefinition]]: root_exl = game_data_instance.GetFile[excel_list_type]("exd/root.exl") - excel_files: dict[int, str] = {} - excel_header_files: dict[str, list[ExcelColumnDefinition]] = {} + excel_header_files: dict[str, tuple[int, list[ExcelColumnDefinition]]] = {} for kvp in root_exl.ExdMap: key = kvp.Key value = kvp.Value - if value != -1: - excel_files[value] = key if value != -1: defs: list[ExcelColumnDefinition] = [] @@ -194,6 +193,6 @@ def get_excel_header_files() -> dict[str, list[ExcelColumnDefinition]]: int(column_definition.Type), column_definition.Offset ) ) - excel_header_files[key] = sorted(defs) + excel_header_files[key] = (value, sorted(defs)) return excel_header_files diff --git a/scripts/structs_schema.py b/scripts/structs_schema.py index 796bf84f17..df22361604 100644 --- a/scripts/structs_schema.py +++ b/scripts/structs_schema.py @@ -1,6 +1,15 @@ from dataclasses import dataclass -from typing import TypeAlias +from typing import TypeAlias, Union +from yaml import load +try: + from yaml import CSafeLoader as Loader +except ImportError: + from yaml import SafeLoader as Loader + +YamlStructValuesExport: TypeAlias = list[dict[str, Union[str, int, "YamlStructValuesExport"]]] +YamlStructExport: TypeAlias = dict[str, YamlStructValuesExport] +YamlExport: TypeAlias = dict[str, list[YamlStructExport]] @dataclass class DefinedStructBase: @@ -82,3 +91,144 @@ class DefinedStruct(DefinedStructBase): class DefinedStructExport: enums: list[DefinedStructEnum] structs: list[DefinedStruct] + + +def get_yaml(stream) -> DefinedStructExport: + dic: YamlExport = load( + stream, Loader + ) + enums = [] + structs = [] + for enum in dic["enums"]: + enums.append( + DefinedStructEnum( + enum["name"], + enum["type"], + enum["namespace"], + enum["underlying"], + enum["flags"], + enum["values"], + ) + ) + for struct in dic["structs"]: + fields = [] + virtual_functions = None + member_functions = [] + static_member_functions = None + static_members = None + for field in struct["fields"]: + base = field["base"] if "base" in field else False + if "size" in field: + fields.append( + DefinedStructFixedField( + field["name"], + field["type"], + field["offset"], + base, + field["size"], + field["is_string"] + ) + ) + elif "return_type" in field: + parameters = [] + for param in field["parameters"]: + parameters.append( + DefinedStructFuncParam(param["name"], param["type"]) + ) + fields.append( + DefinedStructFuncField( + field["name"], + field["type"], + field["offset"], + base, + field["return_type"], + parameters, + ) + ) + else: + fields.append( + DefinedStructField( + field["name"], field["type"], field["offset"], base + ) + ) + if "virtual_functions" in struct: + virtual_functions = [] + for vfunc in struct["virtual_functions"]: + parameters = ( + [ + DefinedStructFuncParam(param["name"], param["type"]) + for param in vfunc["parameters"] + ] + if "parameters" in vfunc + else None + ) + virtual_functions.append( + DefinedStructVFunc( + vfunc["name"], + vfunc["return_type"] if "return_type" in vfunc else None, + vfunc["offset"], + parameters, + ) + ) + for memfunc in struct["member_functions"]: + parameters = [] + for param in memfunc["parameters"]: + parameters.append(DefinedStructFuncParam(param["name"], param["type"])) + member_functions.append( + DefinedStructMemFunc( + memfunc["signature"], + memfunc["return_type"], + parameters, + memfunc["name"], + ) + ) + if "static_member_functions" in struct: + static_member_functions = [] + for smemfunc in struct["static_member_functions"]: + parameters = [] + for param in smemfunc["parameters"]: + parameters.append( + DefinedStructFuncParam(param["name"], param["type"]) + ) + static_member_functions.append( + DefinedStructMemFunc( + smemfunc["signature"], + smemfunc["return_type"], + parameters, + smemfunc["name"], + ) + ) + if "static_members" in struct: + static_members = [] + for sm in struct["static_members"]: + static_members.append( + DefinedStructStaticMember( + sm["signature"], + sm["relative_follow_offsets"], + sm["return_type"], + sm["is_pointer"] if "is_pointer" in sm else False, + ) + ) + size = None + if "size" in struct: + size = struct["size"] + vtable_size = None + if "vtable_size" in struct: + vtable_size = struct["vtable_size"] + structs.append( + DefinedStruct( + struct["name"], + struct["type"], + struct["namespace"], + fields, + size, + vtable_size, + virtual_functions, + member_functions, + struct["union"], + static_member_functions, + static_members, + [] + ) + ) + return DefinedStructExport(enums, structs) \ No newline at end of file From 0f14d0379e19b231e9952d22f859df710ea29997 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Thu, 7 May 2026 00:39:02 +0200 Subject: [PATCH 14/95] move and update exdgetters to use new methods --- scripts/ffxiv_exdgetters.py | 375 ++++++++++++------------------------ scripts/ida/exdgetters.py | 147 ++++++++++++++ 2 files changed, 273 insertions(+), 249 deletions(-) create mode 100644 scripts/ida/exdgetters.py diff --git a/scripts/ffxiv_exdgetters.py b/scripts/ffxiv_exdgetters.py index e6ecef9545..a3896a18a8 100644 --- a/scripts/ffxiv_exdgetters.py +++ b/scripts/ffxiv_exdgetters.py @@ -39,222 +39,19 @@ def process_pattern(self, pattern): if api is None: try: import idaapi - import ida_bytes - import ida_search - import ida_typeinf - import ida_hexrays - import ida_name - import idc - from ida.ida_wrapper import IdaInterface except ImportError as e: print("Warning: Unable to load IDA failed with {0}".format(e)) else: - # noinspection PyUnresolvedReferences - class IdaApi(BaseApi, IdaInterface): - def do_pattern(self, pattern, suffix, sheetSearchPattern, sheetSearchOffset): - ea = 0 - - while True: - ea = self.search_binary(ea + 1, pattern, ida_search.SEARCH_DOWN) - - if ea == 0xFFFFFFFFFFFFFFFF: - break - - if sheetSearchPattern == None: - sheetEa = ea + sheetSearchOffset - else: - sheetEa = self.search_binary(ea, sheetSearchPattern, ida_search.SEARCH_DOWN) + sheetSearchOffset - sheetIdx = self.get_dword(sheetEa) - origName = idc.get_func_name(ea) - - # don't rename any funcs that are already named - if origName[0:4] == "sub_": - if exd_map.get(sheetIdx) == None: - print( - f"Func @ 0x{ea:X} references unknown sheet {sheetIdx}! sheet dword offset = 0x{sheetEa:X}" - ) - continue - - sheetName = exd_map[sheetIdx] - - if suffix == None: - suffix = "" - - fnName = "Component::Exd::ExdModule_Get%s%s" % ( - exd_map[sheetIdx], - suffix, - ) - - uniquifier = 0 - while True: - uniqueName = fnName + ( - f"_{uniquifier}" if uniquifier > 0 else "" - ) - - # check if this name is unique now - if ( - idc.get_name_ea_simple(uniqueName) == idc.BADADDR - and uniquifier > 0 - ): - fnName = uniqueName - break - - uniquifier += 1 - - self.set_func_name( - ea, fnName, "Sheet: {0} ({1})".format(sheetName, sheetIdx) - ) - - if suffix == "SheetIndex": - continue - - tif, funcdata = ( - ida_typeinf.tinfo_t(), - ida_typeinf.func_type_data_t(), - ) - - ida_typeinf.guess_tinfo(tif, ea) - if not tif.get_func_details(funcdata): - ida_hexrays.decompile(ea) - ida_typeinf.guess_tinfo(tif, ea) - if not tif.get_func_details(funcdata): - print( - "Failed to get func details for %s @ %X" - % (fnName, ea) - ) - continue - - - if suffix == "RowAndSubRowId": - tif = self.get_tinfo_from_type( - f"{exd_struct_map[sheetIdx]} *__fastcall func(unsigned int rowId, unsigned int subRowId);" - ) - elif suffix == "RowCount": - tif = self.get_tinfo_from_type( - f"{exd_struct_map[sheetIdx]} *__fastcall func();" - ) - else: - tif = self.get_tinfo_from_type( - f"{exd_struct_map[sheetIdx]} *__fastcall func(unsigned int rowIdOrIndex);" - ) - - ida_typeinf.apply_tinfo(ea, tif, ida_typeinf.TINFO_DEFINITE) - - def create_enum_struct(self, name, values, width = 0): - # type: (str, dict[int, str], int) -> None - if len(name.split("::")) > 3: - sheet_name = name.split("::")[-2] - else: - sheet_name = name.split("::")[-1] - - # create or reset if exists - enum_id = self.get_enum_id(name) - if enum_id == idaapi.BADADDR: - enum_id = self.create_enum(name) - else: - self.delete_enum_members(enum_id) - idc.set_enum_bf(enum_id, False) - - self.set_enum_width(enum_id, width) - if width == 1: - if idaapi.IDA_SDK_VERSION < 900: - self.add_enum_member(enum_id, f"{sheet_name}.tmp", self.get_enum_default_mask(enum_id)) - self.set_enum_as_bf(enum_id) - - for key in values: - self.add_enum_member(enum_id, f"{sheet_name}.{values[key]}", key) - - if width == 1 and idaapi.IDA_SDK_VERSION < 900: - self.remove_enum_member(enum_id, "tmp", sheet_name) - - def create_struct(self, name, fields): - idaapi.begin_type_updating(idaapi.UTP_STRUCT) - struct_id = self.get_struct_id(name) - if struct_id == idaapi.BADADDR: - struct_id = self.create_struct_type(name) - else: - self.remove_struct_members(struct_id) - struct_type = self.get_struct(struct_id) - for [index, [type, name]] in fields.items(): - if ( - self.get_idc_type_from_ida_type( - self.clean_struct_name(type) - ) - == self.get_struct_flag() - ): - type = self.clean_struct_name(type) - self.create_struct_member( - struct_type, - name, - index, - self.get_idc_type_from_ida_type(type), - self.get_struct_opinfo_from_type(type), - self.get_size_from_ida_type(type), - ) - elif ( - self.get_idc_type_from_ida_type(type) - == self.get_enum_flag() - ): - self.create_struct_member( - struct_type, - name, - index, - self.get_idc_type_from_ida_type(type), - self.get_enum_opinfo_from_type(type), - self.get_size_from_ida_type(type), - ) - else: - self.create_struct_member( - struct_type, - name, - index, - self.get_idc_type_from_ida_type(type), - None, - self.get_size_from_ida_type(type), - ) - meminfo = self.get_struct_member_by_name(struct_type, name) - self.set_struct_member_info( - struct_type, meminfo, 0, self.get_tinfo_from_type(type), 0 - ) - idaapi.end_type_updating(idaapi.UTP_STRUCT) - - def set_func_name(self, ea, name, cmt): - ida_name.set_name(ea, name) - ida_bytes.set_cmt(ea, cmt, 0) - - def process_pattern(self, pattern: str): - (suffix, search) = exd_func_patterns[pattern] - if suffix is not None: - print(f"Finding exd funcs of {suffix}... please wait.") - (searchPattern, searchOffset) = search - self.do_pattern(pattern, suffix, searchPattern, searchOffset) - - def get_all_eas(self, pattern: str): - ea = self.search_binary(0, pattern, ida_search.SEARCH_DOWN) - while ea != idc.BADADDR: - yield ea - ea = ea + 1 - ea = self.search_binary(ea, pattern, ida_search.SEARCH_DOWN) - - def comment_rows(self, pattern: dict[str, int], values: dict[int, str]): - for pattern_key in pattern: - for ea in list(self.get_all_eas(pattern_key)): - try: - sheetIdx = self.get_dword(ea + pattern[pattern_key]) - sheetName = values[sheetIdx] - ida_bytes.set_cmt(ea, "Sheet: {0} ({1})".format(sheetName, sheetIdx), 0) - except Exception as e: - print(f"An unexpected error occurred at 0x{hex(ea)}: {e}") - pass - - api = IdaApi() + print("Script has moved to new location inside of ida folder.") + exit(1) if api is None: try: import ghidra import re + try: - from ghidra.ghidra_builtins import * # ghidra-stubs + from ghidra.ghidra_builtins import * # ghidra-stubs except ImportError: pass @@ -269,10 +66,12 @@ def comment_rows(self, pattern: dict[str, int], values: dict[int, str]): else: # noinspection PyUnresolvedReferences class GhidraApi(BaseApi): - def create_enum_struct(self, name, values, width = 0): + def create_enum_struct(self, name, values, width=0): # type: (str, dict[int, str], int) -> None path = self.get_datatype_path(name) - enum_dt = EnumDataType(path.getCategoryPath(), path.getDataTypeName(), width or 8) + enum_dt = EnumDataType( + path.getCategoryPath(), path.getDataTypeName(), width or 8 + ) is_sheets_enum = name == "Component::Exd::SheetsEnum" for enum_value, enum_name in values.items(): if monitor.isCancelled(): @@ -293,14 +92,20 @@ def create_enum_struct(self, name, values, width = 0): def create_struct(self, name, fields): # type: (str, dict[str, str]) -> None dt_path = self.get_datatype_path(name) - struct = StructureDataType(dt_path.getCategoryPath(), dt_path.getDataTypeName(), 0) + struct = StructureDataType( + dt_path.getCategoryPath(), dt_path.getDataTypeName(), 0 + ) struct.setExplicitMinimumAlignment(4) for [offset, [type_name, field_name]] in fields.items(): if monitor.isCancelled(): break - field_dt = currentProgram.getDataTypeManager().getDataType(self.get_datatype_path(type_name)) + field_dt = currentProgram.getDataTypeManager().getDataType( + self.get_datatype_path(type_name) + ) if field_dt is None: - print(f"Warning: Data type {name} field {field_name} has missing type {type_name} ({dt_path})") + print( + f"Warning: Data type {name} field {field_name} has missing type {type_name} ({dt_path})" + ) continue struct.insertAtOffset(offset, field_dt, -1, field_name, None) dt = currentProgram.getDataTypeManager().getDataType(dt_path) @@ -319,8 +124,10 @@ def set_func_name(self, ea, name, cmt): def process_pattern(self, pattern): # type: (str) -> None - (suffix, _) = exd_func_patterns[pattern] - pattern = "".join(["\\x" + x if x != "?" else "." for x in pattern.split(" ")]) + suffix, _ = exd_func_patterns[pattern] + pattern = "".join( + ["\\x" + x if x != "?" else "." for x in pattern.split(" ")] + ) if suffix is not None: print(f"Finding exd funcs of {suffix}... please wait.") self.do_pattern(pattern, suffix) @@ -328,7 +135,9 @@ def process_pattern(self, pattern): def do_pattern(self, pattern, suffix): # type: (string, string) -> None block = currentProgram.getMemory().getBlock(".text") - address_set = currentProgram.getAddressFactory().getAddressSet(block.getStart(), block.getEnd()) + address_set = currentProgram.getAddressFactory().getAddressSet( + block.getStart(), block.getEnd() + ) result_list = findBytes(address_set, pattern, 8192, 1) for ea in result_list: if monitor.isCancelled(): @@ -336,10 +145,14 @@ def do_pattern(self, pattern, suffix): sheet_index = self.get_sheet_index(ea) sheet_name = exd_map.get(sheet_index) if sheet_name is None: - print(f"Warning: sheet index {sheet_index} is undefined at {ea}") + print( + f"Warning: sheet index {sheet_index} is undefined at {ea}" + ) continue func_name = f"Component::Exd::ExdModule.Get{sheet_name}{suffix}" - self.set_func_name(ea, func_name, f"Sheet: {sheet_name} ({sheet_index})") + self.set_func_name( + ea, func_name, f"Sheet: {sheet_name} ({sheet_index})" + ) if suffix == "SheetIndex": return @@ -354,14 +167,29 @@ def do_pattern(self, pattern, suffix): return_var = ReturnParameterImpl(return_type, currentProgram) arg_vars = ArrayList() - arg_vars.add(ParameterImpl("rowId", UnsignedIntegerDataType(), currentProgram)) + arg_vars.add( + ParameterImpl( + "rowId", UnsignedIntegerDataType(), currentProgram + ) + ) if suffix == "RowAndSubRowId": - arg_vars.add(ParameterImpl("subRowId", UnsignedIntegerDataType(), currentProgram)) + arg_vars.add( + ParameterImpl( + "subRowId", UnsignedIntegerDataType(), currentProgram + ) + ) elif suffix == "RowCount": arg_vars.clear() update_type = Function.FunctionUpdateType.DYNAMIC_STORAGE_ALL_PARAMS - getFunctionAt(ea).updateFunction("__fastcall", return_var, arg_vars, update_type, False, SourceType.USER_DEFINED) + getFunctionAt(ea).updateFunction( + "__fastcall", + return_var, + arg_vars, + update_type, + False, + SourceType.USER_DEFINED, + ) def get_sheet_index(self, ea): # type: (Address) -> int @@ -371,7 +199,9 @@ def get_sheet_index(self, ea): if func is None: return -1 min_address = func.getBody().getMinAddress().getOffset() - instructions = currentProgram.getListing().getInstructions(func.getBody(), True) + instructions = currentProgram.getListing().getInstructions( + func.getBody(), True + ) for insn in instructions: if monitor.isCancelled(): break @@ -382,7 +212,9 @@ def get_sheet_index(self, ea): def get_rdx_arg(self, insn, min_address): # type: (Instruction, int) -> int target = insn.getRegister("RDX") - while insn is not None and insn.getMinAddress().getOffset() >= min_address: + while ( + insn is not None and insn.getMinAddress().getOffset() >= min_address + ): if monitor.isCancelled(): break insn = insn.getPrevious() @@ -398,17 +230,25 @@ def get_rdx_arg(self, insn, min_address): def get_datatype_path(self, name): # type: (str) -> DataTypePath - if name == "__int8": name = "char" - if name == "__int16": name = "short" - if name == "__int32": name = "int" - if name == "__int64": name = "longlong" - if name == "unsigned __int8" or name == "unsigned char": name = "byte" - if name == "unsigned __int16" or name == "unsigned short": name = "ushort" - if name == "unsigned __int32" or name == "unsigned int": name = "uint" - if name == "unsigned __int64" or name == "unsigned long long": name = "ulonglong" + if name == "__int8": + name = "char" + if name == "__int16": + name = "short" + if name == "__int32": + name = "int" + if name == "__int64": + name = "longlong" + if name == "unsigned __int8" or name == "unsigned char": + name = "byte" + if name == "unsigned __int16" or name == "unsigned short": + name = "ushort" + if name == "unsigned __int32" or name == "unsigned int": + name = "uint" + if name == "unsigned __int64" or name == "unsigned long long": + name = "ulonglong" path_parts = SymbolPathParser.parse(name) return DataTypePath("/" + "/".join(path_parts[:-1]), path_parts[-1]) - + def comment_rows(self, pattern: dict[str, int], values: dict[int, str]): pass @@ -446,19 +286,55 @@ def comment_rows(self, pattern: dict[str, int], values: dict[int, str]): # nb: "pattern": ("func suffix", ("instance pointer sig" or None, offset from sig start or func start)) exd_func_patterns: dict[str, tuple[str, tuple[str | None, int]]] = { - "48 83 EC 28 48 8B 05 ? ? ? ? 44 8B C1 BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ("Row", ("48 8B 05 ? ? ? ? 44 8B C1 BA", 11)), - "48 83 EC 28 85 C9 74 20 48 8B 05 ? ? ? ? 44 8B C1 BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ("Row", ("48 8B 05 ? ? ? ? 44 8B C1 BA", 11)), - "48 83 EC 28 48 8B 05 ? ? ? ? BA ? ? ? ? 44 0F B6 C1 48 8B 88 ? ? ? ? E8": ("Row", ("48 8B 05 ? ? ? ? BA", 8)), - "48 83 EC 28 48 8B 05 ? ? ? ? BA ? ? ? ? 44 0F B7 C1 48 8B 88 ? ? ? ? E8": ("Row", ("48 8B 05 ? ? ? ? BA", 8)), - "48 83 EC 28 48 8B 05 ? ? ? ? BA ? ? ? ? 44 0F B7 81 ? ? ? ? 48 8B 88 ? ? ? ? E8": ("Row", ("48 8B 05 ? ? ? ? BA", 8)), - "40 53 48 83 EC 20 48 8B 05 ? ? ? ? 44 8B C1 8B DA BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ("Row", ("44 8B C1 8B DA BA", 6)), - "48 83 EC 28 48 8B 05 ? ? ? ? 44 8B C1 84 D2 BA ? ? ? ? 48 8B 88 ? ? ? ? 74 07 E8": ("Row", ("44 8B C1 84 D2 BA", 6)), - "48 83 EC 38 48 8B 05 ? ? ? ? 44 8B CA 44 8B C1 48 C7 44 24 ? ? ? ? ? BA ? ? ? ? 48 C7 44 24 ? ? ? ? ? 48 8B 88 ? ? ? ? E8": ("RowAndSubRowId", ("C1 48 C7 44 24 ? ? ? ? ? BA", 11)), - "48 83 EC 38 48 8B 05 ? ? ? ? 44 8B C1 44 0F ? CA BA ? ? ? ? 48 C7 44 24 28 ? ? ? ? 48 C7 44 24 20 ? ? ? ? 48 8B 88 ? ? ? ? E8": ("RowAndSubRowId", ("44 8B C1 44 0F ? CA BA", 8)), - "48 83 EC 28 48 8B 05 ? ? ? ? 44 8D 81 ? ? ? ? BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ("RowIndex", ("05 ? ? ? ? 44 8D 81 ? ? ? ? BA", 13)), - "48 83 EC 28 8D 41 ? 3D ? ? ? ? 77 20 48 8B 05 ? ? ? ? 44 8B C1 BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ("RowIndex", ("48 8B 05 ? ? ? ? 44 8B C1 BA", 11)), - "48 83 EC 28 48 8B 05 ? ? ? ? BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ("SheetIndex", ("48 8B 05 ? ? ? ? BA", 8)), - "48 8B 05 ? ? ? ? BA ? ? ? ? 48 8B 88 ? ? ? ? E9 ? ? ? ?": ("RowCount", (None, 8)) + "48 83 EC 28 48 8B 05 ? ? ? ? 44 8B C1 BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ( + "Row", + ("48 8B 05 ? ? ? ? 44 8B C1 BA", 11), + ), + "48 83 EC 28 85 C9 74 20 48 8B 05 ? ? ? ? 44 8B C1 BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ( + "Row", + ("48 8B 05 ? ? ? ? 44 8B C1 BA", 11), + ), + "48 83 EC 28 48 8B 05 ? ? ? ? BA ? ? ? ? 44 0F B6 C1 48 8B 88 ? ? ? ? E8": ( + "Row", + ("48 8B 05 ? ? ? ? BA", 8), + ), + "48 83 EC 28 48 8B 05 ? ? ? ? BA ? ? ? ? 44 0F B7 C1 48 8B 88 ? ? ? ? E8": ( + "Row", + ("48 8B 05 ? ? ? ? BA", 8), + ), + "48 83 EC 28 48 8B 05 ? ? ? ? BA ? ? ? ? 44 0F B7 81 ? ? ? ? 48 8B 88 ? ? ? ? E8": ( + "Row", + ("48 8B 05 ? ? ? ? BA", 8), + ), + "40 53 48 83 EC 20 48 8B 05 ? ? ? ? 44 8B C1 8B DA BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ( + "Row", + ("44 8B C1 8B DA BA", 6), + ), + "48 83 EC 28 48 8B 05 ? ? ? ? 44 8B C1 84 D2 BA ? ? ? ? 48 8B 88 ? ? ? ? 74 07 E8": ( + "Row", + ("44 8B C1 84 D2 BA", 6), + ), + "48 83 EC 38 48 8B 05 ? ? ? ? 44 8B CA 44 8B C1 48 C7 44 24 ? ? ? ? ? BA ? ? ? ? 48 C7 44 24 ? ? ? ? ? 48 8B 88 ? ? ? ? E8": ( + "RowAndSubRowId", + ("C1 48 C7 44 24 ? ? ? ? ? BA", 11), + ), + "48 83 EC 38 48 8B 05 ? ? ? ? 44 8B C1 44 0F ? CA BA ? ? ? ? 48 C7 44 24 28 ? ? ? ? 48 C7 44 24 20 ? ? ? ? 48 8B 88 ? ? ? ? E8": ( + "RowAndSubRowId", + ("44 8B C1 44 0F ? CA BA", 8), + ), + "48 83 EC 28 48 8B 05 ? ? ? ? 44 8D 81 ? ? ? ? BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ( + "RowIndex", + ("05 ? ? ? ? 44 8D 81 ? ? ? ? BA", 13), + ), + "48 83 EC 28 8D 41 ? 3D ? ? ? ? 77 20 48 8B 05 ? ? ? ? 44 8B C1 BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ( + "RowIndex", + ("48 8B 05 ? ? ? ? 44 8B C1 BA", 11), + ), + "48 83 EC 28 48 8B 05 ? ? ? ? BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ( + "SheetIndex", + ("48 8B 05 ? ? ? ? BA", 8), + ), + "48 8B 05 ? ? ? ? BA ? ? ? ? 48 8B 88 ? ? ? ? E9 ? ? ? ?": ("RowCount", (None, 8)), } exd_comment_patterns: dict[str, int] = { @@ -469,7 +345,7 @@ def comment_rows(self, pattern: dict[str, int], values: dict[int, str]): "48 8B 05 ? ? ? ? 44 8B CA 44 8B C1 48 C7 44 24 ? ? ? ? ? BA": 23, "48 8B 05 ? ? ? ? 44 8B C1 84 D2 BA": 13, "48 8B 05 ? ? ? ? 44 8B C1 44 0F ? CA BA": 15, - "48 8B 05 ? ? ? ? 44 8D 81 ? ? ? ? BA": 15 + "48 8B 05 ? ? ? ? 44 8D 81 ? ? ? ? BA": 15, } exd_map = ExcelListFile(game_data.get_file(ParsedFileName("exd/root.exl"))).dict @@ -481,17 +357,18 @@ def comment_rows(self, pattern: dict[str, int], values: dict[int, str]): for key in exd_map: print(f"Parsing schema for {exd_map[key]}.") exd_headers[key] = ExcelHeaderFile( - game_data.get_file(ParsedFileName("exd/" + exd_map[key] + ".exh")), - exd_map[key] + game_data.get_file(ParsedFileName("exd/" + exd_map[key] + ".exh")), exd_map[key] ).map_names(game_data.get_exd_schema(exd_map[key])) for key in exd_headers: [exd_header, exd_header_enums, exd_header_count] = exd_headers[key] struct_name = f"Component::Exd::Sheets::{exd_map[key]}" - print(f'Creating struct {struct_name}.') + print(f"Creating struct {struct_name}.") exd_struct_map[key] = struct_name for enum_key in exd_header_enums: - api.create_enum_struct(f"{struct_name}::{enum_key}", exd_header_enums[enum_key], 1) + api.create_enum_struct( + f"{struct_name}::{enum_key}", exd_header_enums[enum_key], 1 + ) api.create_struct(struct_name, exd_header) # if a new pattern is found, add it to the exd_func_patterns dict diff --git a/scripts/ida/exdgetters.py b/scripts/ida/exdgetters.py new file mode 100644 index 0000000000..a11e2644a9 --- /dev/null +++ b/scripts/ida/exdgetters.py @@ -0,0 +1,147 @@ +from os.path import join, dirname +import sys + +sys.path.append(join(dirname(__file__), "..")) + +from lumina_wrapper import get_excel_header_files +from exdschema_wrapper import ( + create_struct_from_header_and_schema, + get_exdschema_data, + exd_comment_patterns, + exd_func_patterns, +) +from src_wrapper import SrcInterface +from ida_wrapper import IdaInterface +import ida_bytes +import ida_search +import ida_typeinf +import ida_hexrays +import idc + +src = SrcInterface() + +excel_header_files = get_excel_header_files() +exdschema = get_exdschema_data("latest") + +struct_export, excel_map = create_struct_from_header_and_schema( + excel_header_files, exdschema +) + +exd_map = {excel_map[k]: k for k in excel_map} + +header, vtables = src.build_export_string(struct_export) + +ida = IdaInterface() + +ida.select_parser() +ida.parse_string(header, vtables) + + +def do_pattern(pattern, suffix, sheetSearchPattern, sheetSearchOffset): + ea = 0 + + while True: + ea = ida.search_binary(ea + 1, pattern, ida_search.SEARCH_DOWN) + + if ea == 0xFFFFFFFFFFFFFFFF: + break + + if sheetSearchPattern == None: + sheetEa = ea + sheetSearchOffset + else: + sheetEa = ( + ida.search_binary(ea, sheetSearchPattern, ida_search.SEARCH_DOWN) + + sheetSearchOffset + ) + sheetIdx = ida.get_dword(sheetEa) + origName = idc.get_func_name(ea) + + # don't rename any funcs that are already named + if origName[0:4] == "sub_": + if exd_map.get(sheetIdx) == None: + print( + f"Func @ 0x{ea:X} references unknown sheet {sheetIdx}! sheet dword offset = 0x{sheetEa:X}" + ) + continue + + sheetName = exd_map[sheetIdx] + + if suffix == None: + suffix = "" + + fnName = "Component::Exd::ExdModule_Get%s%s" % ( + exd_map[sheetIdx], + suffix, + ) + + uniquifier = 0 + while True: + uniqueName = fnName + (f"_{uniquifier}" if uniquifier > 0 else "") + + # check if this name is unique now + if idc.get_name_ea_simple(uniqueName) == idc.BADADDR and uniquifier > 0: + fnName = uniqueName + break + + uniquifier += 1 + + ida.set_func_name( + ea, fnName, "Sheet: {0} ({1})".format(sheetName, sheetIdx) + ) + + if suffix == "SheetIndex": + continue + + tif, funcdata = ( + ida_typeinf.tinfo_t(), + ida_typeinf.func_type_data_t(), + ) + + ida_typeinf.guess_tinfo(tif, ea) + if not tif.get_func_details(funcdata): + ida_hexrays.decompile(ea) + ida_typeinf.guess_tinfo(tif, ea) + if not tif.get_func_details(funcdata): + print("Failed to get func details for %s @ %X" % (fnName, ea)) + continue + + if suffix == "RowAndSubRowId": + tif = ida.get_tinfo_from_type( + f"Component::Exd::Sheets::{exd_map[sheetIdx]} *__fastcall func(unsigned int rowId, unsigned int subRowId);" + ) + elif suffix == "RowCount": + tif = ida.get_tinfo_from_type( + f"Component::Exd::Sheets::{exd_map[sheetIdx]} *__fastcall func();" + ) + else: + tif = ida.get_tinfo_from_type( + f"Component::Exd::Sheets::{exd_map[sheetIdx]} *__fastcall func(unsigned int rowIdOrIndex);" + ) + + ida_typeinf.apply_tinfo(ea, tif, ida_typeinf.TINFO_DEFINITE) + + +def get_all_eas(pattern: str): + ea = ida.search_binary(0, pattern, ida_search.SEARCH_DOWN) + while ea != idc.BADADDR: + yield ea + ea = ea + 1 + ea = ida.search_binary(ea, pattern, ida_search.SEARCH_DOWN) + + +for pattern in exd_func_patterns: + suffix, search = exd_func_patterns[pattern] + if suffix is not None: + print(f"Finding exd funcs of {suffix}... please wait.") + searchPattern, searchOffset = search + do_pattern(pattern, suffix, searchPattern, searchOffset) + + +for pattern_key in exd_comment_patterns: + for ea in list(get_all_eas(pattern_key)): + try: + sheetIdx = ida.get_dword(ea + exd_comment_patterns[pattern_key]) + sheetName = exd_map[sheetIdx] + ida_bytes.set_cmt(ea, "Sheet: {0} ({1})".format(sheetName, sheetIdx), 0) + except Exception as e: + print(f"An unexpected error occurred at {hex(ea)}: {e}") \ No newline at end of file From b7df684de75636d7c2237bbf10598603c41df466 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Sat, 16 May 2026 10:32:40 +0200 Subject: [PATCH 15/95] fix fastcall defines --- scripts/src_wrapper.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/src_wrapper.py b/scripts/src_wrapper.py index 267d62d337..c3415d8ec0 100644 --- a/scripts/src_wrapper.py +++ b/scripts/src_wrapper.py @@ -265,6 +265,8 @@ def build_struct_string( if struct.virtual_functions: struct_string_define += f"{short_name}_vtbl* __vtable;" + offset = 0 + for field in struct.fields: if field.type in base_fields_used: continue @@ -284,12 +286,13 @@ def build_struct_string( struct_string_define += f"{field_define[:-1]}[{field.size}];" prev_size += field_define_size * field.size elif field_type == "__fastcall": - struct_string_define += ( + fastcall_field_define = "" + fastcall_field_define += ( f"{field.return_type} ({field_type} *{field_name})(" ) for param in field.parameters: - struct_string_define += param.type + " " + param.name + ", " - struct_string_define = struct_string_define[:-1] + ");" + fastcall_field_define += param.type + " " + param.name + ", " + struct_string_define += fastcall_field_define[:-1] + ");" prev_size += 8 else: field_define, field_define_size = self.get_string_define_and_size(field, struct.namespace, struct_lookup, enum_lookup) From eaeecf3640ab71da9466b26e9448b089a27f0f38 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Sat, 16 May 2026 10:32:53 +0200 Subject: [PATCH 16/95] allow for overriding the loader type of yaml --- scripts/structs_schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/structs_schema.py b/scripts/structs_schema.py index df22361604..ba685b800a 100644 --- a/scripts/structs_schema.py +++ b/scripts/structs_schema.py @@ -93,9 +93,9 @@ class DefinedStructExport: structs: list[DefinedStruct] -def get_yaml(stream) -> DefinedStructExport: +def get_yaml(stream, loader = Loader) -> DefinedStructExport: dic: YamlExport = load( - stream, Loader + stream, loader ) enums = [] structs = [] From cc36e96c64863f292c77727277f90f7118e8e36d Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Sat, 16 May 2026 10:35:19 +0200 Subject: [PATCH 17/95] update importer to support std import definitions --- CExporter/Exporter.cs | 70 ++++++-- CExporter/ExporterStatics.cs | 11 +- CExporter/Extensions.cs | 337 ++++++++++++++++++----------------- 3 files changed, 237 insertions(+), 181 deletions(-) diff --git a/CExporter/Exporter.cs b/CExporter/Exporter.cs index d84f777457..35cd7dea41 100644 --- a/CExporter/Exporter.cs +++ b/CExporter/Exporter.cs @@ -298,7 +298,58 @@ private static void ProcessType(ProcessingType type, bool quiet) { } } + private static (string typeOverride, Type[] typeOverrideTemplates) ProcessStdField(Type fieldType) { + const string basicString = "BasicString"; + string[] pointerValuesType = ["Vector", "Deque", "Span"]; + var pointerCount = 0; + while (fieldType.IsPointer()) { + fieldType = fieldType.GetPointerType(); + pointerCount++; + } + var readOnlyNameSpan = fieldType.Name.AsSpan()[3..]; + Span stdNameSpan = stackalloc char[readOnlyNameSpan.Length + 5]; + stdNameSpan.Clear(); + ExporterStatics.StdCppNamespace.AsSpan().CopyTo(stdNameSpan); + if (readOnlyNameSpan[^2] == '`') { + for (var i = 0; i < readOnlyNameSpan.Length; i++) { + if (readOnlyNameSpan[i] == '`') break; + stdNameSpan[5 + i] = char.ToLower(readOnlyNameSpan[i]); + } + } else { + if (readOnlyNameSpan.CompareTo(basicString.AsSpan(), StringComparison.InvariantCultureIgnoreCase) != 0) { + for (var i = 0; i < readOnlyNameSpan.Length; i++) { + if (readOnlyNameSpan[i] == '`') break; + stdNameSpan[5 + i] = char.ToLower(readOnlyNameSpan[i]); + } + } else { + throw new NotImplementedException("Yell at me to fix this if it happens."); + } + } + var generics = fieldType.GenericTypeArguments; + var shouldBePointer = false; + foreach(var pointerType in pointerValuesType) { + if (!readOnlyNameSpan.StartsWith(pointerType)) continue; + shouldBePointer = true; + break; + } + while (stdNameSpan[^1] == '\0') stdNameSpan = stdNameSpan[..^1]; + var genericsDefine = generics.Select(t => t is { Namespace: ExporterStatics.StdNamespacePrefix } ? ProcessStdField(t) : (t.FixTypeName(), [shouldBePointer ? t.MakePointerType() : t])).ToArray(); + return ($"{stdNameSpan}<{string.Join(',', genericsDefine.Select(t => t.typeOverride))}>{new string('*', pointerCount)}", genericsDefine.SelectMany(t => t.typeOverrideTemplates).ToArray()); + } + private static ProcessedField ProcessField(FieldInfo field, int offset) { + if (field.FieldType is { Namespace: ExporterStatics.StdNamespacePrefix }) { + var (typeOverride, typeOverrideTemplates) = ProcessStdField(field.FieldType); + return new ProcessedField { + FieldType = field.FieldType, + FieldOffset = field.GetFieldOffset() - offset, + FieldName = field.Name, + IsBase = field.IsDirectBase(), + FieldTypeOverride = typeOverride, + FieldTypeOverrideTemplates = typeOverrideTemplates, + Bits = [] + }; + } if (field.FieldType is { Namespace: "FFXIVClientStructs.STD.Helper", Name: "Node*" }) { var fieldTypeOverride = $"RedBlackTree::Node<{field.FieldType.GetGenericArguments()[0].FixTypeName()}>*"; _processType.Add(new ProcessingType(field.FieldType, fieldTypeOverride[..^1])); @@ -608,7 +659,6 @@ private static ProcessedField ProcessVirtualParameter(Type parameter, int i, Par var memberFunctionClass = type.GetMember("MemberFunctionPointers", ExporterStatics.BindingFlags).FirstOrDefault()?.DeclaringType; ProcessedMemberFunction[] memberFunctionsArray = []; - //if (type == typeof(lua_State)) Debugger.Break(); if (memberFunctionClass != null) { var memberFunctions = memberFunctionClass.GetMethods(ExporterStatics.BindingFlags).Where(t => t.GetCustomAttribute() == null && t.GetCustomAttribute() == null).ToArray(); foreach (var memberFunction in memberFunctions) { @@ -791,6 +841,7 @@ public class ProcessedBitField { public class ProcessedField { public string? FieldTypeOverride; public bool? FieldTypeOverrideCheck; + public Type[]? FieldTypeOverrideTemplates; public required Type FieldType; public required string FieldName; public required int FieldOffset; @@ -844,14 +895,14 @@ public string[] DependencyNames { if (_dependencyNames.Length == 0) { _dependencyNames = Fields .Where(t => { - if (t.FieldTypeOverride != null) { - if (t.FieldTypeOverride.EndsWith('*') || t.FieldTypeOverride.StartsWith("Component::Exd::Sheets::") || ExporterStatics.BaseTypeNames.Contains(t.FieldTypeOverride)) - return false; - return !t.FieldTypeOverrideCheck.HasValue || !t.FieldTypeOverrideCheck.Value; - } - return (t.GetType() == typeof(ProcessedField) || t.GetType() == typeof(ProcessedFixedField)) && (!(t.FieldType.IsPointer() || t.FieldType.IsPrimitive || t.FieldType.IsFixedBuffer() || t.FieldType.IsEnum || t.FieldType.IsBaseType())); + if (t.FieldTypeOverride == null) return (t.GetType() == typeof(ProcessedField) || t.GetType() == typeof(ProcessedFixedField)) && !t.FieldType.ShouldNotExportType() && !ExporterStatics.BaseTypeNames.Contains(t.FieldType.FixTypeName()); + if (t.FieldTypeOverride.EndsWith('*') || t.FieldTypeOverride.StartsWith("Component::Exd::Sheets::") || ExporterStatics.BaseTypeNames.Contains(t.FieldTypeOverride)) return false; + return !t.FieldTypeOverrideCheck.HasValue || !t.FieldTypeOverrideCheck.Value; }) - .Select(t => t.FieldTypeOverride ?? t.FieldType.FixTypeName()).Distinct().ToArray(); + .SelectMany(t => { + if (t.FieldTypeOverride?.StartsWith(ExporterStatics.StdCppNamespace) ?? false) return t.FieldTypeOverrideTemplates!.Where(k => !k.ShouldNotExportType()).Select(k => k.FixTypeName()).Where(k => !ExporterStatics.BaseTypeNames.Contains(k)); + return [t.FieldTypeOverride ?? t.FieldType.FixTypeName()]; + }).Distinct().ToArray(); } return _dependencyNames; } @@ -1052,8 +1103,7 @@ public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializ } emitter.Emit(new Scalar("template_types")); emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Flow)); - foreach (var templateType in s.TemplateTypes) - { + foreach (var templateType in s.TemplateTypes) { emitter.Emit(new Scalar(templateType)); } emitter.Emit(new SequenceEnd()); diff --git a/CExporter/ExporterStatics.cs b/CExporter/ExporterStatics.cs index 05782d0ec4..c594369cc5 100644 --- a/CExporter/ExporterStatics.cs +++ b/CExporter/ExporterStatics.cs @@ -9,15 +9,16 @@ namespace CExporter; public static class ExporterStatics { #pragma warning disable CA2211 // ReSharper disable once InconsistentNaming - public static readonly string FFXIVNamespacePrefix = string.Join(".", nameof(FFXIVClientStructs), nameof(FFXIVClientStructs.FFXIV)); - public static readonly string StdNamespacePrefix = string.Join(".", nameof(FFXIVClientStructs), nameof(FFXIVClientStructs.STD)); - public static readonly string InteropNamespacePrefix = string.Join(".", nameof(FFXIVClientStructs), nameof(FFXIVClientStructs.Interop)); - public static readonly string HavokNamespacePrefix = string.Join(".", nameof(FFXIVClientStructs), nameof(FFXIVClientStructs.Havok)); + public const string FFXIVNamespacePrefix = $"{nameof(FFXIVClientStructs)}.{nameof(FFXIVClientStructs.FFXIV)}"; + public const string StdNamespacePrefix = $"{nameof(FFXIVClientStructs)}.{nameof(FFXIVClientStructs.STD)}"; + public const string InteropNamespacePrefix = $"{nameof(FFXIVClientStructs)}.{nameof(FFXIVClientStructs.Interop)}"; + public const string HavokNamespacePrefix = $"{nameof(FFXIVClientStructs)}.{nameof(FFXIVClientStructs.Havok)}"; + public const string StdCppNamespace = "std::"; public static readonly BindingFlags BindingFlags = BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance; public static readonly BindingFlags StaticBindingFlags = BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static; public static readonly string[] IgnoredTypeNames = ["MemberFunctionPointers", "StaticAddressPointers", "Addresses", "VirtualTable"]; - public static readonly string[] BaseTypeNames = ["byte", "wchar_t", "char", "float", "double", "__int16", "int", "__int64", "unsigned __int16", "unsigned int", "unsigned __int64", "__int8", "unsigned __int8"]; + public static readonly string[] BaseTypeNames = ["byte", "wchar_t", "char", "float", "double", "bool", "int8_t", "uint8_t", "int16_t", "uint16_t", "int32_t", "uint32_t", "int64_t", "uint64_t"]; public static readonly List WarningList = []; public static readonly List ErrorList = []; diff --git a/CExporter/Extensions.cs b/CExporter/Extensions.cs index 9fc105fa8e..e004876f21 100644 --- a/CExporter/Extensions.cs +++ b/CExporter/Extensions.cs @@ -17,202 +17,207 @@ public static partial class TypeExtensions { [GeneratedRegex(@"^`[1-9]\+Node", RegexOptions.Compiled)] public static partial Regex StdNodeRegex(); - public static bool IsFixedBuffer(this Type type) { - return type.Name.EndsWith("e__FixedBuffer"); - } + extension(Type type) { + public bool IsFixedBuffer() { + return type.Name.EndsWith("e__FixedBuffer"); + } - public static bool IsStruct(this Type type) { - return type != typeof(decimal) && type is { IsValueType: true, IsPrimitive: false, IsEnum: false }; - } + public bool IsStruct() { + return type != typeof(decimal) && type is { IsValueType: true, IsPrimitive: false, IsEnum: false }; + } - public static bool IsBaseType(this Type type) { - while (true) { - if (type.IsPointer) { - type = type.GetElementType()!; - continue; - } - if (type.IsGenericPointer()) { - type = type.GenericTypeArguments[0]; - continue; + public bool IsBaseType() { + while (true) { + if (type.IsPointer) { + type = type.GetElementType()!; + continue; + } + if (type.IsGenericPointer()) { + type = type.GenericTypeArguments[0]; + continue; + } + return type == typeof(void) || type == typeof(bool) || type == typeof(char) || + type == typeof(sbyte) || type == typeof(byte) || type == typeof(short) || + type == typeof(ushort) || type == typeof(int) || type == typeof(uint) || + type == typeof(long) || type == typeof(ulong) || type == typeof(float) || + type == typeof(double) || type == typeof(decimal) || type == typeof(nint) || + type == typeof(nuint) || type == typeof(Half) || type == typeof(CStringPointer); } - return type == typeof(void) || type == typeof(bool) || type == typeof(char) || - type == typeof(sbyte) || type == typeof(byte) || type == typeof(short) || - type == typeof(ushort) || type == typeof(int) || type == typeof(uint) || - type == typeof(long) || type == typeof(ulong) || type == typeof(float) || - type == typeof(double) || type == typeof(decimal) || type == typeof(nint) || - type == typeof(nuint) || type == typeof(Half) || type == typeof(CStringPointer); } - } - public static Type[] GetInheritsTypes(this Type type) { - const string inheritsAttribute = "InheritsAttribute`1"; - Type[] inheritances = []; - foreach (var attr in type.GetCustomAttributes()) { - if (attr.GetType().Name.Contains(inheritsAttribute)) { - inheritances = [.. inheritances, attr.GetType().GetGenericArguments()[0]]; + public Type[] GetInheritsTypes() { + const string inheritsAttribute = "InheritsAttribute`1"; + Type[] inheritances = []; + foreach (var attr in type.GetCustomAttributes()) { + if (attr.GetType().Name.Contains(inheritsAttribute)) { + inheritances = [.. inheritances, attr.GetType().GetGenericArguments()[0]]; + } } + return inheritances; } - return inheritances; - } - public static bool IsInheritance(this Type type, FieldInfo field) { - var inheritances = type.GetInheritsTypes(); - if (inheritances.Length == 0) return false; - foreach (var inheritance in inheritances) { - if (inheritance.IsFieldInType(field)) return true; - if (IsInheritance(inheritance, field)) return true; + public bool IsInheritance(FieldInfo field) { + var inheritances = type.GetInheritsTypes(); + if (inheritances.Length == 0) return false; + foreach (var inheritance in inheritances) { + if (inheritance.IsFieldInType(field)) return true; + if (IsInheritance(inheritance, field)) return true; + } + return false; } - return false; - } - - public static bool IsFieldInType(this Type type, FieldInfo field) { - var nameStrings = field.Name.Split('_'); - var index = Array.IndexOf(nameStrings, type.Name); - if (index <= 0) return type.GetFields(ExporterStatics.BindingFlags).Any(f => f.Name == field.Name && f.FieldType == field.FieldType); - var name = string.Join("_", nameStrings[(index + 1)..]); - return type.GetFields(ExporterStatics.BindingFlags).Any(f => f.Name == name && f.FieldType == field.FieldType) || type.GetFields(ExporterStatics.BindingFlags).Any(f => f.Name == field.Name && f.FieldType == field.FieldType); - } - - public static string FixTypeName(this Type type, bool shouldLower = true) { - using var builderPooled = StringBuilderPool.Get(500); - var builder = builderPooled.Builder; - var name = type switch { - _ when type == typeof(void) => "void", - _ when type == typeof(char) => "wchar_t", - _ when type == typeof(CStringPointer) => "char*", - _ when type == typeof(float) => "float", - _ when type == typeof(double) => "double", - _ when type == typeof(byte) || type == typeof(bool) => "size8_t", - _ when type == typeof(sbyte) => "size8_st", - _ when type == typeof(ushort) || type == typeof(Half) => "size16_t", - _ when type == typeof(short) => "size16_st", - _ when type == typeof(uint) => "size32_t", - _ when type == typeof(int) => "size32_st", - _ when type == typeof(long) || type == typeof(nint) => "size64_st", - _ when type == typeof(ulong) || type == typeof(nuint) => "size64_t", - _ when type.IsPointer() => $"{FixTypeName(type.GetElementType()!, shouldLower)}*", // able to handle infinite amount of pointers until stack size says no but this should never happen - _ => type.SanitizeName() - }; - return builder.Append(name).Replace("+", ExporterStatics.Separator).Replace(".", ExporterStatics.Separator).ToString(); - } - public static int SizeOf(this Type type) { - return type switch { - _ when type == typeof(sbyte) || type == typeof(byte) || type == typeof(bool) => 1, - _ when type == typeof(char) || type == typeof(short) || type == typeof(ushort) || type == typeof(Half) => 2, - _ when type == typeof(int) || type == typeof(uint) || type == typeof(float) => 4, - _ when type == typeof(long) || type == typeof(ulong) || type == typeof(double) || type.IsPointer || type.IsFunctionPointer || type.IsUnmanagedFunctionPointer || (type.Name == "Pointer`1" && type.Namespace.AsSpan().SequenceEqual(ExporterStatics.InteropNamespacePrefix)) || type == typeof(CStringPointer) => 8, - _ when type.Name.StartsWith("FixedSizeArray") => type.GetGenericArguments()[0].SizeOf() * int.Parse(type.Name[14..type.Name.IndexOf('`')]), - _ when type.GetCustomAttribute() is { Length: var length } => type.GetGenericArguments()[0].SizeOf() * length, - _ when type.IsStruct() && !type.IsGenericType && (type.StructLayoutAttribute?.Value ?? LayoutKind.Sequential) != LayoutKind.Sequential => type.StructLayoutAttribute?.Size ?? (int?)typeof(Unsafe).GetMethod("SizeOf")?.MakeGenericMethod(type).Invoke(null, null) ?? 0, - _ when type.IsEnum => Enum.GetUnderlyingType(type).SizeOf(), - _ when type.IsGenericType => Marshal.SizeOf(Activator.CreateInstance(type)!), - _ => GetSizeOf(type) - }; - } + public bool IsFieldInType(FieldInfo field) { + var nameStrings = field.Name.Split('_'); + var index = Array.IndexOf(nameStrings, type.Name); + if (index <= 0) return type.GetFields(ExporterStatics.BindingFlags).Any(f => f.Name == field.Name && f.FieldType == field.FieldType); + var name = string.Join("_", nameStrings[(index + 1)..]); + return type.GetFields(ExporterStatics.BindingFlags).Any(f => f.Name == name && f.FieldType == field.FieldType) || type.GetFields(ExporterStatics.BindingFlags).Any(f => f.Name == field.Name && f.FieldType == field.FieldType); + } - private static int GetSizeOf(this Type type) { - try { - return Marshal.SizeOf(Activator.CreateInstance(type)!); - } catch { - return 0; + public string FixTypeName(bool shouldLower = true) { + using var builderPooled = StringBuilderPool.Get(500); + var builder = builderPooled.Builder; + var name = type switch { + _ when type == typeof(void) => "void", + _ when type == typeof(char) => "wchar_t", + _ when type == typeof(CStringPointer) => "char*", + _ when type == typeof(float) => "float", + _ when type == typeof(double) => "double", + _ when type == typeof(bool) => "bool", + _ when type == typeof(byte) => "uint8_t", + _ when type == typeof(sbyte) => "int8_t", + _ when type == typeof(ushort) || type == typeof(Half) => "uint16_t", + _ when type == typeof(short) => "int16_t", + _ when type == typeof(uint) => "uint32_t", + _ when type == typeof(int) => "int32_t", + _ when type == typeof(long) || type == typeof(nint) => "int64_t", + _ when type == typeof(ulong) || type == typeof(nuint) => "uint64_t", + _ when type.IsPointer() && !type.IsGenericPointer() => $"{type.GetElementType()!.FixTypeName(shouldLower)}*", // able to handle infinite amount of pointers until stack size says no but this should never happen + _ when type.IsGenericPointer() => $"{type.GenericTypeArguments[0].FixTypeName(shouldLower)}*", + _ => type.SanitizeName() + }; + return builder.Append(name).Replace("+", ExporterStatics.Separator).Replace(".", ExporterStatics.Separator).ToString(); } - } - public static string GetNamespace(this Type type) { - var ns = type.Namespace!; - var offset = ns.IndexOf('.', ns.IndexOf('.') + 1) + 1; - return offset == 0 ? "" : ns[offset..]; - } + public int SizeOf() { + return type switch { + _ when type == typeof(sbyte) || type == typeof(byte) || type == typeof(bool) => 1, + _ when type == typeof(char) || type == typeof(short) || type == typeof(ushort) || type == typeof(Half) => 2, + _ when type == typeof(int) || type == typeof(uint) || type == typeof(float) => 4, + _ when type == typeof(long) || type == typeof(ulong) || type == typeof(double) || type.IsPointer || type.IsFunctionPointer || type.IsUnmanagedFunctionPointer || (type.Name == "Pointer`1" && type.Namespace.AsSpan().SequenceEqual(ExporterStatics.InteropNamespacePrefix)) || type == typeof(CStringPointer) => 8, + _ when type.Name.StartsWith("FixedSizeArray") => type.GetGenericArguments()[0].SizeOf() * int.Parse(type.Name[14..type.Name.IndexOf('`')]), + _ when type.GetCustomAttribute() is { Length: var length } => type.GetGenericArguments()[0].SizeOf() * length, + _ when type.IsStruct() && !type.IsGenericType && (type.StructLayoutAttribute?.Value ?? LayoutKind.Sequential) != LayoutKind.Sequential => type.StructLayoutAttribute?.Size ?? (int?)typeof(Unsafe).GetMethod("SizeOf")?.MakeGenericMethod(type).Invoke(null, null) ?? 0, + _ when type.IsEnum => Enum.GetUnderlyingType(type).SizeOf(), + _ when type.IsGenericType => Marshal.SizeOf(Activator.CreateInstance(type)!), + _ => type.GetSizeOf() + }; + } - public static string GetFullname(this Type type) { - return type.Namespace + "." + type.Name; - } - private static bool IsHavokStdXivOrInterop(this Type type) { - var ns = type.Namespace.AsSpan(); - const string cs = "FFXIVClientStructs."; - if (!ns.StartsWith(cs)) - return false; - ns = ns[cs.Length..]; + private int GetSizeOf() { + try { + return Marshal.SizeOf(Activator.CreateInstance(type)!); + } catch { + return 0; + } + } - return ns.StartsWith("FFXIV") - || ns.StartsWith("STD") - || ns.StartsWith("Interop") - || ns.StartsWith("Havok"); - } + public string GetNamespace() { + var ns = type.Namespace!; + var offset = ns.IndexOf('.', ns.IndexOf('.') + 1) + 1; + return offset == 0 ? "" : ns[offset..]; + } - public static Type GetPointerType(this Type type) { - while (type.IsPointer()) { - if (type.IsPointer) type = type.GetElementType()!; - else if (type.IsGenericPointer()) type = type.GenericTypeArguments[0]; - else if (type.IsFunctionPointer) type = type.GetFunctionPointerReturnType(); - else if (type.IsUnmanagedFunctionPointer) type = type.GetFunctionPointerReturnType(); + public string GetFullname() { + return type.Namespace + "." + type.Name; } - return type; - } - public static bool IsPointer(this Type type) { - return type.IsPointer || type.IsFunctionPointer || type.IsUnmanagedFunctionPointer || type.IsGenericPointer(); - } + private bool IsHavokStdXivOrInterop() { + var ns = type.Namespace.AsSpan(); + const string cs = "FFXIVClientStructs."; + if (!ns.StartsWith(cs)) + return false; + ns = ns[cs.Length..]; - public static bool IsGenericPointer(this Type type) { - return (type.Name == "Pointer`1" && type.Namespace == ExporterStatics.InteropNamespacePrefix) || (type.Name == "hkRefPtr`1" && type.Namespace == ExporterStatics.HavokNamespacePrefix); - } + return ns.StartsWith("FFXIV") + || ns.StartsWith("STD") + || ns.StartsWith("Interop") + || ns.StartsWith("Havok"); + } - public static string SanitizeName(this Type type) { - if (type.IsPointer || type.IsFunctionPointer || type.IsUnmanagedFunctionPointer) return type.GetElementType()!.FixTypeName() + "*"; - var name = (type.FullName ?? type.GetFullname()).AsSpan(); - if (type.IsHavokStdXivOrInterop()) { - name = name[(name.IndexOf('.') + 1)..]; - name = name[(name.IndexOf('.') + 1)..]; + public Type GetPointerType() { + while (type.IsPointer()) { + if (type.IsPointer) type = type.GetElementType()!; + else if (type.IsGenericPointer()) type = type.GenericTypeArguments[0]; + else if (type.IsFunctionPointer || type.IsUnmanagedFunctionPointer) type = type.GetFunctionPointerReturnType(); + } + return type; } - if (!type.IsGenericType) return name.ToString(); - switch (type.Name) { - case "Pointer`1" when type.Namespace == ExporterStatics.InteropNamespacePrefix: - return type.GenericTypeArguments[0].FixTypeName() + "*"; - case "hkRefPtr`1" when type.Namespace == ExporterStatics.HavokNamespacePrefix: - return type.GenericTypeArguments[0].FixTypeName() + "*"; + + public bool IsPointer() { + return type.IsPointer || type.IsFunctionPointer || type.IsUnmanagedFunctionPointer || type.IsGenericPointer(); } - var afterBacktick = name[name.IndexOf('`')..]; - name = name[..^afterBacktick.Length]; - using var builderPooled = StringBuilderPool.Get(500); - var builder = builderPooled.Builder; - builder.Append(name); + public bool IsGenericPointer() { + return type is { Name: "Pointer`1", Namespace: ExporterStatics.InteropNamespacePrefix } or { Name: "hkRefPtr`1", Namespace: ExporterStatics.HavokNamespacePrefix }; + } - var me = StdNodeRegex().EnumerateMatches(afterBacktick); - if (me.MoveNext()) - builder.Append('.').Append(afterBacktick.Slice(me.Current.Index + 3, me.Current.Length - 3)); + public string SanitizeName() { + if (type.IsPointer || type.IsFunctionPointer || type.IsUnmanagedFunctionPointer) return type.GetElementType()!.FixTypeName() + "*"; + var name = (type.FullName ?? type.GetFullname()).AsSpan(); + if (type.IsHavokStdXivOrInterop()) { + name = name[(name.IndexOf('.') + 1)..]; + name = name[(name.IndexOf('.') + 1)..]; + } + if (!type.IsGenericType) return name.ToString(); + switch (type.Name) { + case "Pointer`1" when type.Namespace == ExporterStatics.InteropNamespacePrefix: + return type.GenericTypeArguments[0].FixTypeName() + "*"; + case "hkRefPtr`1" when type.Namespace == ExporterStatics.HavokNamespacePrefix: + return type.GenericTypeArguments[0].FixTypeName() + "*"; + } - builder.Append('<'); - var gta = type.GenericTypeArguments; - for (var i = 0; i < gta.Length; i++) { - if (i != 0) - builder.Append(", "); - builder.Append(gta[i].FixTypeName()); - } + var afterBacktick = name[name.IndexOf('`')..]; + name = name[..^afterBacktick.Length]; + using var builderPooled = StringBuilderPool.Get(500); + var builder = builderPooled.Builder; + builder.Append(name); - builder.Append('>'); - return builder.ToString(); - } + var me = StdNodeRegex().EnumerateMatches(afterBacktick); + if (me.MoveNext()) + builder.Append('.').Append(afterBacktick.Slice(me.Current.Index + 3, me.Current.Length - 3)); - public static int PackSize(this Type type) { - if (type.GetCustomAttribute() != null) return 1; // FixedSizeArrayAttribute is always packed to 1 as the generated struct gets generated with Pack = 1 - if (!type.IsStruct()) return type.SizeOf(); - var pack = type.StructLayoutAttribute?.Pack ?? 8; - if (pack == 0) pack = 8; - var fields = type.GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); - if (fields.Length == 0) return 1; - return fields.Max(t => Math.Min(pack, t.FieldType.PackSize())); - } + builder.Append('<'); + var gta = type.GenericTypeArguments; + for (var i = 0; i < gta.Length; i++) { + if (i != 0) + builder.Append(", "); + builder.Append(gta[i].FixTypeName()); + } - public static bool IsInStructList(this Type type, List structs) { - var name = type.FixTypeName(); - foreach (var str in structs) { - if (str.StructTypeName == name) return true; + builder.Append('>'); + return builder.ToString(); } - return false; + public int PackSize() { + if (type.GetCustomAttribute() != null) return 1; // FixedSizeArrayAttribute is always packed to 1 as the generated struct gets generated with Pack = 1 + if (!type.IsStruct()) return type.SizeOf(); + var pack = type.StructLayoutAttribute?.Pack ?? 8; + if (pack == 0) pack = 8; + var fields = type.GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + if (fields.Length == 0) return 1; + return fields.Max(t => Math.Min(pack, t.FieldType.PackSize())); + } + public bool IsInStructList(List structs) { + var name = type.FixTypeName(); + foreach (var str in structs) { + if (str.StructTypeName == name) return true; + } + return false; + } + + public bool ShouldNotExportType() => type.IsPointer() || type.IsPrimitive || type.IsFixedBuffer() || type.IsEnum || type.IsBaseType(); } + } public static class FieldInfoExtensions { From b4a214fd1df61a5830a805c2206d03389e4bc45c Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Sun, 17 May 2026 11:13:53 +0200 Subject: [PATCH 18/95] change exporter to use template types and types from cpp std library --- CExporter/Exporter.cs | 381 ++++++++++++++++++++++++----------- CExporter/ExporterStatics.cs | 1 + CExporter/Extensions.cs | 2 + 3 files changed, 262 insertions(+), 122 deletions(-) diff --git a/CExporter/Exporter.cs b/CExporter/Exporter.cs index 35cd7dea41..da2da5fdf8 100644 --- a/CExporter/Exporter.cs +++ b/CExporter/Exporter.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.Drawing; using System.IO; using System.Linq; @@ -8,12 +9,14 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using FFXIVClientStructs.Attributes; +using FFXIVClientStructs.FFXIV.Client.Game.MassivePcContent; using InteropGenerator.Runtime.Attributes; using Pastel; using YamlDotNet.Core; using YamlDotNet.Core.Events; using YamlDotNet.Serialization; using YamlDotNet.Serialization.NamingConventions; +using static FFXIVClientStructs.FFXIV.Client.System.Resource.ResourceGraph; namespace CExporter; @@ -86,7 +89,7 @@ public static void ProcessTypes(bool quiet) { while (_processType.Count > 0) { if (!quiet) Console.WriteLine($"{PassString(count)} with {_processType.Count} structs and enum types"); var tmp = _processType - .Where(t => t.Type is { IsUnmanagedFunctionPointer: false, IsFunctionPointer: false }) + .Where(t => t.Type is { IsUnmanagedFunctionPointer: false, IsFunctionPointer: false } and not { Namespace: ExporterStatics.StdNamespacePrefix or ExporterStatics.StdHelperNamespacePrefix }) .ToArray(); _processType.Clear(); foreach (var @struct in tmp) { @@ -142,7 +145,21 @@ public static void ProcessStaticFunctions(bool quiet) { if (memberFunction != null || staticAddress != null) _processType.Add(methodInfo.ReturnType); if (memberFunction == null) continue; - var memberFunctionReturnType = new ProcessedMemberFunctionReturn(methodInfo.ReturnType, null); + var returnType = methodInfo.ReturnType; + var pointerCount = 0; + while (returnType.IsPointer()) { + pointerCount++; + returnType = returnType.GetPointerType(); + } + string? overrideType = null; + if (returnType is { Namespace: ExporterStatics.StdNamespacePrefix }) { + (overrideType, var requiredTypes) = ProcessStdType(returnType); + foreach (var requiredType in requiredTypes) { + _processType.Add(requiredType); + } + overrideType += new string('*', pointerCount); + } + var memberFunctionReturnType = new ProcessedMemberFunctionReturn(methodInfo.ReturnType, overrideType); if (methodInfo.GetCustomAttribute() is not { } excelAttribute) _processType.Add(memberFunctionReturnType.Type); else @@ -200,8 +217,6 @@ public static void VerifyNoOverlap() { var checks = sizes.Where(t => t != size && t.StartOffset <= size.StartOffset).ToArray(); if (checks.Any(t => t.EndOffset > size.StartOffset && t.StartOffset != size.StartOffset)) ExporterStatics.ErrorList.Add($"Field overlap detected in {processedStruct.StructType.FixTypeName().Pastel(Color.MediumSlateBlue)} with field {size.Field.Pastel(Color.BlueViolet)}"); - if (checks.Any(t => t.StartOffset == size.StartOffset)) - ExporterStatics.ErrorList.Add($"Union field detected but not defined in {processedStruct.StructType.FixTypeName().Pastel(Color.MediumSlateBlue)} with field {size.Field.Pastel(Color.BlueViolet)}"); if (size.StartOffset >= processedStruct.StructSize) ExporterStatics.ErrorList.Add($"Field offset exceeds struct size in {processedStruct.StructType.FixTypeName().Pastel(Color.MediumSlateBlue)} with field {size.Field.Pastel(Color.BlueViolet)}"); if (size.EndOffset > processedStruct.StructSize) @@ -234,7 +249,7 @@ public static void ProcessDefinedVTables(DataDefinition def) { var offset = (int)index * 8; var vf = s.VirtualFunctions.FirstOrDefault(vf => vf.Offset == offset); if (vf == null) - s.VirtualFunctions = [.. s.VirtualFunctions, new() { VirtualFunctionName = name, Offset = offset }]; + s.VirtualFunctions = [.. s.VirtualFunctions, new() { VirtualFunctionName = name, Offset = offset, VirtualFunctionParameters = null, VirtualFunctionReturnType = null, VirtualFunctionReturnTypeOverride = null }]; else if (vf.VirtualFunctionName != name) ExporterStatics.WarningList.Add($"Virtual function name mismatch: {c.Key} vf#{index}, '{vf.VirtualFunctionName}' in CS, '{name}' in data.yml"); } @@ -245,6 +260,7 @@ public static void Write(DirectoryInfo dir) { // make sure we have all the dependencies for each struct before we write them var structs = _structs.ToFrozenDictionary(x => x.StructTypeName, x => x); var structToFullDeps = new Dictionary>(); + var structDeps = structs.SelectMany(t => ResolveFullDependencies(t.Value)).Distinct().ToArray(); _structs.Sort((a, b) => { var cmp = ResolveFullDependencies(a).Count.CompareTo(ResolveFullDependencies(b).Count); return cmp != 0 ? cmp : string.Compare(a.StructTypeName, b.StructTypeName, StringComparison.Ordinal); @@ -298,7 +314,7 @@ private static void ProcessType(ProcessingType type, bool quiet) { } } - private static (string typeOverride, Type[] typeOverrideTemplates) ProcessStdField(Type fieldType) { + private static (string typeOverride, Type[] typeOverrideTemplates) ProcessStdType(Type fieldType) { const string basicString = "BasicString"; string[] pointerValuesType = ["Vector", "Deque", "Span"]; var pointerCount = 0; @@ -327,19 +343,37 @@ private static (string typeOverride, Type[] typeOverrideTemplates) ProcessStdFie } var generics = fieldType.GenericTypeArguments; var shouldBePointer = false; - foreach(var pointerType in pointerValuesType) { + foreach (var pointerType in pointerValuesType) { if (!readOnlyNameSpan.StartsWith(pointerType)) continue; shouldBePointer = true; break; } while (stdNameSpan[^1] == '\0') stdNameSpan = stdNameSpan[..^1]; - var genericsDefine = generics.Select(t => t is { Namespace: ExporterStatics.StdNamespacePrefix } ? ProcessStdField(t) : (t.FixTypeName(), [shouldBePointer ? t.MakePointerType() : t])).ToArray(); + var genericsDefine = generics.Select(t => { + var pointerCount = 0; + while (t.IsPointer()) { + t = t.GetPointerType(); + pointerCount++; + } + + if (t is { Namespace: ExporterStatics.StdNamespacePrefix }) { + var (typeOverride, requiredTypes) = ProcessStdType(t); + return (typeOverride + new string('*', pointerCount), requiredTypes); + } + + while (pointerCount > 0) { + t = t.MakePointerType(); + pointerCount--; + } + + return (t.FixTypeName(), [shouldBePointer ? t.MakePointerType() : t]); + }).Cast<(string typeOverride, Type[] typeOverrideTemplates)>().ToArray(); return ($"{stdNameSpan}<{string.Join(',', genericsDefine.Select(t => t.typeOverride))}>{new string('*', pointerCount)}", genericsDefine.SelectMany(t => t.typeOverrideTemplates).ToArray()); } private static ProcessedField ProcessField(FieldInfo field, int offset) { - if (field.FieldType is { Namespace: ExporterStatics.StdNamespacePrefix }) { - var (typeOverride, typeOverrideTemplates) = ProcessStdField(field.FieldType); + if (field.FieldType is { Namespace: ExporterStatics.StdNamespacePrefix } && !field.FieldType.IsGenericTypeParameterOrPointer()) { + var (typeOverride, typeOverrideTemplates) = ProcessStdType(field.FieldType); return new ProcessedField { FieldType = field.FieldType, FieldOffset = field.GetFieldOffset() - offset, @@ -347,19 +381,22 @@ private static ProcessedField ProcessField(FieldInfo field, int offset) { IsBase = field.IsDirectBase(), FieldTypeOverride = typeOverride, FieldTypeOverrideTemplates = typeOverrideTemplates, - Bits = [] + Bits = [], + IsUnion = false, + UnionValues = [] }; } if (field.FieldType is { Namespace: "FFXIVClientStructs.STD.Helper", Name: "Node*" }) { - var fieldTypeOverride = $"RedBlackTree::Node<{field.FieldType.GetGenericArguments()[0].FixTypeName()}>*"; - _processType.Add(new ProcessingType(field.FieldType, fieldTypeOverride[..^1])); + var fieldTypeOverride = $"std::map<{string.Join(',', field.FieldType.GetGenericArguments()[0].GetGenericArguments().Select(t => t.FixTypeName()))}>::iterator"; return new ProcessedField { FieldType = field.FieldType, FieldOffset = field.GetFieldOffset() - offset, FieldName = field.Name, IsBase = field.IsDirectBase(), FieldTypeOverride = fieldTypeOverride, - Bits = [] + Bits = [], + IsUnion = false, + UnionValues = [] }; } if (field.FieldType.IsFunctionPointer || field.FieldType.IsUnmanagedFunctionPointer) { @@ -375,10 +412,14 @@ private static ProcessedField ProcessField(FieldInfo field, int offset) { FieldType = p, FieldOffset = -1, FieldName = 'a' + (i + 1).ToString(), - Bits = [] + Bits = [], + IsUnion = false, + UnionValues = [] }; }).ToArray(), - Bits = [] + Bits = [], + IsUnion = false, + UnionValues = [] }; } if (field.FieldType.IsFixedBuffer()) { @@ -394,7 +435,9 @@ private static ProcessedField ProcessField(FieldInfo field, int offset) { FieldOffset = field.GetFieldOffset() - offset, FieldName = field.Name, FixedSize = size, - Bits = [] + Bits = [], + IsUnion = false, + UnionValues = [] }; } var fixedType = field.FieldType.GetFields()[0].FieldType; @@ -410,7 +453,9 @@ private static ProcessedField ProcessField(FieldInfo field, int offset) { FieldName = field.Name, FixedSize = size, FieldTypeOverride = fieldOverrideType, - Bits = [] + Bits = [], + IsUnion = false, + UnionValues = [] }; } if (field.FieldType.GetCustomAttribute() != null) { @@ -418,11 +463,18 @@ private static ProcessedField ProcessField(FieldInfo field, int offset) { var elementType = field.FieldType.GetGenericArguments()[0]; var isString = field.GetCustomAttribute()?.IsString ?? false; var fieldOverrideType = elementType == typeof(nint) ? field.GetCustomAttribute()?.SheetName ?? null : null; + Type[] requiredOverrideTypes = []; if (fieldOverrideType != null) fieldOverrideType = $"Component::Exd::Sheets::{fieldOverrideType}*"; if (isString && elementType == typeof(byte)) fieldOverrideType = "char"; - _processType.Add(elementType); + if (elementType.IsPointer() && elementType.GetPointerType() is { Namespace: ExporterStatics.StdNamespacePrefix } || elementType is { Namespace: ExporterStatics.StdNamespacePrefix }) { + (fieldOverrideType, requiredOverrideTypes) = ProcessStdType(elementType); + foreach (var requiredType in requiredOverrideTypes) { + _processType.Add(requiredType); + } + } else + _processType.Add(elementType); return new ProcessedFixedField { FieldType = elementType, FieldOffset = field.GetFieldOffset() - offset, @@ -430,7 +482,10 @@ private static ProcessedField ProcessField(FieldInfo field, int offset) { FixedSize = arrLength, FixedString = isString, FieldTypeOverride = fieldOverrideType, - Bits = [] + Bits = [], + IsUnion = false, + UnionValues = [], + FieldTypeOverrideTemplates = requiredOverrideTypes }; } if (field.GetCustomAttribute() != null) { @@ -440,7 +495,9 @@ private static ProcessedField ProcessField(FieldInfo field, int offset) { FieldOffset = field.GetFieldOffset() - offset, FieldName = $"{sheetName}Sheet", FieldTypeOverride = $"Component::Exd::Sheets::{sheetName}", - Bits = [] + Bits = [], + IsUnion = false, + UnionValues = [] }; } if (field.GetCustomAttribute() != null) { @@ -450,7 +507,9 @@ private static ProcessedField ProcessField(FieldInfo field, int offset) { FieldOffset = field.GetFieldOffset() - offset, FieldName = field.Name, FieldTypeOverride = $"Component::Exd::Sheets::{sheetName}*", - Bits = [] + Bits = [], + IsUnion = false, + UnionValues = [] }; } if (field.GetCustomAttribute() is { } typeForceAttribute) { @@ -460,7 +519,9 @@ private static ProcessedField ProcessField(FieldInfo field, int offset) { FieldName = field.Name, FieldTypeOverride = typeForceAttribute.TypeName, FieldTypeOverrideCheck = typeForceAttribute.DontCheck, - Bits = [] + Bits = [], + IsUnion = false, + UnionValues = [] }; } if (field.IsBitArray()) { @@ -510,7 +571,9 @@ private static ProcessedField ProcessField(FieldInfo field, int offset) { FieldOffset = field.GetFieldOffset() - offset, FieldName = field.Name, IsBase = field.IsDirectBase(), - Bits = bits.ToArray() + Bits = bits.ToArray(), + IsUnion = false, + UnionValues = [] }; } _processType.Add(field.FieldType); @@ -519,20 +582,36 @@ private static ProcessedField ProcessField(FieldInfo field, int offset) { FieldOffset = field.GetFieldOffset() - offset, FieldName = field.Name, IsBase = field.IsDirectBase(), - Bits = [] + Bits = [], + IsUnion = false, + UnionValues = [] }; } private static ProcessedField ProcessParameter(ParameterInfo parameter) { + if (parameter.ParameterType is { Namespace: ExporterStatics.StdNamespacePrefix }) { + var (typeOverride, typeOverrideTemplates) = ProcessStdType(parameter.ParameterType); + return new ProcessedField { + FieldType = parameter.ParameterType, + FieldOffset = -1, + FieldName = parameter.Name!, + FieldTypeOverride = typeOverride, + FieldTypeOverrideTemplates = typeOverrideTemplates, + Bits = [], + IsUnion = false, + UnionValues = [] + }; + } if (parameter.ParameterType is { Namespace: "FFXIVClientStructs.STD.Helper", Name: "Node*" }) { - var fieldTypeOverride = $"RedBlackTree::Node<{parameter.ParameterType.GetGenericArguments()[0].FixTypeName()}>*"; - _processType.Add(new ProcessingType(parameter.ParameterType, fieldTypeOverride[..^1])); + var fieldTypeOverride = $"std::map<{string.Join(',', parameter.ParameterType.GetGenericArguments()[0].GetGenericArguments().Select(t => t.FixTypeName()))}>::iterator"; return new ProcessedField { FieldType = parameter.ParameterType, FieldOffset = -1, FieldName = parameter.Name!, FieldTypeOverride = fieldTypeOverride, - Bits = [] + Bits = [], + IsUnion = false, + UnionValues = [] }; } if (parameter.GetCustomAttribute() != null) { @@ -542,7 +621,9 @@ private static ProcessedField ProcessParameter(ParameterInfo parameter) { FieldOffset = -1, FieldName = parameter.Name!, FieldTypeOverride = $"Component::Exd::Sheets::{sheetName}*", - Bits = [] + Bits = [], + IsUnion = false, + UnionValues = [] }; } if (parameter.GetCustomAttribute() != null) { @@ -551,7 +632,9 @@ private static ProcessedField ProcessParameter(ParameterInfo parameter) { FieldOffset = -1, FieldName = parameter.Name!, FieldTypeOverride = parameter.GetCustomAttribute()!.TypeName, - Bits = [] + Bits = [], + IsUnion = false, + UnionValues = [] }; } _processType.Add(parameter.ParameterType); @@ -559,20 +642,36 @@ private static ProcessedField ProcessParameter(ParameterInfo parameter) { FieldType = parameter.ParameterType, FieldOffset = -1, FieldName = parameter.Name!, - Bits = [] + Bits = [], + IsUnion = false, + UnionValues = [] }; } private static ProcessedField ProcessVirtualParameter(Type parameter, int i, ParameterInfo[]? parameters) { + if (parameter is { Namespace: ExporterStatics.StdNamespacePrefix }) { + var (typeOverride, typeOverrideTemplates) = ProcessStdType(parameter); + return new ProcessedField { + FieldType = parameter, + FieldOffset = -1, + FieldName = i == 0 ? "this" : parameters?[i - 1].Name ?? $"a{i + 1}", + FieldTypeOverride = typeOverride, + FieldTypeOverrideTemplates = typeOverrideTemplates, + Bits = [], + IsUnion = false, + UnionValues = [] + }; + } if (parameter is { Namespace: "FFXIVClientStructs.STD.Helper", Name: "Node*" }) { - var fieldTypeOverride = $"RedBlackTree::Node<{parameter.GetGenericArguments()[0].FixTypeName()}>*"; - _processType.Add(new ProcessingType(parameter, fieldTypeOverride[..^1])); + var fieldTypeOverride = $"std::map<{string.Join(',', parameter.GetGenericArguments()[0].GetGenericArguments().Select(t => t.FixTypeName()))}>::iterator"; return new ProcessedField { FieldType = parameter, FieldOffset = -1, FieldName = i == 0 ? "this" : parameters?[i - 1].Name ?? $"a{i + 1}", FieldTypeOverride = fieldTypeOverride, - Bits = [] + Bits = [], + IsUnion = false, + UnionValues = [] }; } if (parameter.GetCustomAttribute() != null) { @@ -582,7 +681,9 @@ private static ProcessedField ProcessVirtualParameter(Type parameter, int i, Par FieldOffset = -1, FieldName = i == 0 ? "this" : parameters?[i - 1].Name ?? $"a{i + 1}", FieldTypeOverride = $"Component::Exd::Sheets::{sheetName}*", - Bits = [] + Bits = [], + IsUnion = false, + UnionValues = [] }; } if (i != 0 && parameters?[i - 1] is not null && parameters[i - 1].GetCustomAttribute() != null) { @@ -591,7 +692,9 @@ private static ProcessedField ProcessVirtualParameter(Type parameter, int i, Par FieldOffset = -1, FieldName = parameters[i - 1].Name ?? $"a{i + 1}", FieldTypeOverride = parameters[i - 1].GetCustomAttribute()!.TypeName, - Bits = [] + Bits = [], + IsUnion = false, + UnionValues = [] }; } _processType.Add(parameter); @@ -599,7 +702,9 @@ private static ProcessedField ProcessVirtualParameter(Type parameter, int i, Par FieldType = parameter, FieldOffset = -1, FieldName = i == 0 ? "this" : parameters?[i - 1].Name ?? $"a{i + 1}", - Bits = [] + Bits = [], + IsUnion = false, + UnionValues = [] }; } @@ -631,6 +736,10 @@ private static ProcessedField ProcessVirtualParameter(Type parameter, int i, Par } } if (!type.IsStruct() || type.IsEnum) return null; + var size = type.SizeOf(); + if (type.Name.Contains('`')) { + type = type.GetGenericTypeDefinition(); + } if (type.IsInStructList(_structs) || (overrideType?.IsInStructList(_structs) ?? false)) return null; var vtable = type.GetField("VirtualTable", ExporterStatics.BindingFlags)?.FieldType; var vtableSize = 0; @@ -642,8 +751,17 @@ private static ProcessedField ProcessVirtualParameter(Type parameter, int i, Par var parameterTypes = f.FieldType.GetFunctionPointerParameterTypes(); var memberFunction = memberFunctions.FirstOrDefault(t => t.Item1 == f.Name && t.Item2.Length == parameterTypes.Length - 1); var returnType = f.FieldType.GetFunctionPointerReturnType(); + string? returnTypeOverride = null; + Type[] requiredTypes = []; + if (returnType is { Namespace: ExporterStatics.StdNamespacePrefix }) + (returnTypeOverride, requiredTypes) = ProcessStdType(returnType); if (memberFunction?.Item3 != returnType) memberFunction = null; - _processType.Add(f.FieldType.GetFunctionPointerReturnType()); + if (returnTypeOverride != null) { + foreach (var requiredType in requiredTypes) { + _processType.Add(requiredType); + } + } else + _processType.Add(f.FieldType.GetFunctionPointerReturnType()); var vfName = f.Name; if (vfName.StartsWith("Ctor") || vfName.StartsWith("Dtor")) vfName = char.ToLowerInvariant(vfName[0]) + vfName[1..]; // lowercase ctor/dtor @@ -651,7 +769,8 @@ private static ProcessedField ProcessVirtualParameter(Type parameter, int i, Par VirtualFunctionName = vfName, Offset = f.GetFieldOffset(), VirtualFunctionReturnType = f.FieldType.GetFunctionPointerReturnType(), - VirtualFunctionParameters = parameterTypes.Select((p, i) => ProcessVirtualParameter(p, i, memberFunction?.Item2)).ToArray() + VirtualFunctionParameters = parameterTypes.Select((p, i) => ProcessVirtualParameter(p, i, memberFunction?.Item2)).ToArray(), + VirtualFunctionReturnTypeOverride = returnTypeOverride }; }).ToArray(); vtableSize = vtable.StructLayoutAttribute?.Size ?? vtableSize; @@ -669,7 +788,20 @@ private static ProcessedField ProcessVirtualParameter(Type parameter, int i, Par if (memberFunction.ReturnType.IsFunctionPointer) { memberFunctionOverrideType = "__int64"; } - var memberFunctionReturnType = new ProcessedMemberFunctionReturn(memberFunction.ReturnType, memberFunctionOverrideType); + var returnType = memberFunction.ReturnType; + var pointerCount = 0; + while (returnType.IsPointer()) { + pointerCount++; + returnType = returnType.GetPointerType(); + } + if (returnType is { Namespace: ExporterStatics.StdNamespacePrefix }) { + (memberFunctionOverrideType, var requiredTypes) = ProcessStdType(returnType); + foreach (var requiredType in requiredTypes) { + _processType.Add(requiredType); + } + memberFunctionOverrideType += new string('*', pointerCount); + } + var memberFunctionReturnType = new ProcessedMemberFunctionReturn(returnType, memberFunctionOverrideType); if (memberFunction.GetCustomAttribute() is not { } excelAttribute) _processType.Add(memberFunctionReturnType.Type); else @@ -690,7 +822,9 @@ private static ProcessedField ProcessVirtualParameter(Type parameter, int i, Par FieldType = memberFunctionClass, FieldOffset = -1, FieldName = "this", - Bits = [] + Bits = [], + IsUnion = false, + UnionValues = [] }, .. memberFunctionParameters.Select(ProcessParameter).ToArray()] }, @@ -699,94 +833,75 @@ .. memberFunctionParameters.Select(ProcessParameter).ToArray()] } var fields = type.GetFields(ExporterStatics.BindingFlags).Where(t => !type.IsInheritance(t)).ToArray(); - var unionFields = fields.Where(t => t.GetCustomAttribute() == null && t.GetCustomAttribute() == null && t.GetCustomAttribute() != null).ToArray(); - - var unionOffsets = new Dictionary(new CExporterUnionCompare()); - - if (unionFields.Any()) { - var unions = new List(); - foreach (var unionField in unionFields) { - var attr = unionField.GetCustomAttribute()!; - if (!unionOffsets.TryGetValue(attr, out var unionStartField)) { - unionOffsets[attr] = unionField; - unionStartField = unionField; - } - var index = attr.IsStruct ? unions.FindIndex(t => t.StructName == attr.Struct) : unions.FindIndex(t => t.StructName == attr.Union); - if (index == -1) { - unions.Add(new ProcessedStruct { - StructType = type, - IsUnion = !attr.IsStruct, - StructName = attr.IsStruct ? attr.Struct : attr.Union, - StructNamespace = type.FixTypeName() + (attr.IsStruct ? $"{ExporterStatics.Separator}{attr.Union}" : ""), - StructTypeName = type.FixTypeName() + $"{ExporterStatics.Separator}{attr.Union}{(attr.IsStruct ? $"{ExporterStatics.Separator}{attr.Struct}" : "")}", - StructSize = 0, - VirtualFunctionSize = 0, - Fields = [ - ProcessField(unionField, unionStartField.GetFieldOffset()) - ], - MemberFunctions = [], - StructTypeOverride = type.FixTypeName() + $"{ExporterStatics.Separator}{attr.Union}{(attr.IsStruct ? $"{ExporterStatics.Separator}{attr.Struct}" : "")}", - TemplateTypes = [] - }); - } else - unions[index].Fields = [ - .. unions[index].Fields, - ProcessField(unionField, unionStartField.GetFieldOffset()) - ]; - _processType.Add(unionField.FieldType); - } - foreach (var t1 in unions) { - t1.StructSize = t1.Fields.Max(t => t.FieldOffset + t.FieldType.SizeOf()); - } - var subUnions = unions.Where(t => !t.IsUnion).ToArray(); - foreach (var subStruct in subUnions) { - var unionName = subStruct.StructNamespace.Split(ExporterStatics.Separator)[^1]; - var unionNamespace = subStruct.StructNamespace[..subStruct.StructNamespace.LastIndexOf(ExporterStatics.Separator, StringComparison.Ordinal)]; - var unionStructIndex = unions.FindIndex(t => t.StructNamespace == unionNamespace && t.StructName == unionName); - var unionStruct = unions[unionStructIndex]; - unionStruct.Fields = [ - .. unionStruct.Fields, - new ProcessedField { - FieldTypeOverride = subStruct.StructTypeOverride, - FieldType = type, - FieldOffset = 0, - FieldName = subStruct.StructName, - Bits = [] - } - ]; - if (!quiet) Console.WriteLine($"Processed {unionStruct.StructTypeOverride} with {unionStruct.Fields.Length} fields and methods"); - unions[unionStructIndex] = unionStruct; - } - _structs.AddRange(unions); - } + var unionInfoFields = fields.Where(t => t.GetCustomAttribute() == null && t.GetCustomAttribute() == null && t.GetCustomAttribute() != null) + .GroupBy(t => t.GetCustomAttribute()!.Union) + .ToFrozenDictionary(t => t.Key, t => t.GroupBy(k => k.GetCustomAttribute()!.Struct).OrderBy(k => k.Key).ToFrozenDictionary(k => k.Key, k => k.ToArray())); var processedStruct = new ProcessedStruct { StructType = type, IsUnion = type.GetCustomAttribute() != null, StructName = type.Name, StructNamespace = type.GetNamespace(), - StructTypeName = overrideType ?? type.FixTypeName(), - StructSize = type.SizeOf(), + StructSize = size, VirtualFunctionSize = vtableSize, Fields = ProcessFields(fields), VirtualFunctions = virtualFunctions, MemberFunctions = memberFunctionsArray, StructTypeOverride = overrideType, - TemplateTypes = [] + TemplateTypes = type.IsGenericTypeDefinition ? type.GetGenericArguments().Select(t => t.Name).ToArray() : [] }; - foreach (var (unionAttr, fieldInfo) in unionOffsets.Where(t => !t.Key.IsStruct)) { - processedStruct.Fields = [ - .. processedStruct.Fields, - new ProcessedField { - FieldType = fieldInfo.FieldType, - FieldOffset = fieldInfo.GetFieldOffset(), - FieldName = unionAttr.Union, - FieldTypeOverride = type.FixTypeName() + $"{ExporterStatics.Separator}{unionAttr.Union}", - Bits = [] + foreach (var (fieldName, unionInfos) in unionInfoFields) { + var processedUnionStructFields = new List(); + FieldInfo[] unionFields; + if (unionInfos.Count > 1) { + foreach (var unionStructName in unionInfos.Keys.Where(t => t != "")) { + unionFields = unionInfos[unionStructName]; + var offsetStart = unionFields[0].GetCustomAttribute()!.Value; + var unionStructType = $"{type.GetNamespace().Replace(".", "::")}::{type.Name}_union_{unionStructName}"; + _structs.Add(new ProcessedStruct { + MemberFunctions = [], + StructName = $"{type.Name}_union_{unionStructName}", + StructNamespace = type.GetNamespace(), + StructSize = unionInfos[""][0].FieldType.SizeOf(), + StructType = type, + TemplateTypes = [], + VirtualFunctionSize = 0, + StructTypeOverride = unionStructType, + Fields = unionFields.Select(t => ProcessField(t, offsetStart)).ToArray(), + IsUnion = false + }); + processedUnionStructFields.Add(new ProcessedFieldUnion { + FieldType = unionFields[0].FieldType, + FieldName = unionStructName, + FieldTypeOverride = unionStructType + }); } - ]; - processedStruct.Fields = [.. processedStruct.Fields.OrderBy(t => t.FieldOffset)]; + } + + unionFields = unionInfos[""]; + + var unionValues = unionFields.Select(unionField => { + (string? typeOverride, Type[] typeOverrideTemplates) = (null, []); + if (unionField.FieldType is { Namespace: ExporterStatics.StdNamespacePrefix }) + (typeOverride, typeOverrideTemplates) = ProcessStdType(unionField.FieldType); + + return (new ProcessedFieldUnion { + FieldName = unionField.Name, + FieldType = unionField.FieldType, + FieldTypeOverride = typeOverride + }, typeOverrideTemplates); + }).ToArray(); + + processedStruct.Fields = [..processedStruct.Fields, new ProcessedField { + Bits = [], + FieldName = fieldName, + FieldOffset = unionFields[0].GetCustomAttribute()!.Value, + FieldType = unionFields[0].FieldType, + IsUnion = true, + UnionValues = unionValues.Select(t => t.Item1).Concat(processedUnionStructFields).ToArray(), + FieldTypeOverrideTemplates = unionValues.SelectMany(t => t.typeOverrideTemplates).ToArray() + }]; } if (!quiet) Console.WriteLine($"Processed {processedStruct.StructTypeName} with {processedStruct.Fields.Length + (processedStruct.VirtualFunctions?.Length ?? 0) + processedStruct.MemberFunctions.Length} fields and methods"); @@ -838,6 +953,12 @@ public class ProcessedBitField { public required string Name; } +public class ProcessedFieldUnion { + public required Type FieldType; + public required string FieldName; + public required string? FieldTypeOverride; +} + public class ProcessedField { public string? FieldTypeOverride; public bool? FieldTypeOverrideCheck; @@ -845,6 +966,8 @@ public class ProcessedField { public required Type FieldType; public required string FieldName; public required int FieldOffset; + public required bool IsUnion; + public required ProcessedFieldUnion[] UnionValues; public bool IsBase; public required ProcessedBitField[] Bits; public virtual int FieldSize => FieldType.SizeOf(); @@ -872,8 +995,8 @@ public class ProcessedStaticMembers { // TODO: Export if struct is generic with types public class ProcessedStruct { - public string? StructTypeOverride; - public bool IsUnion; + public required string? StructTypeOverride; + public required bool IsUnion; public required Type StructType; public required string StructName; public required string StructNamespace; @@ -886,7 +1009,7 @@ public class ProcessedStruct { public ProcessedStaticMembers[]? StaticMembers; public required string[] TemplateTypes; [YamlIgnore] - public required string StructTypeName; + public string StructTypeName => StructTypeOverride ?? StructType.FixTypeName(); [YamlIgnore] private string[] _dependencyNames = []; [YamlIgnore] @@ -895,7 +1018,7 @@ public string[] DependencyNames { if (_dependencyNames.Length == 0) { _dependencyNames = Fields .Where(t => { - if (t.FieldTypeOverride == null) return (t.GetType() == typeof(ProcessedField) || t.GetType() == typeof(ProcessedFixedField)) && !t.FieldType.ShouldNotExportType() && !ExporterStatics.BaseTypeNames.Contains(t.FieldType.FixTypeName()); + if (t.FieldTypeOverride == null) return (t.GetType() == typeof(ProcessedField) || t.GetType() == typeof(ProcessedFixedField)) && !t.FieldType.ShouldNotExportType() && !t.FieldType.IsConstructedGenericType && !ExporterStatics.BaseTypeNames.Contains(t.FieldType.FixTypeName()); if (t.FieldTypeOverride.EndsWith('*') || t.FieldTypeOverride.StartsWith("Component::Exd::Sheets::") || ExporterStatics.BaseTypeNames.Contains(t.FieldTypeOverride)) return false; return !t.FieldTypeOverrideCheck.HasValue || !t.FieldTypeOverrideCheck.Value; }) @@ -924,8 +1047,9 @@ public class ProcessedEnum { public class ProcessedVirtualFunction { public required string VirtualFunctionName; public required int Offset; - public Type? VirtualFunctionReturnType; - public ProcessedField[]? VirtualFunctionParameters; + public required Type? VirtualFunctionReturnType; + public required ProcessedField[]? VirtualFunctionParameters; + public required string? VirtualFunctionReturnTypeOverride; } public class ProcessedMemberFunction { @@ -1037,6 +1161,19 @@ public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializ emitter.Emit(new Scalar(fix.FixedString.ToString())); break; } + if (f.IsUnion) { + emitter.Emit(new Scalar("union_values")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); + foreach (var unionValue in f.UnionValues) { + emitter.Emit(new MappingStart()); + emitter.Emit(new Scalar("type")); + emitter.Emit(new Scalar(unionValue.FieldTypeOverride ?? unionValue.FieldType.FixTypeName())); + emitter.Emit(new Scalar("name")); + emitter.Emit(new Scalar(unionValue.FieldName)); + emitter.Emit(new MappingEnd()); + } + emitter.Emit(new SequenceEnd()); + } emitter.Emit(new MappingEnd()); } @@ -1150,7 +1287,7 @@ public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializ emitter.Emit(new Scalar(v.Offset.ToString())); if (v.VirtualFunctionReturnType != null) { emitter.Emit(new Scalar("return_type")); - emitter.Emit(new Scalar(v.VirtualFunctionReturnType.FixTypeName())); + emitter.Emit(new Scalar(v.VirtualFunctionReturnTypeOverride ?? v.VirtualFunctionReturnType.FixTypeName())); } if (v.VirtualFunctionParameters != null) { emitter.Emit(new Scalar("parameters")); diff --git a/CExporter/ExporterStatics.cs b/CExporter/ExporterStatics.cs index c594369cc5..f55fcba5eb 100644 --- a/CExporter/ExporterStatics.cs +++ b/CExporter/ExporterStatics.cs @@ -11,6 +11,7 @@ public static class ExporterStatics { // ReSharper disable once InconsistentNaming public const string FFXIVNamespacePrefix = $"{nameof(FFXIVClientStructs)}.{nameof(FFXIVClientStructs.FFXIV)}"; public const string StdNamespacePrefix = $"{nameof(FFXIVClientStructs)}.{nameof(FFXIVClientStructs.STD)}"; + public const string StdHelperNamespacePrefix = $"{StdNamespacePrefix}.{nameof(FFXIVClientStructs.STD.Helper)}"; public const string InteropNamespacePrefix = $"{nameof(FFXIVClientStructs)}.{nameof(FFXIVClientStructs.Interop)}"; public const string HavokNamespacePrefix = $"{nameof(FFXIVClientStructs)}.{nameof(FFXIVClientStructs.Havok)}"; public const string StdCppNamespace = "std::"; diff --git a/CExporter/Extensions.cs b/CExporter/Extensions.cs index e004876f21..1e927e0a45 100644 --- a/CExporter/Extensions.cs +++ b/CExporter/Extensions.cs @@ -216,6 +216,8 @@ public bool IsInStructList(List structs) { } public bool ShouldNotExportType() => type.IsPointer() || type.IsPrimitive || type.IsFixedBuffer() || type.IsEnum || type.IsBaseType(); + + public bool IsGenericTypeParameterOrPointer() => type.IsGenericTypeParameter || (type.IsPointer() && type.GetPointerType().IsGenericTypeParameterOrPointer()); } } From 25419bdd50fd1da6ac0d7ad1e5e653bba42c4ed8 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Sun, 17 May 2026 12:21:32 +0200 Subject: [PATCH 19/95] cleanup test script --- scripts/test_execute.py | 223 +++++----------------------------------- 1 file changed, 25 insertions(+), 198 deletions(-) diff --git a/scripts/test_execute.py b/scripts/test_execute.py index ad60c3af51..2c404534e1 100644 --- a/scripts/test_execute.py +++ b/scripts/test_execute.py @@ -1,207 +1,34 @@ -from lumina_wrapper import get_excel_header_files -from exdschema_wrapper import create_struct_from_header_and_schema, get_exdschema_data -from structs_schema import ( - DefinedStruct, - DefinedStructFixedField, - DefinedStructField, -) +from time import time +current_time = time() -class SrcInterface(object): - def get_idc_type_from_size(self, size: int, offset=0) -> int: - if offset == 0: - offset = size - if offset % 8 == 0 and size >= 8: - return 4 - elif offset % 4 == 0 and size >= 4: - return 3 - elif offset % 2 == 0 and size >= 2: - return 2 - else: - return 1 +from structs_schema import get_yaml +from src_wrapper import SrcInterface +from io import open - def get_size_from_idc_type(self, type: int) -> int: - if type == 1: - return 1 - elif type == 2: - return 2 - elif type == 3: - return 4 - elif type == 4: - return 8 - elif type == 5: - return 4 - elif type == 6: - return 8 - else: - return 0 +predef_header = """ +#define _HAS_ITERATOR_DEBUGGING 0 +#define _ITERATOR_DEBUG_LEVEL 0 +#include +#include +#include +#include +#include +#include +#include - def get_string_from_idc_type_and_sign(self, type: int, signed: bool = False) -> str: - if type == 1: - if signed: - return "size8_st" - else: - return "size8_t" - elif type == 2: - if signed: - return "size16_st" - else: - return "size16_t" - elif type == 3: - if signed: - return "size32_st" - else: - return "size32_t" - elif type == 4: - if signed: - return "size32_st" - else: - return "size64_t" - elif type == 5: - return "float" - elif type == 6: - return "double" - else: - return "" - - def get_size_from_string(self, type: str) -> int: - """ - Gets the size of a base type string. - """ - if type == "size8_st" or type == "size8_t": - return 1 - if type == "size16_st" or type == "size16_t": - return 2 - if type == "size32_st" or type == "size32_t" or type == "float": - return 4 - if ( - type == "size64_st" - or type == "size64_t" - or type == "double" - or type.endswith("*") - ): - return 8 - return 0 - - def get_string_define_and_size(self, field: DefinedStructField | DefinedStructFixedField, namespace: str, struct_lookup: dict[str, int], enum_lookup: dict[str, int]) -> tuple[str, int]: - field_type = field.type - if field_type.startswith(namespace): - field_type = field_type.removeprefix(f"{namespace}::") - field_name = field.name - if field.type in struct_lookup: - return (f" {field_type} {field_name};\n", struct_lookup[field.type]) - elif field.type in enum_lookup: - return (f" {field_type} {field_name};\n", enum_lookup[field.type]) - else: - return (f" {field_type} {field_name};\n", self.get_size_from_string(field_type)) - - - def build_struct_string( - self, - struct: DefinedStruct, - struct_lookup: dict[str, int], - enum_lookup: dict[str, int], - ) -> tuple[str, str, list[str]]: - """Builds the srclang definition required to parse the object. - - Args: - struct (DefinedStruct): The struct data to build""" - struct_string_forward: str = "" - struct_string_define: str = "" - struct_string_vtables: list[str] = [] - struct_name = struct.type - - struct_string_forward += f"struct {struct_name};\n" - if struct.virtual_functions: - struct_string_forward += f"struct {struct_name}_vtbl;\n" - struct_string_vtables.append(f"{struct_name}_vtbl") - - struct_string_define += f"struct {struct_name}" - - prev_size = 0 - contiguous_fields = True - for field in struct.fields: - offset = field.offset - - while offset > prev_size: - contiguous_fields = False - fill_size = offset - prev_size - if self.full_padding: - flag = self.get_idc_type_from_size(prev_size) - size = self.get_size_from_idc_type(flag) - if size > fill_size: - flag = self.get_idc_type_from_size(fill_size, prev_size) - size = self.get_size_from_idc_type(flag) - - struct_string_define += f" {self.get_string_from_idc_type_and_sign(flag)} field_{prev_size:X};\n" - else: - struct_string_define += f" char field_{prev_size:X}[{fill_size}];\n" - - prev_size += size - - field_is_base = field.base and contiguous_fields - if contiguous_fields != field_is_base: - struct_string_define += " {\n" - contiguous_fields = False - field_name = ( - field.name if not field_is_base else "baseclass_{0:X}".format(offset) - ) - field_type = field.type - if isinstance(field, DefinedStructFixedField): - field_define, field_define_size = self.get_string_define_and_size(field, struct.namespace, struct_lookup, enum_lookup) - struct_string_define += f"{field_define[:-2]}[{field.size}];\n" - prev_size += field_define_size * field.size - elif field_type == "__fastcall": - struct_string_define += ( - f" {field.return_type} ({field_type} *{field_name})(" - ) - for param in field.parameters: - struct_string_define += param.type + " " + param.name + ", " - struct_string_define = struct_string_define[:-2] + ");\n" - prev_size += 8 - else: - field_define, field_define_size = self.get_string_define_and_size(field, struct.namespace, struct_lookup, enum_lookup) - struct_string_define += field_define - prev_size += field_define_size - - if struct.size is not None and struct.size != 0: - while struct.size > prev_size: - fill_size = struct.size - prev_size - if self.full_padding: - flag = self.get_idc_type_from_size(prev_size) - size = self.get_size_from_idc_type(flag) - if size > fill_size: - flag = self.get_idc_type_from_size(fill_size, prev_size) - size = self.get_size_from_idc_type(flag) - - struct_string_define += f" {self.get_string_from_idc_type_and_sign(flag)} field_{prev_size:X};\n" - else: - struct_string_define += f" char field_{prev_size:X}[{fill_size}];\n" - - prev_size += size - - struct_string_define += "};\n" - - return (struct_string_forward, struct_string_define, struct_string_vtables) - -excel_header_files = get_excel_header_files() -exdschema = get_exdschema_data("latest") - -struct_export = create_struct_from_header_and_schema( - excel_header_files["GFATE"], exdschema["GFATE"], 0, "GFATE" -) +#define DTOR(type) type *(__fastcall *Dtor)(type *__hidden self, uint8_t freeFlags); +""" src = SrcInterface() -src.full_padding = True -struct_lookup = {struct.name: struct.size for struct in struct_export.structs} -enum_lookup = {enum.name: 1 for enum in struct_export.enums} +with open("ffxiv_structs.yml") as f: + struct_export = get_yaml(f) + +header, vtables = src.build_export_string(struct_export) -forwards: list[str] = [] -defines: list[str] = [] -for struct in struct_export.structs: - forward, define, _ = src.build_struct_string(struct, struct_lookup, enum_lookup) - forwards.append(forward) - defines.append(define) +with open("test.h", "w") as f: + f.write(header) -print("".join(forwards + defines)) +running_time = time() - current_time +print(f"execution took {running_time}s") \ No newline at end of file From 34e1cbef24c43910cdfe9495f60332d3e9de2e2a Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:05:57 +0200 Subject: [PATCH 20/95] some cleanup and actually get proper template export --- CExporter/Exporter.TypeProcessing.cs | 579 ++++++++++++++ CExporter/Exporter.cs | 1066 +------------------------- CExporter/ExporterStatics.cs | 17 +- CExporter/Extensions.cs | 105 +-- CExporter/ProcessedData.cs | 138 ++++ CExporter/YamlConverters.cs | 280 +++++++ 6 files changed, 1085 insertions(+), 1100 deletions(-) create mode 100644 CExporter/Exporter.TypeProcessing.cs create mode 100644 CExporter/ProcessedData.cs create mode 100644 CExporter/YamlConverters.cs diff --git a/CExporter/Exporter.TypeProcessing.cs b/CExporter/Exporter.TypeProcessing.cs new file mode 100644 index 0000000000..de1c68d81b --- /dev/null +++ b/CExporter/Exporter.TypeProcessing.cs @@ -0,0 +1,579 @@ +using System; +using System.Collections.Frozen; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using FFXIVClientStructs.Attributes; +using InteropGenerator.Runtime.Attributes; + +namespace CExporter; + +public partial class Exporter { + private static void ProcessType(ProcessingType type, bool quiet) { + var ret = PreProcessType(type, quiet); + switch (ret) { + case null: + return; + case ProcessedStruct s: + _structs.Add(s); + break; + case ProcessedEnum e: + _enums.Add(e); + break; + } + } + + private static (string typeOverride, Type[] typeOverrideTemplates) ProcessStdType(Type fieldType) { + const string basicString = "BasicString"; + string[] pointerValuesType = ["Vector", "Deque", "Span"]; + var pointerCount = 0; + while (fieldType.IsPointer()) { + fieldType = fieldType.GetPointerType(); + pointerCount++; + } + var readOnlyNameSpan = fieldType.Name.AsSpan()[3..]; + Span stdNameSpan = stackalloc char[readOnlyNameSpan.Length + 5]; + stdNameSpan.Clear(); + ExporterStatics.StdCppNamespace.AsSpan().CopyTo(stdNameSpan); + if (readOnlyNameSpan[^2] == '`') { + for (var i = 0; i < readOnlyNameSpan.Length; i++) { + if (readOnlyNameSpan[i] == '`') break; + stdNameSpan[5 + i] = char.ToLower(readOnlyNameSpan[i]); + } + } else { + if (readOnlyNameSpan.CompareTo(basicString.AsSpan(), StringComparison.InvariantCultureIgnoreCase) != 0) { + for (var i = 0; i < readOnlyNameSpan.Length; i++) { + if (readOnlyNameSpan[i] == '`') break; + stdNameSpan[5 + i] = char.ToLower(readOnlyNameSpan[i]); + } + } else { + throw new NotImplementedException("Yell at me to fix this if it happens."); + } + } + var generics = fieldType.GenericTypeArguments; + var shouldBePointer = false; + foreach (var pointerType in pointerValuesType) { + if (!readOnlyNameSpan.StartsWith(pointerType)) continue; + shouldBePointer = true; + break; + } + while (stdNameSpan[^1] == '\0') stdNameSpan = stdNameSpan[..^1]; + if (stdNameSpan is "std::string") return ($"{stdNameSpan}", []); + var genericsDefine = generics.Select(t => { + var pointerCount = 0; + while (t.IsPointer()) { + t = t.GetPointerType(); + pointerCount++; + } + + if (t is { Namespace: ExporterStatics.StdNamespacePrefix }) { + var (typeOverride, requiredTypes) = ProcessStdType(t); + return (typeOverride + new string('*', pointerCount), requiredTypes); + } + + while (pointerCount > 0) { + t = t.MakePointerType(); + pointerCount--; + } + + return (t.FixTypeName(), [shouldBePointer ? t.MakePointerType() : t]); + }).Cast<(string typeOverride, Type[] typeOverrideTemplates)>().ToArray(); + return ($"{stdNameSpan}<{string.Join(',', genericsDefine.Select(t => t.typeOverride))}>{new string('*', pointerCount)}", genericsDefine.SelectMany(t => t.typeOverrideTemplates).ToArray()); + } + + private static ProcessedField ProcessField(FieldInfo field, int offset) { + if (field.FieldType is { Namespace: ExporterStatics.StdNamespacePrefix } && !field.FieldType.IsGenericTypeParameterOrPointer()) { + var (typeOverride, typeOverrideTemplates) = ProcessStdType(field.FieldType); + return new ProcessedField { + FieldType = field.FieldType, + FieldOffset = field.GetFieldOffset() - offset, + FieldName = field.Name, + IsBase = field.IsDirectBase(), + FieldTypeOverride = typeOverride, + FieldTypeOverrideTemplates = typeOverrideTemplates, + Bits = [], + IsUnion = false, + UnionValues = [] + }; + } + if (field.FieldType is { Namespace: "FFXIVClientStructs.STD.Helper", Name: "Node*" }) { + var fieldTypeOverride = $"std::map<{string.Join(',', field.FieldType.GetGenericArguments()[0].GetGenericArguments().Select(t => t.FixTypeName()))}>::iterator"; + return new ProcessedField { + FieldType = field.FieldType, + FieldOffset = field.GetFieldOffset() - offset, + FieldName = field.Name, + IsBase = field.IsDirectBase(), + FieldTypeOverride = fieldTypeOverride, + Bits = [], + IsUnion = false, + UnionValues = [] + }; + } + if (field.FieldType.IsFunctionPointer || field.FieldType.IsUnmanagedFunctionPointer) { + _processType.Add(field.FieldType.GetFunctionPointerReturnType()); + return new ProcessedFunctionField { + FieldType = field.FieldType, + FieldOffset = field.GetFieldOffset() - offset, + FieldName = field.Name, + FunctionReturnType = field.FieldType.GetFunctionPointerReturnType(), + FunctionParameters = field.FieldType.GetFunctionPointerParameterTypes().Select((p, i) => { + _processType.Add(p); + return new ProcessedField { + FieldType = p, + FieldOffset = -1, + FieldName = 'a' + (i + 1).ToString(), + Bits = [], + IsUnion = false, + UnionValues = [] + }; + }).ToArray(), + Bits = [], + IsUnion = false, + UnionValues = [] + }; + } + if (field.FieldType.IsFixedBuffer()) { + var arr = field.GetCustomAttributes().Where(t => t.GetType().Name.Contains("FixedSizeArrayAttribute")).ToArray(); + var size = field.FieldType.StructLayoutAttribute!.Size; + if (arr.Length != 0) { + var type = arr[0].GetType(); + size = (int)type.GetProperty("Count")!.GetValue(arr[0])!; + var elementType = type.GenericTypeArguments[0]; + _processType.Add(elementType); + return new ProcessedFixedField { + FieldType = elementType, + FieldOffset = field.GetFieldOffset() - offset, + FieldName = field.Name, + FixedSize = size, + Bits = [], + IsUnion = false, + UnionValues = [] + }; + } + var fixedType = field.FieldType.GetFields()[0].FieldType; + _processType.Add(fixedType); + if (fixedType.IsBaseType()) + size /= fixedType.SizeOf(); + var fieldOverrideType = fixedType == typeof(nint) ? field.GetCustomAttribute()?.SheetName ?? null : null; + if (fieldOverrideType != null) + fieldOverrideType = $"Component::Exd::Sheets::{fieldOverrideType}*"; + return new ProcessedFixedField { + FieldType = fixedType, + FieldOffset = field.GetFieldOffset() - offset, + FieldName = field.Name, + FixedSize = size, + FieldTypeOverride = fieldOverrideType, + Bits = [], + IsUnion = false, + UnionValues = [] + }; + } + if (field.FieldType.GetCustomAttribute() != null) { + var arrLength = field.FieldType.GetCustomAttribute()!.Length; + var elementType = field.FieldType.GetGenericArguments()[0]; + var isString = field.GetCustomAttribute()?.IsString ?? false; + var fieldOverrideType = elementType == typeof(nint) ? field.GetCustomAttribute()?.SheetName ?? null : null; + Type[] requiredOverrideTypes = []; + if (fieldOverrideType != null) + fieldOverrideType = $"Component::Exd::Sheets::{fieldOverrideType}*"; + if (isString && elementType == typeof(byte)) + fieldOverrideType = "char"; + if (elementType.IsPointer() && elementType.GetPointerType() is { Namespace: ExporterStatics.StdNamespacePrefix } || elementType is { Namespace: ExporterStatics.StdNamespacePrefix }) { + (fieldOverrideType, requiredOverrideTypes) = ProcessStdType(elementType); + foreach (var requiredType in requiredOverrideTypes) { + _processType.Add(requiredType); + } + } else + _processType.Add(elementType); + return new ProcessedFixedField { + FieldType = elementType, + FieldOffset = field.GetFieldOffset() - offset, + FieldName = field.Name[1].ToString().ToUpper() + field.Name[2..], + FixedSize = arrLength, + FixedString = isString, + FieldTypeOverride = fieldOverrideType, + Bits = [], + IsUnion = false, + UnionValues = [], + FieldTypeOverrideTemplates = requiredOverrideTypes + }; + } + if (field.GetCustomAttribute() != null) { + var sheetName = field.GetCustomAttribute()!.SheetName; + return new ProcessedField { + FieldType = field.FieldType, + FieldOffset = field.GetFieldOffset() - offset, + FieldName = $"{sheetName}Sheet", + FieldTypeOverride = $"Component::Exd::Sheets::{sheetName}", + Bits = [], + IsUnion = false, + UnionValues = [] + }; + } + if (field.GetCustomAttribute() != null) { + var sheetName = field.GetCustomAttribute()!.SheetName; + return new ProcessedField { + FieldType = field.FieldType, + FieldOffset = field.GetFieldOffset() - offset, + FieldName = field.Name, + FieldTypeOverride = $"Component::Exd::Sheets::{sheetName}*", + Bits = [], + IsUnion = false, + UnionValues = [] + }; + } + if (field.GetCustomAttribute() is { } typeForceAttribute) { + return new ProcessedField { + FieldType = field.FieldType, + FieldOffset = field.GetFieldOffset() - offset, + FieldName = field.Name, + FieldTypeOverride = typeForceAttribute.TypeName, + FieldTypeOverrideCheck = typeForceAttribute.DontCheck, + Bits = [], + IsUnion = false, + UnionValues = [] + }; + } + if (field.IsBitArray()) { + var bitsDefinition = field.GetCustomAttributes().Where(t => t.GetType().Name.Contains("BitFieldAttribute`1")).Select(t => { + var attrType = t.GetType(); + var name = (string)attrType.GetProperty("Name")!.GetValue(t)!; + var offset = (int)attrType.GetProperty("Index")!.GetValue(t)!; + var size = (int)attrType.GetProperty("Length")!.GetValue(t)!; + var type = attrType.GenericTypeArguments[0]; + _processType.Add(type); + return new ProcessedBitField { + Name = name, + Offset = offset, + Size = size, + Type = ExporterStatics.GetBestMatchFromSize(size) + }; + }).ToArray(); + var bits = new List(); + foreach (var bit in bitsDefinition) { + if (bits.Count == 0 && bit.Offset != 0) + bits.Add(new ProcessedBitField { + Name = "UnkBits0", + Offset = 0, + Size = bit.Offset, + Type = ExporterStatics.GetBestMatchFromSize(bit.Offset) + }); + else if (bit.Offset == 0) { + bits.Add(bit); + continue; + } + var last = bits.Last(); + if (last.Offset + last.Size != bit.Offset) { + var newOffset = last.Offset + last.Size; + var size = bit.Offset - newOffset; + + bits.Add(new ProcessedBitField { + Name = $"UnkBits{newOffset}", + Offset = newOffset, + Size = size, + Type = ExporterStatics.GetBestMatchFromSize(size) + }); + } + bits.Add(bit); + } + return new ProcessedField { + FieldType = field.FieldType, + FieldOffset = field.GetFieldOffset() - offset, + FieldName = field.Name, + IsBase = field.IsDirectBase(), + Bits = bits.ToArray(), + IsUnion = false, + UnionValues = [] + }; + } + _processType.Add(field.FieldType); + return new ProcessedField { + FieldType = field.FieldType, + FieldOffset = field.GetFieldOffset() - offset, + FieldName = field.Name, + IsBase = field.IsDirectBase(), + Bits = [], + IsUnion = false, + UnionValues = [] + }; + } + + private static ProcessedField ProcessParameter(ParameterInfo parameter) { + return ProcessParameter( + parameter.ParameterType, + parameter.Name!, + parameter.GetCustomAttribute(), + parameter.GetCustomAttribute()); + } + + private static ProcessedField ProcessVirtualParameter(Type parameter, int i, ParameterInfo[]? parameters) { + var parameterInfo = i == 0 ? null : parameters?[i - 1]; + var name = i == 0 ? "this" : parameterInfo?.Name ?? $"a{i + 1}"; + return ProcessParameter( + parameter, + name, + parameter.GetCustomAttribute(), + parameterInfo?.GetCustomAttribute()); + } + + private static ProcessedField ProcessParameter( + Type parameterType, + string parameterName, + CExporterExcelAttribute? excelAttribute, + CExporterTypeForceAttribute? typeForceAttribute) { + string? typeOverride = null; + Type[]? typeOverrideTemplates = null; + + if (parameterType is { Namespace: ExporterStatics.StdNamespacePrefix }) { + (typeOverride, typeOverrideTemplates) = ProcessStdType(parameterType); + } else if (parameterType is { Namespace: "FFXIVClientStructs.STD.Helper", Name: "Node*" }) { + typeOverride = $"std::map<{string.Join(',', parameterType.GetGenericArguments()[0].GetGenericArguments().Select(t => t.FixTypeName()))}>::iterator"; + } else if (excelAttribute != null) { + typeOverride = $"Component::Exd::Sheets::{excelAttribute.SheetName}*"; + } else if (typeForceAttribute != null) { + typeOverride = typeForceAttribute.TypeName; + } else { + _processType.Add(parameterType); + } + + return new ProcessedField { + FieldType = parameterType, + FieldOffset = -1, + FieldName = parameterName, + FieldTypeOverride = typeOverride, + FieldTypeOverrideTemplates = typeOverrideTemplates, + Bits = [], + IsUnion = false, + UnionValues = [] + }; + } + + private static object? PreProcessType(ProcessingType processingType, bool quiet) { + var (type, overrideType) = processingType; + if (type.IsFixedBuffer() || type.IsBaseType()) return null; + while (type.IsPointer) type = type.GetElementType()!; + while (type.IsGenericPointer()) type = type.GenericTypeArguments[0]; + + if (type.IsEnum) { + if (_enums.Any(t => t.EnumType == type)) return null; + var processedEnum = new ProcessedEnum { + EnumType = type, + EnumName = type.Name, + EnumNamespace = type.GetNamespace(), + IsFlags = type.GetCustomAttribute() != null, + EnumValues = type.GetFields().Where(t => t.GetCustomAttribute() == null && t.FieldType == type).ToDictionary(f => f.Name, f => f.GetRawConstantValue()!.ToString()!) + }; + + if (!quiet) Console.WriteLine($"Processed {processedEnum.EnumNamespace}::{processedEnum.EnumName} with {processedEnum.EnumValues.Count} fields"); + _enums.Add(processedEnum); + } else if (type.IsStruct()) { + while (type.IsPointer) type = type.GetElementType()!; + while (type.IsGenericPointer()) type = type.GenericTypeArguments[0]; + if (type.IsFunctionPointer || type.IsUnmanagedFunctionPointer) { + foreach (var functionPointerParameterType in type.GetFunctionPointerParameterTypes()) { + ProcessType(functionPointerParameterType, quiet); + type = type.GetFunctionPointerReturnType(); + } + } + if (!type.IsStruct() || type.IsEnum) return null; + var size = type.SizeOf(); + if (type.Name.Contains('`')) { + type = type.GetGenericTypeDefinition(); + } + if (type.IsInStructList(_structs) || (overrideType?.IsInStructList(_structs) ?? false)) return null; + var vtable = type.GetField("VirtualTable", ExporterStatics.BindingFlags)?.FieldType; + var vtableSize = 0; + ProcessedVirtualFunction[]? virtualFunctions = null; + if (vtable != null) { + vtable = vtable.GetElementType()!; + var memberFunctions = type.GetMethods(ExporterStatics.BindingFlags).Select(t => Tuple.Create(t.Name, t.GetParameters(), t.ReturnType)).ToArray(); + virtualFunctions = vtable.GetFields(ExporterStatics.BindingFlags).Where(t => t.GetCustomAttribute() == null && t.GetCustomAttribute() == null).Select(f => { + var parameterTypes = f.FieldType.GetFunctionPointerParameterTypes(); + var memberFunction = memberFunctions.FirstOrDefault(t => t.Item1 == f.Name && t.Item2.Length == parameterTypes.Length - 1); + var returnType = f.FieldType.GetFunctionPointerReturnType(); + string? returnTypeOverride = null; + Type[] requiredTypes = []; + if (returnType is { Namespace: ExporterStatics.StdNamespacePrefix }) + (returnTypeOverride, requiredTypes) = ProcessStdType(returnType); + if (memberFunction?.Item3 != returnType) memberFunction = null; + if (returnTypeOverride != null) { + foreach (var requiredType in requiredTypes) { + _processType.Add(requiredType); + } + } else + _processType.Add(f.FieldType.GetFunctionPointerReturnType()); + return new ProcessedVirtualFunction { + VirtualFunctionName = f.Name.GetCorrectedName(), + Offset = f.GetFieldOffset(), + VirtualFunctionReturnType = f.FieldType.GetFunctionPointerReturnType(), + VirtualFunctionParameters = parameterTypes.Select((p, i) => ProcessVirtualParameter(p, i, memberFunction?.Item2)).ToArray(), + VirtualFunctionReturnTypeOverride = returnTypeOverride + }; + }).ToArray(); + vtableSize = vtable.StructLayoutAttribute?.Size ?? vtableSize; + } + + var memberFunctionClass = type.GetMember("MemberFunctionPointers", ExporterStatics.BindingFlags).FirstOrDefault()?.DeclaringType; + ProcessedMemberFunction[] memberFunctionsArray = []; + if (memberFunctionClass != null) { + var memberFunctions = memberFunctionClass.GetMethods(ExporterStatics.BindingFlags).Where(t => t.GetCustomAttribute() == null && t.GetCustomAttribute() == null).ToArray(); + foreach (var memberFunction in memberFunctions) { + var memberFunctionAddress = memberFunction.GetCustomAttribute(); + if (memberFunctionAddress == null) continue; + var memberFunctionParameters = memberFunction.GetParameters(); + string? memberFunctionOverrideType = null; + if (memberFunction.ReturnType.IsFunctionPointer) { + memberFunctionOverrideType = "__int64"; + } + var returnType = memberFunction.ReturnType; + var pointerCount = 0; + while (returnType.IsPointer()) { + pointerCount++; + returnType = returnType.GetPointerType(); + } + if (returnType is { Namespace: ExporterStatics.StdNamespacePrefix }) { + (memberFunctionOverrideType, var requiredTypes) = ProcessStdType(returnType); + foreach (var requiredType in requiredTypes) { + _processType.Add(requiredType); + } + memberFunctionOverrideType += new string('*', pointerCount); + } + var memberFunctionReturnType = new ProcessedMemberFunctionReturn(returnType, memberFunctionOverrideType); + if (memberFunction.GetCustomAttribute() is not { } excelAttribute) + _processType.Add(memberFunctionReturnType.Type); + else + memberFunctionReturnType.OverrideType = $"Component::Exd::Sheets::{excelAttribute.SheetName}*"; + memberFunctionsArray = + [ + .. memberFunctionsArray, + new ProcessedMemberFunction { + MemberFunctionSignature = memberFunctionAddress!.Signature, + MemberFunctionName = memberFunction.Name.GetCorrectedName(), + MemberFunctionReturnType = memberFunctionReturnType, + MemberFunctionParameters = [ + new ProcessedField { + FieldTypeOverride = memberFunctionClass.FixTypeName() + "*", + FieldType = memberFunctionClass, + FieldOffset = -1, + FieldName = "this", + Bits = [], + IsUnion = false, + UnionValues = [] + }, + .. memberFunctionParameters.Select(ProcessParameter).ToArray()] + }, + ]; + } + } + + var fields = type.GetFields(ExporterStatics.BindingFlags).Where(t => !type.IsInheritance(t)).ToArray(); + var unionInfoFields = fields.Where(t => t.GetCustomAttribute() == null && t.GetCustomAttribute() == null && t.GetCustomAttribute() != null) + .GroupBy(t => t.GetCustomAttribute()!.Union) + .ToFrozenDictionary(t => t.Key, t => t.GroupBy(k => k.GetCustomAttribute()!.Struct).OrderBy(k => k.Key).ToFrozenDictionary(k => k.Key, k => k.ToArray())); + + var processedStruct = new ProcessedStruct { + StructType = type, + IsUnion = type.GetCustomAttribute() != null, + StructName = type.Name, + StructNamespace = type.GetNamespace(), + StructSize = size, + VirtualFunctionSize = vtableSize, + Fields = ProcessFields(fields), + VirtualFunctions = virtualFunctions, + MemberFunctions = memberFunctionsArray, + StructTypeOverride = overrideType, + TemplateTypes = type.IsGenericTypeDefinition ? type.GetGenericArguments().Select(t => t.Name).ToArray() : [] + }; + + foreach (var (fieldName, unionInfos) in unionInfoFields) { + var processedUnionStructFields = new List(); + FieldInfo[] unionFields; + if (unionInfos.Count > 1) { + foreach (var unionStructName in unionInfos.Keys.Where(t => t != "")) { + unionFields = unionInfos[unionStructName]; + var offsetStart = unionFields[0].GetCustomAttribute()!.Value; + var unionStructType = $"{type.GetNamespace().Replace(".", "::")}::{type.Name}_union_{unionStructName}"; + _structs.Add(new ProcessedStruct { + MemberFunctions = [], + StructName = $"{type.Name}_union_{unionStructName}", + StructNamespace = type.GetNamespace(), + StructSize = unionInfos[""][0].FieldType.SizeOf(), + StructType = type, + TemplateTypes = [], + VirtualFunctionSize = 0, + StructTypeOverride = unionStructType, + Fields = unionFields.Select(t => ProcessField(t, offsetStart)).ToArray(), + IsUnion = false + }); + processedUnionStructFields.Add(new ProcessedFieldUnion { + FieldType = unionFields[0].FieldType, + FieldName = unionStructName, + FieldTypeOverride = unionStructType + }); + } + } + + unionFields = unionInfos[""]; + + var unionValues = unionFields.Select(unionField => { + (string? typeOverride, Type[] typeOverrideTemplates) = (null, []); + if (unionField.FieldType is { Namespace: ExporterStatics.StdNamespacePrefix }) + (typeOverride, typeOverrideTemplates) = ProcessStdType(unionField.FieldType); + + return (new ProcessedFieldUnion { + FieldName = unionField.Name, + FieldType = unionField.FieldType, + FieldTypeOverride = typeOverride + }, typeOverrideTemplates); + }).ToArray(); + + processedStruct.Fields = [..processedStruct.Fields, new ProcessedField { + Bits = [], + FieldName = fieldName, + FieldOffset = unionFields[0].GetCustomAttribute()!.Value, + FieldType = unionFields[0].FieldType, + IsUnion = true, + UnionValues = unionValues.Select(t => t.Item1).Concat(processedUnionStructFields).ToArray(), + FieldTypeOverrideTemplates = unionValues.SelectMany(t => t.typeOverrideTemplates).ToArray() + }]; + } + + if (!quiet) Console.WriteLine($"Processed {processedStruct.StructTypeName} with {processedStruct.Fields.Length + (processedStruct.VirtualFunctions?.Length ?? 0) + processedStruct.MemberFunctions.Length} fields and methods"); + _structs.Add(processedStruct); + } + return null; + } + + public static ProcessedField[] ProcessFields(FieldInfo[] fields) { + FieldInfo[] fieldsToProcess = fields.Where(t => !ExporterStatics.IgnoredTypeNames.Contains(t.Name) && t.GetCustomAttribute() == null && t.GetCustomAttribute() == null && t.GetCustomAttribute() == null).ToArray(); + int[][] fieldsToUse = []; + bool isExcel = false; + int currentField = 0; + while (fieldsToProcess is [{ Name: "WithOps" }] or [{ Name: "Tree" }]) { + fieldsToProcess = fieldsToProcess[0].FieldType.GetFields(ExporterStatics.BindingFlags).Where(t => !ExporterStatics.IgnoredTypeNames.Contains(t.Name) && t.GetCustomAttribute() == null && t.GetCustomAttribute() == null && t.GetCustomAttribute() == null).ToArray(); + } + for (var i = 0; i < fieldsToProcess.Length; i++) { + var field = fieldsToProcess[i]; + if (field.FieldType.ContainsGenericParameters) continue; + if (field.GetCustomAttribute() != null) + _processType.Add(field.FieldType); + if (field.GetCustomAttribute() != null) { + isExcel = true; + fieldsToUse = [.. fieldsToUse, [currentField, i + 1]]; + continue; + } + if (field.GetCustomAttribute() != null) { + if (isExcel) + currentField = i + 1; + + isExcel = false; + continue; + } + } + if (fieldsToUse.Length == 0) return fieldsToProcess.Select(f => ProcessField(f, 0)).ToArray(); + fieldsToUse = [.. fieldsToUse, [currentField, fieldsToProcess.Length]]; + return fieldsToUse.SelectMany(t => fieldsToProcess[t[0]..t[1]].Select(f => ProcessField(f, 0))).ToArray(); + } +} diff --git a/CExporter/Exporter.cs b/CExporter/Exporter.cs index da2da5fdf8..6a84ee3c0d 100644 --- a/CExporter/Exporter.cs +++ b/CExporter/Exporter.cs @@ -1,26 +1,19 @@ using System; using System.Collections.Frozen; using System.Collections.Generic; -using System.Diagnostics; using System.Drawing; using System.IO; using System.Linq; using System.Reflection; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; using FFXIVClientStructs.Attributes; -using FFXIVClientStructs.FFXIV.Client.Game.MassivePcContent; using InteropGenerator.Runtime.Attributes; using Pastel; -using YamlDotNet.Core; -using YamlDotNet.Core.Events; using YamlDotNet.Serialization; using YamlDotNet.Serialization.NamingConventions; -using static FFXIVClientStructs.FFXIV.Client.System.Resource.ResourceGraph; namespace CExporter; -public class Exporter { +public partial class Exporter { private static List _enums = []; private static List _structs = []; private static HashSet _processType = []; @@ -248,8 +241,31 @@ public static void ProcessDefinedVTables(DataDefinition def) { foreach (var (index, name) in c.Value.vfuncs) { var offset = (int)index * 8; var vf = s.VirtualFunctions.FirstOrDefault(vf => vf.Offset == offset); + ProcessedField[]? vparams = null; + Type? vreturnType = null; + if (name == "dtor") { + vreturnType = s.StructType.MakePointerType(); + vparams = [ + new ProcessedField{ + FieldType = vreturnType, + FieldName = "this", + Bits = [], + IsUnion = false, + UnionValues = [], + FieldOffset = -1 + }, + new ProcessedField{ + FieldType = typeof(byte), + FieldName = "freeFlags", + Bits = [], + IsUnion = false, + UnionValues = [], + FieldOffset = -1 + } + ]; + } if (vf == null) - s.VirtualFunctions = [.. s.VirtualFunctions, new() { VirtualFunctionName = name, Offset = offset, VirtualFunctionParameters = null, VirtualFunctionReturnType = null, VirtualFunctionReturnTypeOverride = null }]; + s.VirtualFunctions = [.. s.VirtualFunctions, new() { VirtualFunctionName = name, Offset = offset, VirtualFunctionParameters = vparams, VirtualFunctionReturnType = vreturnType, VirtualFunctionReturnTypeOverride = null }]; else if (vf.VirtualFunctionName != name) ExporterStatics.WarningList.Add($"Virtual function name mismatch: {c.Key} vf#{index}, '{vf.VirtualFunctionName}' in CS, '{name}' in data.yml"); } @@ -300,1036 +316,4 @@ HashSet ResolveFullDependencies(ProcessedStruct s) { new FileInfo(Path.Join(dir.FullName, "ffxiv_structs.yml")).WriteFile(yaml); } - private static void ProcessType(ProcessingType type, bool quiet) { - var ret = PreProcessType(type, quiet); - switch (ret) { - case null: - return; - case ProcessedStruct s: - _structs.Add(s); - break; - case ProcessedEnum e: - _enums.Add(e); - break; - } - } - - private static (string typeOverride, Type[] typeOverrideTemplates) ProcessStdType(Type fieldType) { - const string basicString = "BasicString"; - string[] pointerValuesType = ["Vector", "Deque", "Span"]; - var pointerCount = 0; - while (fieldType.IsPointer()) { - fieldType = fieldType.GetPointerType(); - pointerCount++; - } - var readOnlyNameSpan = fieldType.Name.AsSpan()[3..]; - Span stdNameSpan = stackalloc char[readOnlyNameSpan.Length + 5]; - stdNameSpan.Clear(); - ExporterStatics.StdCppNamespace.AsSpan().CopyTo(stdNameSpan); - if (readOnlyNameSpan[^2] == '`') { - for (var i = 0; i < readOnlyNameSpan.Length; i++) { - if (readOnlyNameSpan[i] == '`') break; - stdNameSpan[5 + i] = char.ToLower(readOnlyNameSpan[i]); - } - } else { - if (readOnlyNameSpan.CompareTo(basicString.AsSpan(), StringComparison.InvariantCultureIgnoreCase) != 0) { - for (var i = 0; i < readOnlyNameSpan.Length; i++) { - if (readOnlyNameSpan[i] == '`') break; - stdNameSpan[5 + i] = char.ToLower(readOnlyNameSpan[i]); - } - } else { - throw new NotImplementedException("Yell at me to fix this if it happens."); - } - } - var generics = fieldType.GenericTypeArguments; - var shouldBePointer = false; - foreach (var pointerType in pointerValuesType) { - if (!readOnlyNameSpan.StartsWith(pointerType)) continue; - shouldBePointer = true; - break; - } - while (stdNameSpan[^1] == '\0') stdNameSpan = stdNameSpan[..^1]; - var genericsDefine = generics.Select(t => { - var pointerCount = 0; - while (t.IsPointer()) { - t = t.GetPointerType(); - pointerCount++; - } - - if (t is { Namespace: ExporterStatics.StdNamespacePrefix }) { - var (typeOverride, requiredTypes) = ProcessStdType(t); - return (typeOverride + new string('*', pointerCount), requiredTypes); - } - - while (pointerCount > 0) { - t = t.MakePointerType(); - pointerCount--; - } - - return (t.FixTypeName(), [shouldBePointer ? t.MakePointerType() : t]); - }).Cast<(string typeOverride, Type[] typeOverrideTemplates)>().ToArray(); - return ($"{stdNameSpan}<{string.Join(',', genericsDefine.Select(t => t.typeOverride))}>{new string('*', pointerCount)}", genericsDefine.SelectMany(t => t.typeOverrideTemplates).ToArray()); - } - - private static ProcessedField ProcessField(FieldInfo field, int offset) { - if (field.FieldType is { Namespace: ExporterStatics.StdNamespacePrefix } && !field.FieldType.IsGenericTypeParameterOrPointer()) { - var (typeOverride, typeOverrideTemplates) = ProcessStdType(field.FieldType); - return new ProcessedField { - FieldType = field.FieldType, - FieldOffset = field.GetFieldOffset() - offset, - FieldName = field.Name, - IsBase = field.IsDirectBase(), - FieldTypeOverride = typeOverride, - FieldTypeOverrideTemplates = typeOverrideTemplates, - Bits = [], - IsUnion = false, - UnionValues = [] - }; - } - if (field.FieldType is { Namespace: "FFXIVClientStructs.STD.Helper", Name: "Node*" }) { - var fieldTypeOverride = $"std::map<{string.Join(',', field.FieldType.GetGenericArguments()[0].GetGenericArguments().Select(t => t.FixTypeName()))}>::iterator"; - return new ProcessedField { - FieldType = field.FieldType, - FieldOffset = field.GetFieldOffset() - offset, - FieldName = field.Name, - IsBase = field.IsDirectBase(), - FieldTypeOverride = fieldTypeOverride, - Bits = [], - IsUnion = false, - UnionValues = [] - }; - } - if (field.FieldType.IsFunctionPointer || field.FieldType.IsUnmanagedFunctionPointer) { - _processType.Add(field.FieldType.GetFunctionPointerReturnType()); - return new ProcessedFunctionField { - FieldType = field.FieldType, - FieldOffset = field.GetFieldOffset() - offset, - FieldName = field.Name, - FunctionReturnType = field.FieldType.GetFunctionPointerReturnType(), - FunctionParameters = field.FieldType.GetFunctionPointerParameterTypes().Select((p, i) => { - _processType.Add(p); - return new ProcessedField { - FieldType = p, - FieldOffset = -1, - FieldName = 'a' + (i + 1).ToString(), - Bits = [], - IsUnion = false, - UnionValues = [] - }; - }).ToArray(), - Bits = [], - IsUnion = false, - UnionValues = [] - }; - } - if (field.FieldType.IsFixedBuffer()) { - var arr = field.GetCustomAttributes().Where(t => t.GetType().Name.Contains("FixedSizeArrayAttribute")).ToArray(); - var size = field.FieldType.StructLayoutAttribute!.Size; - if (arr.Length != 0) { - var type = arr[0].GetType(); - size = (int)type.GetProperty("Count")!.GetValue(arr[0])!; - var elementType = type.GenericTypeArguments[0]; - _processType.Add(elementType); - return new ProcessedFixedField { - FieldType = elementType, - FieldOffset = field.GetFieldOffset() - offset, - FieldName = field.Name, - FixedSize = size, - Bits = [], - IsUnion = false, - UnionValues = [] - }; - } - var fixedType = field.FieldType.GetFields()[0].FieldType; - _processType.Add(fixedType); - if (fixedType.IsBaseType()) - size /= fixedType.SizeOf(); - var fieldOverrideType = fixedType == typeof(nint) ? field.GetCustomAttribute()?.SheetName ?? null : null; - if (fieldOverrideType != null) - fieldOverrideType = $"Component::Exd::Sheets::{fieldOverrideType}*"; - return new ProcessedFixedField { - FieldType = fixedType, - FieldOffset = field.GetFieldOffset() - offset, - FieldName = field.Name, - FixedSize = size, - FieldTypeOverride = fieldOverrideType, - Bits = [], - IsUnion = false, - UnionValues = [] - }; - } - if (field.FieldType.GetCustomAttribute() != null) { - var arrLength = field.FieldType.GetCustomAttribute()!.Length; - var elementType = field.FieldType.GetGenericArguments()[0]; - var isString = field.GetCustomAttribute()?.IsString ?? false; - var fieldOverrideType = elementType == typeof(nint) ? field.GetCustomAttribute()?.SheetName ?? null : null; - Type[] requiredOverrideTypes = []; - if (fieldOverrideType != null) - fieldOverrideType = $"Component::Exd::Sheets::{fieldOverrideType}*"; - if (isString && elementType == typeof(byte)) - fieldOverrideType = "char"; - if (elementType.IsPointer() && elementType.GetPointerType() is { Namespace: ExporterStatics.StdNamespacePrefix } || elementType is { Namespace: ExporterStatics.StdNamespacePrefix }) { - (fieldOverrideType, requiredOverrideTypes) = ProcessStdType(elementType); - foreach (var requiredType in requiredOverrideTypes) { - _processType.Add(requiredType); - } - } else - _processType.Add(elementType); - return new ProcessedFixedField { - FieldType = elementType, - FieldOffset = field.GetFieldOffset() - offset, - FieldName = field.Name[1].ToString().ToUpper() + field.Name[2..], - FixedSize = arrLength, - FixedString = isString, - FieldTypeOverride = fieldOverrideType, - Bits = [], - IsUnion = false, - UnionValues = [], - FieldTypeOverrideTemplates = requiredOverrideTypes - }; - } - if (field.GetCustomAttribute() != null) { - var sheetName = field.GetCustomAttribute()!.SheetName; - return new ProcessedField { - FieldType = field.FieldType, - FieldOffset = field.GetFieldOffset() - offset, - FieldName = $"{sheetName}Sheet", - FieldTypeOverride = $"Component::Exd::Sheets::{sheetName}", - Bits = [], - IsUnion = false, - UnionValues = [] - }; - } - if (field.GetCustomAttribute() != null) { - var sheetName = field.GetCustomAttribute()!.SheetName; - return new ProcessedField { - FieldType = field.FieldType, - FieldOffset = field.GetFieldOffset() - offset, - FieldName = field.Name, - FieldTypeOverride = $"Component::Exd::Sheets::{sheetName}*", - Bits = [], - IsUnion = false, - UnionValues = [] - }; - } - if (field.GetCustomAttribute() is { } typeForceAttribute) { - return new ProcessedField { - FieldType = field.FieldType, - FieldOffset = field.GetFieldOffset() - offset, - FieldName = field.Name, - FieldTypeOverride = typeForceAttribute.TypeName, - FieldTypeOverrideCheck = typeForceAttribute.DontCheck, - Bits = [], - IsUnion = false, - UnionValues = [] - }; - } - if (field.IsBitArray()) { - var bitsDefinition = field.GetCustomAttributes().Where(t => t.GetType().Name.Contains("BitFieldAttribute`1")).Select(t => { - var attrType = t.GetType(); - var name = (string)attrType.GetProperty("Name")!.GetValue(t)!; - var offset = (int)attrType.GetProperty("Index")!.GetValue(t)!; - var size = (int)attrType.GetProperty("Length")!.GetValue(t)!; - var type = attrType.GenericTypeArguments[0]; - _processType.Add(type); - return new ProcessedBitField { - Name = name, - Offset = offset, - Size = size, - Type = ExporterStatics.GetBestMatchFromSize(size) - }; - }).ToArray(); - var bits = new List(); - foreach (var bit in bitsDefinition) { - if (bits.Count == 0 && bit.Offset != 0) - bits.Add(new ProcessedBitField { - Name = "UnkBits0", - Offset = 0, - Size = bit.Offset, - Type = ExporterStatics.GetBestMatchFromSize(bit.Offset) - }); - else if (bit.Offset == 0) { - bits.Add(bit); - continue; - } - var last = bits.Last(); - if (last.Offset + last.Size != bit.Offset) { - var newOffset = last.Offset + last.Size; - var size = bit.Offset - newOffset; - - bits.Add(new ProcessedBitField { - Name = $"UnkBits{newOffset}", - Offset = newOffset, - Size = size, - Type = ExporterStatics.GetBestMatchFromSize(size) - }); - } - bits.Add(bit); - } - return new ProcessedField { - FieldType = field.FieldType, - FieldOffset = field.GetFieldOffset() - offset, - FieldName = field.Name, - IsBase = field.IsDirectBase(), - Bits = bits.ToArray(), - IsUnion = false, - UnionValues = [] - }; - } - _processType.Add(field.FieldType); - return new ProcessedField { - FieldType = field.FieldType, - FieldOffset = field.GetFieldOffset() - offset, - FieldName = field.Name, - IsBase = field.IsDirectBase(), - Bits = [], - IsUnion = false, - UnionValues = [] - }; - } - - private static ProcessedField ProcessParameter(ParameterInfo parameter) { - if (parameter.ParameterType is { Namespace: ExporterStatics.StdNamespacePrefix }) { - var (typeOverride, typeOverrideTemplates) = ProcessStdType(parameter.ParameterType); - return new ProcessedField { - FieldType = parameter.ParameterType, - FieldOffset = -1, - FieldName = parameter.Name!, - FieldTypeOverride = typeOverride, - FieldTypeOverrideTemplates = typeOverrideTemplates, - Bits = [], - IsUnion = false, - UnionValues = [] - }; - } - if (parameter.ParameterType is { Namespace: "FFXIVClientStructs.STD.Helper", Name: "Node*" }) { - var fieldTypeOverride = $"std::map<{string.Join(',', parameter.ParameterType.GetGenericArguments()[0].GetGenericArguments().Select(t => t.FixTypeName()))}>::iterator"; - return new ProcessedField { - FieldType = parameter.ParameterType, - FieldOffset = -1, - FieldName = parameter.Name!, - FieldTypeOverride = fieldTypeOverride, - Bits = [], - IsUnion = false, - UnionValues = [] - }; - } - if (parameter.GetCustomAttribute() != null) { - var sheetName = parameter.GetCustomAttribute()!.SheetName; - return new ProcessedField { - FieldType = parameter.ParameterType, - FieldOffset = -1, - FieldName = parameter.Name!, - FieldTypeOverride = $"Component::Exd::Sheets::{sheetName}*", - Bits = [], - IsUnion = false, - UnionValues = [] - }; - } - if (parameter.GetCustomAttribute() != null) { - return new ProcessedField { - FieldType = parameter.ParameterType, - FieldOffset = -1, - FieldName = parameter.Name!, - FieldTypeOverride = parameter.GetCustomAttribute()!.TypeName, - Bits = [], - IsUnion = false, - UnionValues = [] - }; - } - _processType.Add(parameter.ParameterType); - return new ProcessedField { - FieldType = parameter.ParameterType, - FieldOffset = -1, - FieldName = parameter.Name!, - Bits = [], - IsUnion = false, - UnionValues = [] - }; - } - - private static ProcessedField ProcessVirtualParameter(Type parameter, int i, ParameterInfo[]? parameters) { - if (parameter is { Namespace: ExporterStatics.StdNamespacePrefix }) { - var (typeOverride, typeOverrideTemplates) = ProcessStdType(parameter); - return new ProcessedField { - FieldType = parameter, - FieldOffset = -1, - FieldName = i == 0 ? "this" : parameters?[i - 1].Name ?? $"a{i + 1}", - FieldTypeOverride = typeOverride, - FieldTypeOverrideTemplates = typeOverrideTemplates, - Bits = [], - IsUnion = false, - UnionValues = [] - }; - } - if (parameter is { Namespace: "FFXIVClientStructs.STD.Helper", Name: "Node*" }) { - var fieldTypeOverride = $"std::map<{string.Join(',', parameter.GetGenericArguments()[0].GetGenericArguments().Select(t => t.FixTypeName()))}>::iterator"; - return new ProcessedField { - FieldType = parameter, - FieldOffset = -1, - FieldName = i == 0 ? "this" : parameters?[i - 1].Name ?? $"a{i + 1}", - FieldTypeOverride = fieldTypeOverride, - Bits = [], - IsUnion = false, - UnionValues = [] - }; - } - if (parameter.GetCustomAttribute() != null) { - var sheetName = parameter.GetCustomAttribute()!.SheetName; - return new ProcessedField { - FieldType = parameter, - FieldOffset = -1, - FieldName = i == 0 ? "this" : parameters?[i - 1].Name ?? $"a{i + 1}", - FieldTypeOverride = $"Component::Exd::Sheets::{sheetName}*", - Bits = [], - IsUnion = false, - UnionValues = [] - }; - } - if (i != 0 && parameters?[i - 1] is not null && parameters[i - 1].GetCustomAttribute() != null) { - return new ProcessedField { - FieldType = parameter, - FieldOffset = -1, - FieldName = parameters[i - 1].Name ?? $"a{i + 1}", - FieldTypeOverride = parameters[i - 1].GetCustomAttribute()!.TypeName, - Bits = [], - IsUnion = false, - UnionValues = [] - }; - } - _processType.Add(parameter); - return new ProcessedField { - FieldType = parameter, - FieldOffset = -1, - FieldName = i == 0 ? "this" : parameters?[i - 1].Name ?? $"a{i + 1}", - Bits = [], - IsUnion = false, - UnionValues = [] - }; - } - - private static object? PreProcessType(ProcessingType processingType, bool quiet) { - var (type, overrideType) = processingType; - if (type.IsFixedBuffer() || type.IsBaseType()) return null; - while (type.IsPointer) type = type.GetElementType()!; - while (type.IsGenericPointer()) type = type.GenericTypeArguments[0]; - - if (type.IsEnum) { - if (_enums.Any(t => t.EnumType == type)) return null; - var processedEnum = new ProcessedEnum { - EnumType = type, - EnumName = type.Name, - EnumNamespace = type.GetNamespace(), - IsFlags = type.GetCustomAttribute() != null, - EnumValues = type.GetFields().Where(t => t.GetCustomAttribute() == null && t.FieldType == type).ToDictionary(f => f.Name, f => f.GetRawConstantValue()!.ToString()!) - }; - - if (!quiet) Console.WriteLine($"Processed {processedEnum.EnumNamespace}::{processedEnum.EnumName} with {processedEnum.EnumValues.Count} fields"); - _enums.Add(processedEnum); - } else if (type.IsStruct()) { - while (type.IsPointer) type = type.GetElementType()!; - while (type.IsGenericPointer()) type = type.GenericTypeArguments[0]; - if (type.IsFunctionPointer || type.IsUnmanagedFunctionPointer) { - foreach (var functionPointerParameterType in type.GetFunctionPointerParameterTypes()) { - ProcessType(functionPointerParameterType, quiet); - type = type.GetFunctionPointerReturnType(); - } - } - if (!type.IsStruct() || type.IsEnum) return null; - var size = type.SizeOf(); - if (type.Name.Contains('`')) { - type = type.GetGenericTypeDefinition(); - } - if (type.IsInStructList(_structs) || (overrideType?.IsInStructList(_structs) ?? false)) return null; - var vtable = type.GetField("VirtualTable", ExporterStatics.BindingFlags)?.FieldType; - var vtableSize = 0; - ProcessedVirtualFunction[]? virtualFunctions = null; - if (vtable != null) { - vtable = vtable.GetElementType()!; - var memberFunctions = type.GetMethods(ExporterStatics.BindingFlags).Select(t => Tuple.Create(t.Name, t.GetParameters(), t.ReturnType)).ToArray(); - virtualFunctions = vtable.GetFields(ExporterStatics.BindingFlags).Where(t => t.GetCustomAttribute() == null && t.GetCustomAttribute() == null).Select(f => { - var parameterTypes = f.FieldType.GetFunctionPointerParameterTypes(); - var memberFunction = memberFunctions.FirstOrDefault(t => t.Item1 == f.Name && t.Item2.Length == parameterTypes.Length - 1); - var returnType = f.FieldType.GetFunctionPointerReturnType(); - string? returnTypeOverride = null; - Type[] requiredTypes = []; - if (returnType is { Namespace: ExporterStatics.StdNamespacePrefix }) - (returnTypeOverride, requiredTypes) = ProcessStdType(returnType); - if (memberFunction?.Item3 != returnType) memberFunction = null; - if (returnTypeOverride != null) { - foreach (var requiredType in requiredTypes) { - _processType.Add(requiredType); - } - } else - _processType.Add(f.FieldType.GetFunctionPointerReturnType()); - var vfName = f.Name; - if (vfName.StartsWith("Ctor") || vfName.StartsWith("Dtor")) - vfName = char.ToLowerInvariant(vfName[0]) + vfName[1..]; // lowercase ctor/dtor - return new ProcessedVirtualFunction { - VirtualFunctionName = vfName, - Offset = f.GetFieldOffset(), - VirtualFunctionReturnType = f.FieldType.GetFunctionPointerReturnType(), - VirtualFunctionParameters = parameterTypes.Select((p, i) => ProcessVirtualParameter(p, i, memberFunction?.Item2)).ToArray(), - VirtualFunctionReturnTypeOverride = returnTypeOverride - }; - }).ToArray(); - vtableSize = vtable.StructLayoutAttribute?.Size ?? vtableSize; - } - - var memberFunctionClass = type.GetMember("MemberFunctionPointers", ExporterStatics.BindingFlags).FirstOrDefault()?.DeclaringType; - ProcessedMemberFunction[] memberFunctionsArray = []; - if (memberFunctionClass != null) { - var memberFunctions = memberFunctionClass.GetMethods(ExporterStatics.BindingFlags).Where(t => t.GetCustomAttribute() == null && t.GetCustomAttribute() == null).ToArray(); - foreach (var memberFunction in memberFunctions) { - var memberFunctionAddress = memberFunction.GetCustomAttribute(); - if (memberFunctionAddress == null) continue; - var memberFunctionParameters = memberFunction.GetParameters(); - string? memberFunctionOverrideType = null; - if (memberFunction.ReturnType.IsFunctionPointer) { - memberFunctionOverrideType = "__int64"; - } - var returnType = memberFunction.ReturnType; - var pointerCount = 0; - while (returnType.IsPointer()) { - pointerCount++; - returnType = returnType.GetPointerType(); - } - if (returnType is { Namespace: ExporterStatics.StdNamespacePrefix }) { - (memberFunctionOverrideType, var requiredTypes) = ProcessStdType(returnType); - foreach (var requiredType in requiredTypes) { - _processType.Add(requiredType); - } - memberFunctionOverrideType += new string('*', pointerCount); - } - var memberFunctionReturnType = new ProcessedMemberFunctionReturn(returnType, memberFunctionOverrideType); - if (memberFunction.GetCustomAttribute() is not { } excelAttribute) - _processType.Add(memberFunctionReturnType.Type); - else - memberFunctionReturnType.OverrideType = $"Component::Exd::Sheets::{excelAttribute.SheetName}*"; - var funcName = memberFunction.Name; - if (funcName.StartsWith("Ctor") || funcName.StartsWith("Dtor")) - funcName = char.ToLowerInvariant(funcName[0]) + funcName[1..]; // lowercase ctor/dtor - memberFunctionsArray = - [ - .. memberFunctionsArray, - new ProcessedMemberFunction { - MemberFunctionSignature = memberFunctionAddress!.Signature, - MemberFunctionName = funcName, - MemberFunctionReturnType = memberFunctionReturnType, - MemberFunctionParameters = [ - new ProcessedField { - FieldTypeOverride = memberFunctionClass.FixTypeName() + "*", - FieldType = memberFunctionClass, - FieldOffset = -1, - FieldName = "this", - Bits = [], - IsUnion = false, - UnionValues = [] - }, - .. memberFunctionParameters.Select(ProcessParameter).ToArray()] - }, - ]; - } - } - - var fields = type.GetFields(ExporterStatics.BindingFlags).Where(t => !type.IsInheritance(t)).ToArray(); - var unionInfoFields = fields.Where(t => t.GetCustomAttribute() == null && t.GetCustomAttribute() == null && t.GetCustomAttribute() != null) - .GroupBy(t => t.GetCustomAttribute()!.Union) - .ToFrozenDictionary(t => t.Key, t => t.GroupBy(k => k.GetCustomAttribute()!.Struct).OrderBy(k => k.Key).ToFrozenDictionary(k => k.Key, k => k.ToArray())); - - var processedStruct = new ProcessedStruct { - StructType = type, - IsUnion = type.GetCustomAttribute() != null, - StructName = type.Name, - StructNamespace = type.GetNamespace(), - StructSize = size, - VirtualFunctionSize = vtableSize, - Fields = ProcessFields(fields), - VirtualFunctions = virtualFunctions, - MemberFunctions = memberFunctionsArray, - StructTypeOverride = overrideType, - TemplateTypes = type.IsGenericTypeDefinition ? type.GetGenericArguments().Select(t => t.Name).ToArray() : [] - }; - - foreach (var (fieldName, unionInfos) in unionInfoFields) { - var processedUnionStructFields = new List(); - FieldInfo[] unionFields; - if (unionInfos.Count > 1) { - foreach (var unionStructName in unionInfos.Keys.Where(t => t != "")) { - unionFields = unionInfos[unionStructName]; - var offsetStart = unionFields[0].GetCustomAttribute()!.Value; - var unionStructType = $"{type.GetNamespace().Replace(".", "::")}::{type.Name}_union_{unionStructName}"; - _structs.Add(new ProcessedStruct { - MemberFunctions = [], - StructName = $"{type.Name}_union_{unionStructName}", - StructNamespace = type.GetNamespace(), - StructSize = unionInfos[""][0].FieldType.SizeOf(), - StructType = type, - TemplateTypes = [], - VirtualFunctionSize = 0, - StructTypeOverride = unionStructType, - Fields = unionFields.Select(t => ProcessField(t, offsetStart)).ToArray(), - IsUnion = false - }); - processedUnionStructFields.Add(new ProcessedFieldUnion { - FieldType = unionFields[0].FieldType, - FieldName = unionStructName, - FieldTypeOverride = unionStructType - }); - } - } - - unionFields = unionInfos[""]; - - var unionValues = unionFields.Select(unionField => { - (string? typeOverride, Type[] typeOverrideTemplates) = (null, []); - if (unionField.FieldType is { Namespace: ExporterStatics.StdNamespacePrefix }) - (typeOverride, typeOverrideTemplates) = ProcessStdType(unionField.FieldType); - - return (new ProcessedFieldUnion { - FieldName = unionField.Name, - FieldType = unionField.FieldType, - FieldTypeOverride = typeOverride - }, typeOverrideTemplates); - }).ToArray(); - - processedStruct.Fields = [..processedStruct.Fields, new ProcessedField { - Bits = [], - FieldName = fieldName, - FieldOffset = unionFields[0].GetCustomAttribute()!.Value, - FieldType = unionFields[0].FieldType, - IsUnion = true, - UnionValues = unionValues.Select(t => t.Item1).Concat(processedUnionStructFields).ToArray(), - FieldTypeOverrideTemplates = unionValues.SelectMany(t => t.typeOverrideTemplates).ToArray() - }]; - } - - if (!quiet) Console.WriteLine($"Processed {processedStruct.StructTypeName} with {processedStruct.Fields.Length + (processedStruct.VirtualFunctions?.Length ?? 0) + processedStruct.MemberFunctions.Length} fields and methods"); - _structs.Add(processedStruct); - } - return null; - } - - public static ProcessedField[] ProcessFields(FieldInfo[] fields) { - FieldInfo[] fieldsToProcess = fields.Where(t => !ExporterStatics.IgnoredTypeNames.Contains(t.Name) && t.GetCustomAttribute() == null && t.GetCustomAttribute() == null && t.GetCustomAttribute() == null).ToArray(); - int[][] fieldsToUse = []; - bool isExcel = false; - int currentField = 0; - while (fieldsToProcess is [{ Name: "WithOps" }] or [{ Name: "Tree" }]) { - fieldsToProcess = fieldsToProcess[0].FieldType.GetFields(ExporterStatics.BindingFlags).Where(t => !ExporterStatics.IgnoredTypeNames.Contains(t.Name) && t.GetCustomAttribute() == null && t.GetCustomAttribute() == null && t.GetCustomAttribute() == null).ToArray(); - } - for (var i = 0; i < fieldsToProcess.Length; i++) { - var field = fieldsToProcess[i]; - if (field.GetCustomAttribute() != null) - _processType.Add(field.FieldType); - if (field.GetCustomAttribute() != null) { - isExcel = true; - fieldsToUse = [.. fieldsToUse, [currentField, i + 1]]; - continue; - } - if (field.GetCustomAttribute() != null) { - if (isExcel) - currentField = i + 1; - - isExcel = false; - continue; - } - } - if (fieldsToUse.Length == 0) return fieldsToProcess.Select(f => ProcessField(f, 0)).ToArray(); - fieldsToUse = [.. fieldsToUse, [currentField, fieldsToProcess.Length]]; - return fieldsToUse.SelectMany(t => fieldsToProcess[t[0]..t[1]].Select(f => ProcessField(f, 0))).ToArray(); - } -} - -public class YamlExport { - public required ProcessedEnum[] Enums; - public required ProcessedStruct[] Structs; -} - -public class ProcessedBitField { - public required int Size; - public required int Offset; - public required Type Type; - public required string Name; -} - -public class ProcessedFieldUnion { - public required Type FieldType; - public required string FieldName; - public required string? FieldTypeOverride; -} - -public class ProcessedField { - public string? FieldTypeOverride; - public bool? FieldTypeOverrideCheck; - public Type[]? FieldTypeOverrideTemplates; - public required Type FieldType; - public required string FieldName; - public required int FieldOffset; - public required bool IsUnion; - public required ProcessedFieldUnion[] UnionValues; - public bool IsBase; - public required ProcessedBitField[] Bits; - public virtual int FieldSize => FieldType.SizeOf(); -} - -public class ProcessedFixedField : ProcessedField { - public required int FixedSize; - public bool FixedString; - - public override int FieldSize => FixedSize * FieldType.SizeOf(); -} - -public class ProcessedFunctionField : ProcessedField { - public required Type FunctionReturnType; - public required ProcessedField[] FunctionParameters; -} - -public class ProcessedStaticMembers { - public required string Name; - public required string Signature; - public required ushort[] RelativeFollowOffsets; - public bool IsPointer; - public required Type ReturnType; -} - -// TODO: Export if struct is generic with types -public class ProcessedStruct { - public required string? StructTypeOverride; - public required bool IsUnion; - public required Type StructType; - public required string StructName; - public required string StructNamespace; - public required int StructSize; - public required ProcessedField[] Fields; - public required int VirtualFunctionSize; - public ProcessedVirtualFunction[]? VirtualFunctions; // null if there are no virtual functions, empty if there's a vtable with unknown contents - public required ProcessedMemberFunction[] MemberFunctions; - public ProcessedMemberFunction[]? StaticMemberFunctions; - public ProcessedStaticMembers[]? StaticMembers; - public required string[] TemplateTypes; - [YamlIgnore] - public string StructTypeName => StructTypeOverride ?? StructType.FixTypeName(); - [YamlIgnore] - private string[] _dependencyNames = []; - [YamlIgnore] - public string[] DependencyNames { - get { - if (_dependencyNames.Length == 0) { - _dependencyNames = Fields - .Where(t => { - if (t.FieldTypeOverride == null) return (t.GetType() == typeof(ProcessedField) || t.GetType() == typeof(ProcessedFixedField)) && !t.FieldType.ShouldNotExportType() && !t.FieldType.IsConstructedGenericType && !ExporterStatics.BaseTypeNames.Contains(t.FieldType.FixTypeName()); - if (t.FieldTypeOverride.EndsWith('*') || t.FieldTypeOverride.StartsWith("Component::Exd::Sheets::") || ExporterStatics.BaseTypeNames.Contains(t.FieldTypeOverride)) return false; - return !t.FieldTypeOverrideCheck.HasValue || !t.FieldTypeOverrideCheck.Value; - }) - .SelectMany(t => { - if (t.FieldTypeOverride?.StartsWith(ExporterStatics.StdCppNamespace) ?? false) return t.FieldTypeOverrideTemplates!.Where(k => !k.ShouldNotExportType()).Select(k => k.FixTypeName()).Where(k => !ExporterStatics.BaseTypeNames.Contains(k)); - return [t.FieldTypeOverride ?? t.FieldType.FixTypeName()]; - }).Distinct().ToArray(); - } - return _dependencyNames; - } - } - public ProcessedStruct FixOrder() { - Fields = [.. Fields.OrderBy(t => t.FieldOffset)]; - return this; - } -} - -public class ProcessedEnum { - public required Type EnumType; - public required string EnumName; - public required string EnumNamespace; - public required bool IsFlags; - public required Dictionary EnumValues; -} - -public class ProcessedVirtualFunction { - public required string VirtualFunctionName; - public required int Offset; - public required Type? VirtualFunctionReturnType; - public required ProcessedField[]? VirtualFunctionParameters; - public required string? VirtualFunctionReturnTypeOverride; -} - -public class ProcessedMemberFunction { - public required string MemberFunctionSignature; - public required string MemberFunctionName; - public required ProcessedMemberFunctionReturn MemberFunctionReturnType; - public required ProcessedField[] MemberFunctionParameters; -} - -public class ProcessedMemberFunctionReturn(Type type, string? overrideType) { - public Type Type = type; - public string? OverrideType = overrideType; - public static implicit operator ProcessedMemberFunctionReturn((Type Type, string OverrideType) value) => new(value.Type, value.OverrideType); - public static implicit operator (Type Type, string? OverrideType)(ProcessedMemberFunctionReturn value) => (value.Type, value.OverrideType); - public static implicit operator ProcessedMemberFunctionReturn(Type type) => new(type, null); - public static implicit operator Type(ProcessedMemberFunctionReturn value) => value.Type; - public static implicit operator string(ProcessedMemberFunctionReturn value) => value.OverrideType ?? value.Type.FixTypeName(); -} - -public class ProcessedEnumConverter : IYamlTypeConverter { - public bool Accepts(Type type) => type == typeof(ProcessedEnum); - public object? ReadYaml(IParser parser, Type type, ObjectDeserializer rootDeserializer) => throw new NotImplementedException(); - public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializer serializer) { - if (value is not ProcessedEnum e) return; - emitter.Emit(new MappingStart()); - emitter.Emit(new Scalar("type")); - emitter.Emit(new Scalar(e.EnumType.FixTypeName())); - emitter.Emit(new Scalar("name")); - emitter.Emit(new Scalar(e.EnumName)); - emitter.Emit(new Scalar("underlying")); - emitter.Emit(new Scalar(e.EnumType.GetEnumUnderlyingType().FixTypeName())); - emitter.Emit(new Scalar("namespace")); - emitter.Emit(new Scalar(e.EnumNamespace)); - emitter.Emit(new Scalar("flags")); - emitter.Emit(e.IsFlags ? new Scalar("True") : new Scalar("False")); - emitter.Emit(new Scalar("values")); - emitter.Emit(new MappingStart()); - foreach (var (key, val) in e.EnumValues) { - emitter.Emit(new Scalar(AnchorName.Empty, TagName.Empty, key, ScalarStyle.DoubleQuoted, true, false)); - emitter.Emit(new Scalar(val)); - } - emitter.Emit(new MappingEnd()); - emitter.Emit(new MappingEnd()); - } - - public static readonly IYamlTypeConverter Instance = new ProcessedEnumConverter(); -} - -public class ProcessedFieldConverter : IYamlTypeConverter { - public bool Accepts(Type type) => type == typeof(ProcessedField); - public object? ReadYaml(IParser parser, Type type, ObjectDeserializer rootDeserializer) => throw new NotImplementedException(); - public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializer serializer) { - if (value is not ProcessedField f) return; - emitter.Emit(new MappingStart()); - emitter.Emit(new Scalar("type")); - if (f.FieldType.IsFunctionPointer || f.FieldType.IsUnmanagedFunctionPointer) { - emitter.Emit(new Scalar("__fastcall")); - } else { - emitter.Emit(new Scalar(f.FieldTypeOverride ?? f.FieldType.FixTypeName())); - } - emitter.Emit(new Scalar("name")); - emitter.Emit(new Scalar(f.FieldName)); - if (f.FieldOffset >= 0) { - emitter.Emit(new Scalar("offset")); - emitter.Emit(new Scalar(f.FieldOffset.ToString())); - } - if (f.Bits.Length > 0) { - emitter.Emit(new Scalar("bits")); - emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); - foreach (var bit in f.Bits) { - emitter.Emit(new MappingStart()); - emitter.Emit(new Scalar("type")); - emitter.Emit(new Scalar(bit.Type.FixTypeName())); - emitter.Emit(new Scalar("offset")); - emitter.Emit(new Scalar(bit.Offset.ToString())); - emitter.Emit(new Scalar("size")); - emitter.Emit(new Scalar(bit.Size.ToString())); - emitter.Emit(new Scalar("name")); - emitter.Emit(new Scalar(bit.Name)); - emitter.Emit(new MappingEnd()); - } - emitter.Emit(new SequenceEnd()); - } - if (f.IsBase) { - emitter.Emit(new Scalar("base")); - emitter.Emit(new Scalar("true")); - } - switch (f) { - case ProcessedFunctionField func: { - emitter.Emit(new Scalar("return_type")); - emitter.Emit(new Scalar(func.FunctionReturnType.FixTypeName())); - emitter.Emit(new Scalar("parameters")); - emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); - foreach (var p in func.FunctionParameters) { - emitter.Emit(new MappingStart()); - emitter.Emit(new Scalar("type")); - emitter.Emit(new Scalar(p.FieldTypeOverride ?? p.FieldType.FixTypeName())); - emitter.Emit(new Scalar("name")); - emitter.Emit(new Scalar(p.FieldName)); - emitter.Emit(new MappingEnd()); - } - emitter.Emit(new SequenceEnd()); - break; - } - case ProcessedFixedField fix: - emitter.Emit(new Scalar("size")); - emitter.Emit(new Scalar(fix.FixedSize.ToString())); - emitter.Emit(new Scalar("is_string")); - emitter.Emit(new Scalar(fix.FixedString.ToString())); - break; - } - if (f.IsUnion) { - emitter.Emit(new Scalar("union_values")); - emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); - foreach (var unionValue in f.UnionValues) { - emitter.Emit(new MappingStart()); - emitter.Emit(new Scalar("type")); - emitter.Emit(new Scalar(unionValue.FieldTypeOverride ?? unionValue.FieldType.FixTypeName())); - emitter.Emit(new Scalar("name")); - emitter.Emit(new Scalar(unionValue.FieldName)); - emitter.Emit(new MappingEnd()); - } - emitter.Emit(new SequenceEnd()); - } - emitter.Emit(new MappingEnd()); - } - - public static readonly IYamlTypeConverter Instance = new ProcessedFieldConverter(); -} - -public class ProcessedStructConverter : IYamlTypeConverter { - public bool Accepts(Type type) => type == typeof(ProcessedStruct); - public object? ReadYaml(IParser parser, Type type, ObjectDeserializer rootDeserializer) => throw new NotImplementedException(); - public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializer serializer) { - if (value is not ProcessedStruct s) return; - emitter.Emit(new MappingStart()); - emitter.Emit(new Scalar("type")); - emitter.Emit(new Scalar(!string.IsNullOrWhiteSpace(s.StructTypeOverride) ? s.StructTypeOverride : s.StructType.FixTypeName())); - emitter.Emit(new Scalar("name")); - emitter.Emit(new Scalar(s.StructName)); - emitter.Emit(new Scalar("namespace")); - emitter.Emit(new Scalar(s.StructNamespace)); - emitter.Emit(new Scalar("union")); - emitter.Emit(new Scalar(s.IsUnion.ToString())); - if (s.StructType.StructLayoutAttribute?.Value == LayoutKind.Explicit && s.StructSize != 0) { - emitter.Emit(new Scalar("size")); - emitter.Emit(new Scalar(s.StructSize.ToString())); - } - emitter.Emit(new Scalar("fields")); - emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); - foreach (var field in s.Fields) { - ProcessedFieldConverter.Instance.WriteYaml(emitter, field, field.GetType(), serializer); - } - emitter.Emit(new SequenceEnd()); - if (s.VirtualFunctionSize != 0) { - emitter.Emit(new Scalar("vtable_size")); - emitter.Emit(new Scalar(s.VirtualFunctionSize.ToString())); - } - if (s.VirtualFunctions != null) { - emitter.Emit(new Scalar("virtual_functions")); - emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); - foreach (var virtualFunction in s.VirtualFunctions) { - ProcessedVirtualFunctionConverter.Instance.WriteYaml(emitter, virtualFunction, virtualFunction.GetType(), serializer); - } - emitter.Emit(new SequenceEnd()); - } - emitter.Emit(new Scalar("member_functions")); - emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); - foreach (var memberFunction in s.MemberFunctions) { - ProcessedMemberFunctionConverter.Instance.WriteYaml(emitter, memberFunction, memberFunction.GetType(), serializer); - } - emitter.Emit(new SequenceEnd()); - if (s.StaticMembers != null) { - emitter.Emit(new Scalar("static_members")); - emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); - foreach (var staticMember in s.StaticMembers) { - ProcessedStaticMembersConverter.Instance.WriteYaml(emitter, staticMember, staticMember.GetType(), serializer); - } - emitter.Emit(new SequenceEnd()); - } - if (s.StaticMemberFunctions != null) { - emitter.Emit(new Scalar("static_member_functions")); - emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); - foreach (var staticMemberFunction in s.StaticMemberFunctions) { - ProcessedMemberFunctionConverter.Instance.WriteYaml(emitter, staticMemberFunction, staticMemberFunction.GetType(), serializer); - } - emitter.Emit(new SequenceEnd()); - } - emitter.Emit(new Scalar("template_types")); - emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Flow)); - foreach (var templateType in s.TemplateTypes) { - emitter.Emit(new Scalar(templateType)); - } - emitter.Emit(new SequenceEnd()); - emitter.Emit(new MappingEnd()); - } - - public static readonly IYamlTypeConverter Instance = new ProcessedStructConverter(); -} - -public class ProcessedMemberFunctionConverter : IYamlTypeConverter { - public bool Accepts(Type type) => type == typeof(ProcessedMemberFunction); - public object? ReadYaml(IParser parser, Type type, ObjectDeserializer rootDeserializer) => throw new NotImplementedException(); - public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializer serializer) { - if (value is not ProcessedMemberFunction m) return; - emitter.Emit(new MappingStart()); - emitter.Emit(new Scalar("signature")); - emitter.Emit(new Scalar(m.MemberFunctionSignature)); - emitter.Emit(new Scalar("return_type")); - emitter.Emit(new Scalar(m.MemberFunctionReturnType)); - emitter.Emit(new Scalar("name")); - emitter.Emit(new Scalar(m.MemberFunctionName)); - emitter.Emit(new Scalar("parameters")); - emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); - foreach (var parameter in m.MemberFunctionParameters) { - ProcessedFieldConverter.Instance.WriteYaml(emitter, parameter, parameter.GetType(), serializer); - } - emitter.Emit(new SequenceEnd()); - emitter.Emit(new MappingEnd()); - } - - public static readonly IYamlTypeConverter Instance = new ProcessedMemberFunctionConverter(); -} - -public class ProcessedVirtualFunctionConverter : IYamlTypeConverter { - - public bool Accepts(Type type) => type == typeof(ProcessedVirtualFunction); - public object? ReadYaml(IParser parser, Type type, ObjectDeserializer rootDeserializer) => throw new NotImplementedException(); - public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializer serializer) { - if (value is not ProcessedVirtualFunction v) return; - emitter.Emit(new MappingStart()); - emitter.Emit(new Scalar("name")); - emitter.Emit(new Scalar(v.VirtualFunctionName)); - emitter.Emit(new Scalar("offset")); - emitter.Emit(new Scalar(v.Offset.ToString())); - if (v.VirtualFunctionReturnType != null) { - emitter.Emit(new Scalar("return_type")); - emitter.Emit(new Scalar(v.VirtualFunctionReturnTypeOverride ?? v.VirtualFunctionReturnType.FixTypeName())); - } - if (v.VirtualFunctionParameters != null) { - emitter.Emit(new Scalar("parameters")); - emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); - foreach (var parameter in v.VirtualFunctionParameters) { - ProcessedFieldConverter.Instance.WriteYaml(emitter, parameter, parameter.GetType(), serializer); - } - emitter.Emit(new SequenceEnd()); - } - emitter.Emit(new MappingEnd()); - } - - public static readonly IYamlTypeConverter Instance = new ProcessedVirtualFunctionConverter(); -} - -public class ProcessedStaticMembersConverter : IYamlTypeConverter { - public bool Accepts(Type type) => type == typeof(ProcessedStaticMembers); - public object? ReadYaml(IParser parser, Type type, ObjectDeserializer rootDeserializer) => throw new NotImplementedException(); - public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializer serializer) { - if (value is not ProcessedStaticMembers s) return; - emitter.Emit(new MappingStart()); - emitter.Emit(new Scalar("name")); - emitter.Emit(new Scalar(s.Name)); - emitter.Emit(new Scalar("signature")); - emitter.Emit(new Scalar(s.Signature)); - emitter.Emit(new Scalar("relative_follow_offsets")); - emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); - foreach (var offset in s.RelativeFollowOffsets) { - emitter.Emit(new Scalar(offset.ToString())); - } - emitter.Emit(new SequenceEnd()); - emitter.Emit(new Scalar("is_pointer")); - emitter.Emit(new Scalar(s.IsPointer.ToString())); - emitter.Emit(new Scalar("return_type")); - emitter.Emit(new Scalar(s.ReturnType.FixTypeName())); - emitter.Emit(new MappingEnd()); - } - - public static readonly IYamlTypeConverter Instance = new ProcessedStaticMembersConverter(); -} - -public record ProcessingType(Type Type, string? OverrideTypeName) { - public static implicit operator Type(ProcessingType t) => t.Type; - public static implicit operator ProcessingType(Type t) => new(t, null); } diff --git a/CExporter/ExporterStatics.cs b/CExporter/ExporterStatics.cs index f55fcba5eb..d4375661bf 100644 --- a/CExporter/ExporterStatics.cs +++ b/CExporter/ExporterStatics.cs @@ -26,20 +26,11 @@ public static class ExporterStatics { public static readonly string Separator = "::"; #pragma warning restore CA2211 - public static Type[] GetXIVTypes() { - var assembly = AppDomain.CurrentDomain.Load(nameof(FFXIVClientStructs)); + public static Type[] GetXIVTypes() => GetTypes(FFXIVNamespacePrefix); - Type[] definedTypes; - try { - definedTypes = assembly.DefinedTypes.Select(ti => ti.AsType()).ToArray(); - } catch (ReflectionTypeLoadException ex) { - definedTypes = ex.Types.Where(t => t != null).ToArray()!; - } - - return definedTypes.Where(t => t.FullName!.StartsWith(FFXIVNamespacePrefix) && !t.FullName.EndsWith("VirtualTable") && t.GetCustomAttribute() == null).ToArray(); - } + public static Type[] GetHavokTypes() => GetTypes(HavokNamespacePrefix); - public static Type[] GetHavokTypes() { + private static Type[] GetTypes(string namespacePrefix) { var assembly = AppDomain.CurrentDomain.Load(nameof(FFXIVClientStructs)); Type[] definedTypes; @@ -49,7 +40,7 @@ public static Type[] GetHavokTypes() { definedTypes = ex.Types.Where(t => t != null).ToArray()!; } - return definedTypes.Where(t => t.FullName!.StartsWith(HavokNamespacePrefix) && !t.FullName.EndsWith("VirtualTable") && t.GetCustomAttribute() == null).ToArray(); + return definedTypes.Where(t => t.FullName!.StartsWith(namespacePrefix) && !t.FullName.EndsWith("VirtualTable") && t.GetCustomAttribute() == null).ToArray(); } public static Type GetBestMatchFromSize(int size) => size switch { diff --git a/CExporter/Extensions.cs b/CExporter/Extensions.cs index 1e927e0a45..29b2758f21 100644 --- a/CExporter/Extensions.cs +++ b/CExporter/Extensions.cs @@ -94,6 +94,8 @@ public string FixTypeName(bool shouldLower = true) { _ when type == typeof(ulong) || type == typeof(nuint) => "uint64_t", _ when type.IsPointer() && !type.IsGenericPointer() => $"{type.GetElementType()!.FixTypeName(shouldLower)}*", // able to handle infinite amount of pointers until stack size says no but this should never happen _ when type.IsGenericPointer() => $"{type.GenericTypeArguments[0].FixTypeName(shouldLower)}*", + _ when type.IsGenericTypeParameter || type.IsGenericParameter => type.Name, + _ when type.IsConstructedGenericType || type.IsGenericTypeDefinition => type.SanitizeName(true), _ => type.SanitizeName() }; return builder.Append(name).Replace("+", ExporterStatics.Separator).Replace(".", ExporterStatics.Separator).ToString(); @@ -109,6 +111,7 @@ _ when type.Name.StartsWith("FixedSizeArray") => type.GetGenericArguments()[0].S _ when type.GetCustomAttribute() is { Length: var length } => type.GetGenericArguments()[0].SizeOf() * length, _ when type.IsStruct() && !type.IsGenericType && (type.StructLayoutAttribute?.Value ?? LayoutKind.Sequential) != LayoutKind.Sequential => type.StructLayoutAttribute?.Size ?? (int?)typeof(Unsafe).GetMethod("SizeOf")?.MakeGenericMethod(type).Invoke(null, null) ?? 0, _ when type.IsEnum => Enum.GetUnderlyingType(type).SizeOf(), + _ when type.ContainsGenericParameters => 1, _ when type.IsGenericType => Marshal.SizeOf(Activator.CreateInstance(type)!), _ => type.GetSizeOf() }; @@ -159,10 +162,10 @@ public bool IsPointer() { } public bool IsGenericPointer() { - return type is { Name: "Pointer`1", Namespace: ExporterStatics.InteropNamespacePrefix } or { Name: "hkRefPtr`1", Namespace: ExporterStatics.HavokNamespacePrefix }; + return type is { Name: "Pointer`1", Namespace: ExporterStatics.InteropNamespacePrefix } || (type.Name == "hkRefPtr`1" && type.Namespace!.StartsWith(ExporterStatics.HavokNamespacePrefix)); } - public string SanitizeName() { + public string SanitizeName(bool ignoreGenericTypes = false) { if (type.IsPointer || type.IsFunctionPointer || type.IsUnmanagedFunctionPointer) return type.GetElementType()!.FixTypeName() + "*"; var name = (type.FullName ?? type.GetFullname()).AsSpan(); if (type.IsHavokStdXivOrInterop()) { @@ -170,12 +173,8 @@ public string SanitizeName() { name = name[(name.IndexOf('.') + 1)..]; } if (!type.IsGenericType) return name.ToString(); - switch (type.Name) { - case "Pointer`1" when type.Namespace == ExporterStatics.InteropNamespacePrefix: - return type.GenericTypeArguments[0].FixTypeName() + "*"; - case "hkRefPtr`1" when type.Namespace == ExporterStatics.HavokNamespacePrefix: - return type.GenericTypeArguments[0].FixTypeName() + "*"; - } + if (type.IsGenericPointer()) + return type.GenericTypeArguments[0].FixTypeName() + "*"; var afterBacktick = name[name.IndexOf('`')..]; name = name[..^afterBacktick.Length]; @@ -187,15 +186,18 @@ public string SanitizeName() { if (me.MoveNext()) builder.Append('.').Append(afterBacktick.Slice(me.Current.Index + 3, me.Current.Length - 3)); - builder.Append('<'); - var gta = type.GenericTypeArguments; - for (var i = 0; i < gta.Length; i++) { - if (i != 0) - builder.Append(", "); - builder.Append(gta[i].FixTypeName()); + if (!ignoreGenericTypes) + { + builder.Append('<'); + var gta = type.GenericTypeArguments; + for (var i = 0; i < gta.Length; i++) { + if (i != 0) + builder.Append(", "); + builder.Append(gta[i].FixTypeName()); + } + builder.Append('>'); } - builder.Append('>'); return builder.ToString(); } public int PackSize() { @@ -218,43 +220,50 @@ public bool IsInStructList(List structs) { public bool ShouldNotExportType() => type.IsPointer() || type.IsPrimitive || type.IsFixedBuffer() || type.IsEnum || type.IsBaseType(); public bool IsGenericTypeParameterOrPointer() => type.IsGenericTypeParameter || (type.IsPointer() && type.GetPointerType().IsGenericTypeParameterOrPointer()); + + public bool IsStdDefiniton() => type.Namespace.AsSpan().SequenceEqual(ExporterStatics.StdNamespacePrefix); } } public static class FieldInfoExtensions { - public static int GetFieldOffset(this FieldInfo info) { - var attrs = info.GetCustomAttributes(typeof(FieldOffsetAttribute), false); - return attrs.Length != 0 ? attrs.Cast().Single().Value : GetFieldOffsetSequential(info); - } + extension(FieldInfo info) { + + public int GetFieldOffset() { + var attrs = info.GetCustomAttributes(typeof(FieldOffsetAttribute), false); + return attrs.Length != 0 ? attrs.Cast().Single().Value : GetFieldOffsetSequential(info); + } - public static int GetFieldOffsetSequential(this FieldInfo info) { - if (info.DeclaringType is not { } declaring) - throw new Exception($"Unable to access declaring type of field {info.Name}"); - var pack = declaring.StructLayoutAttribute?.Pack ?? 0; // Default to 0 if no pack is specified - var fields = declaring.GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); - var offset = 0; - foreach (var field in fields) { - var fieldPack = field.FieldType.PackSize(); - if (pack != 0) { - fieldPack = Math.Min(pack, field.FieldType.PackSize()); - } - offset = (offset + fieldPack - 1) / fieldPack * fieldPack; - if (field == info) { - return offset; + public int GetFieldOffsetSequential() { + if (info.DeclaringType is not { } declaring) + throw new Exception($"Unable to access declaring type of field {info.Name}"); + var pack = declaring.StructLayoutAttribute?.Pack ?? 0; // Default to 0 if no pack is specified + var fields = declaring.GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + var offset = 0; + foreach (var field in fields) { + var fieldPack = field.FieldType.PackSize(); + if (pack != 0) { + fieldPack = Math.Min(pack, field.FieldType.PackSize()); + } + offset = (offset + fieldPack - 1) / fieldPack * fieldPack; + if (field == info) { + return offset; + } + offset += field.FieldType.SizeOf(); } - offset += field.FieldType.SizeOf(); + throw new Exception("Field not found"); } - throw new Exception("Field not found"); - } - public static bool IsDirectBase(this FieldInfo field) { - var bases = field.DeclaringType?.GetInheritsTypes() ?? []; - return bases.Any(b => field.FieldType == b && field.Name == (b.Name == field.DeclaringType?.Name ? b.Name + "Base" : b.Name)) || field.GetCustomAttribute() != null; - } + public bool IsDirectBase() { + var bases = info.DeclaringType?.GetInheritsTypes() ?? []; + return bases.Any(b => info.FieldType == b && info.Name == (b.Name == info.DeclaringType?.Name ? b.Name + "Base" : b.Name)) || info.GetCustomAttribute() != null; + } + + public bool IsBitArray() { + return info.GetCustomAttributes().Any(t => t.GetType().Name.Contains("BitFieldAttribute`1")); + } - public static bool IsBitArray(this FieldInfo type) { - return type.GetCustomAttributes().Any(t => t.GetType().Name.Contains("BitFieldAttribute`1")); + public bool IsStdDefinition() => info.FieldType.IsStdDefiniton(); } } public static class Extensions { @@ -263,10 +272,14 @@ public static void WriteFile(this FileInfo file, string content) { stream.Write(content); } - public static bool IsInStructList(this string name, List structs) { - foreach (var str in structs) { - if (str.StructTypeName == name) return true; + extension(string name) { + public bool IsInStructList(List structs) { + foreach (var str in structs) { + if (str.StructTypeName == name) return true; + } + return false; } - return false; + + public string GetCorrectedName() => name is "Dtor" or "Ctor" ? name.ToLower() : name; } } diff --git a/CExporter/ProcessedData.cs b/CExporter/ProcessedData.cs new file mode 100644 index 0000000000..b11dc7bd3d --- /dev/null +++ b/CExporter/ProcessedData.cs @@ -0,0 +1,138 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using YamlDotNet.Serialization; + +namespace CExporter; + +public class YamlExport { + public required ProcessedEnum[] Enums; + public required ProcessedStruct[] Structs; +} + +public class ProcessedBitField { + public required int Size; + public required int Offset; + public required Type Type; + public required string Name; +} + +public class ProcessedFieldUnion { + public required Type FieldType; + public required string FieldName; + public required string? FieldTypeOverride; +} + +public class ProcessedField { + public string? FieldTypeOverride; + public bool? FieldTypeOverrideCheck; + public Type[]? FieldTypeOverrideTemplates; + public required Type FieldType; + public required string FieldName; + public required int FieldOffset; + public required bool IsUnion; + public required ProcessedFieldUnion[] UnionValues; + public bool IsBase; + public required ProcessedBitField[] Bits; + public virtual int FieldSize => FieldType.SizeOf(); +} + +public class ProcessedFixedField : ProcessedField { + public required int FixedSize; + public bool FixedString; + + public override int FieldSize => FixedSize * FieldType.SizeOf(); +} + +public class ProcessedFunctionField : ProcessedField { + public required Type FunctionReturnType; + public required ProcessedField[] FunctionParameters; +} + +public class ProcessedStaticMembers { + public required string Signature; + public required ushort[] RelativeFollowOffsets; + public bool IsPointer; + public required Type ReturnType; +} + +// TODO: Export if struct is generic with types +public class ProcessedStruct { + public required string? StructTypeOverride; + public required bool IsUnion; + public required Type StructType; + public required string StructName; + public required string StructNamespace; + public required int StructSize; + public required ProcessedField[] Fields; + public required int VirtualFunctionSize; + public ProcessedVirtualFunction[]? VirtualFunctions; // null if there are no virtual functions, empty if there's a vtable with unknown contents + public required ProcessedMemberFunction[] MemberFunctions; + public ProcessedMemberFunction[]? StaticMemberFunctions; + public ProcessedStaticMembers[]? StaticMembers; + public required string[] TemplateTypes; + [YamlIgnore] + public string StructTypeName => StructTypeOverride ?? StructType.FixTypeName(); + [YamlIgnore] + private string[] _dependencyNames = []; + [YamlIgnore] + public string[] DependencyNames { + get { + if (_dependencyNames.Length == 0) { + _dependencyNames = Fields + .Where(t => { + if (t.FieldTypeOverride == null) return (t.GetType() == typeof(ProcessedField) || t.GetType() == typeof(ProcessedFixedField)) && !t.FieldType.ShouldNotExportType() && !ExporterStatics.BaseTypeNames.Contains(t.FieldType.FixTypeName()) && !t.FieldType.ContainsGenericParameters; + if (t.FieldTypeOverride.EndsWith('*') || t.FieldTypeOverride.StartsWith("Component::Exd::Sheets::") || ExporterStatics.BaseTypeNames.Contains(t.FieldTypeOverride)) return false; + return !t.FieldTypeOverrideCheck.HasValue || !t.FieldTypeOverrideCheck.Value; + }) + .SelectMany(t => { + if (t.FieldTypeOverride?.StartsWith(ExporterStatics.StdCppNamespace) ?? false) return t.FieldTypeOverrideTemplates!.Where(k => !k.ShouldNotExportType()).Select(k => k.FixTypeName()).Where(k => !ExporterStatics.BaseTypeNames.Contains(k)); + return [t.FieldTypeOverride ?? t.FieldType.FixTypeName()]; + }).Distinct().ToArray(); + } + return _dependencyNames; + } + } + public ProcessedStruct FixOrder() { + Fields = [.. Fields.OrderBy(t => t.FieldOffset)]; + return this; + } +} + +public class ProcessedEnum { + public required Type EnumType; + public required string EnumName; + public required string EnumNamespace; + public required bool IsFlags; + public required Dictionary EnumValues; +} + +public class ProcessedVirtualFunction { + public required string VirtualFunctionName; + public required int Offset; + public required Type? VirtualFunctionReturnType; + public required ProcessedField[]? VirtualFunctionParameters; + public required string? VirtualFunctionReturnTypeOverride; +} + +public class ProcessedMemberFunction { + public required string MemberFunctionSignature; + public required string MemberFunctionName; + public required ProcessedMemberFunctionReturn MemberFunctionReturnType; + public required ProcessedField[] MemberFunctionParameters; +} + +public class ProcessedMemberFunctionReturn(Type type, string? overrideType) { + public Type Type = type; + public string? OverrideType = overrideType; + public static implicit operator ProcessedMemberFunctionReturn((Type Type, string OverrideType) value) => new(value.Type, value.OverrideType); + public static implicit operator (Type Type, string? OverrideType)(ProcessedMemberFunctionReturn value) => (value.Type, value.OverrideType); + public static implicit operator ProcessedMemberFunctionReturn(Type type) => new(type, null); + public static implicit operator Type(ProcessedMemberFunctionReturn value) => value.Type; + public static implicit operator string(ProcessedMemberFunctionReturn value) => value.OverrideType ?? value.Type.FixTypeName(); +} + +public record ProcessingType(Type Type, string? OverrideTypeName) { + public static implicit operator Type(ProcessingType t) => t.Type; + public static implicit operator ProcessingType(Type t) => new(t, null); +} diff --git a/CExporter/YamlConverters.cs b/CExporter/YamlConverters.cs new file mode 100644 index 0000000000..9aa8ac16cd --- /dev/null +++ b/CExporter/YamlConverters.cs @@ -0,0 +1,280 @@ +using System; +using System.Linq; +using System.Runtime.InteropServices; +using YamlDotNet.Core; +using YamlDotNet.Core.Events; +using YamlDotNet.Serialization; + +namespace CExporter; + +public class ProcessedEnumConverter : IYamlTypeConverter { + public bool Accepts(Type type) => type == typeof(ProcessedEnum); + public object? ReadYaml(IParser parser, Type type, ObjectDeserializer rootDeserializer) => throw new NotImplementedException(); + public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializer serializer) { + if (value is not ProcessedEnum e) return; + emitter.Emit(new MappingStart()); + emitter.Emit(new Scalar("type")); + emitter.Emit(new Scalar(e.EnumType.FixTypeName())); + emitter.Emit(new Scalar("name")); + emitter.Emit(new Scalar(e.EnumName)); + emitter.Emit(new Scalar("underlying")); + emitter.Emit(new Scalar(e.EnumType.GetEnumUnderlyingType().FixTypeName())); + emitter.Emit(new Scalar("namespace")); + emitter.Emit(new Scalar(e.EnumNamespace)); + emitter.Emit(new Scalar("flags")); + emitter.Emit(e.IsFlags ? new Scalar("True") : new Scalar("False")); + emitter.Emit(new Scalar("values")); + emitter.Emit(new MappingStart()); + foreach (var (key, val) in e.EnumValues) { + emitter.Emit(new Scalar(AnchorName.Empty, TagName.Empty, key, ScalarStyle.DoubleQuoted, true, false)); + emitter.Emit(new Scalar(val)); + } + emitter.Emit(new MappingEnd()); + emitter.Emit(new MappingEnd()); + } + + public static readonly IYamlTypeConverter Instance = new ProcessedEnumConverter(); +} + +public class ProcessedFieldConverter : IYamlTypeConverter { + public bool Accepts(Type type) => type == typeof(ProcessedField); + public object? ReadYaml(IParser parser, Type type, ObjectDeserializer rootDeserializer) => throw new NotImplementedException(); + public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializer serializer) { + if (value is not ProcessedField f) return; + emitter.Emit(new MappingStart()); + emitter.Emit(new Scalar("type")); + if (f.FieldType.IsFunctionPointer || f.FieldType.IsUnmanagedFunctionPointer) { + emitter.Emit(new Scalar("__fastcall")); + } + else { + emitter.Emit(new Scalar(f.FieldTypeOverride ?? f.FieldType.FixTypeName())); + } + emitter.Emit(new Scalar("name")); + emitter.Emit(new Scalar(f.FieldName)); + if (f.FieldOffset >= 0) { + emitter.Emit(new Scalar("offset")); + emitter.Emit(new Scalar(f.FieldOffset.ToString())); + } + if (f.Bits.Length > 0) { + emitter.Emit(new Scalar("bits")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); + foreach (var bit in f.Bits) { + emitter.Emit(new MappingStart()); + emitter.Emit(new Scalar("type")); + emitter.Emit(new Scalar(bit.Type.FixTypeName())); + emitter.Emit(new Scalar("offset")); + emitter.Emit(new Scalar(bit.Offset.ToString())); + emitter.Emit(new Scalar("size")); + emitter.Emit(new Scalar(bit.Size.ToString())); + emitter.Emit(new Scalar("name")); + emitter.Emit(new Scalar(bit.Name)); + emitter.Emit(new MappingEnd()); + } + emitter.Emit(new SequenceEnd()); + } + if (f.IsBase) { + emitter.Emit(new Scalar("base")); + emitter.Emit(new Scalar("true")); + } + switch (f) { + case ProcessedFunctionField func: { + emitter.Emit(new Scalar("return_type")); + emitter.Emit(new Scalar(func.FunctionReturnType.FixTypeName())); + emitter.Emit(new Scalar("parameters")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); + foreach (var p in func.FunctionParameters) { + emitter.Emit(new MappingStart()); + emitter.Emit(new Scalar("type")); + emitter.Emit(new Scalar(p.FieldTypeOverride ?? p.FieldType.FixTypeName())); + emitter.Emit(new Scalar("name")); + emitter.Emit(new Scalar(p.FieldName)); + emitter.Emit(new MappingEnd()); + } + emitter.Emit(new SequenceEnd()); + break; + } + case ProcessedFixedField fix: + emitter.Emit(new Scalar("size")); + emitter.Emit(new Scalar(fix.FixedSize.ToString())); + emitter.Emit(new Scalar("is_string")); + emitter.Emit(new Scalar(fix.FixedString.ToString())); + break; + } + if (f.IsUnion) { + emitter.Emit(new Scalar("union_values")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); + foreach (var unionValue in f.UnionValues) { + emitter.Emit(new MappingStart()); + emitter.Emit(new Scalar("type")); + emitter.Emit(new Scalar(unionValue.FieldTypeOverride ?? unionValue.FieldType.FixTypeName())); + emitter.Emit(new Scalar("name")); + emitter.Emit(new Scalar(unionValue.FieldName)); + emitter.Emit(new MappingEnd()); + } + emitter.Emit(new SequenceEnd()); + } + if(f.FieldType.IsConstructedGenericType && !f.FieldType.IsStdDefiniton()) + { + emitter.Emit(new Scalar("template_args")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Flow)); + foreach (var templateArg in f.FieldType.GenericTypeArguments){ + emitter.Emit(new Scalar(templateArg.FixTypeName())); + } + emitter.Emit(new SequenceEnd()); + } + emitter.Emit(new MappingEnd()); + } + + public static readonly IYamlTypeConverter Instance = new ProcessedFieldConverter(); +} + +public class ProcessedStructConverter : IYamlTypeConverter { + public bool Accepts(Type type) => type == typeof(ProcessedStruct); + public object? ReadYaml(IParser parser, Type type, ObjectDeserializer rootDeserializer) => throw new NotImplementedException(); + public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializer serializer) { + if (value is not ProcessedStruct s) return; + emitter.Emit(new MappingStart()); + emitter.Emit(new Scalar("type")); + emitter.Emit(new Scalar(!string.IsNullOrWhiteSpace(s.StructTypeOverride) ? s.StructTypeOverride : s.StructType.FixTypeName())); + emitter.Emit(new Scalar("name")); + emitter.Emit(new Scalar(s.StructName)); + emitter.Emit(new Scalar("namespace")); + emitter.Emit(new Scalar(s.StructNamespace)); + emitter.Emit(new Scalar("union")); + emitter.Emit(new Scalar(s.IsUnion.ToString())); + if (s.StructType.StructLayoutAttribute?.Value == LayoutKind.Explicit && s.StructSize != 0) { + emitter.Emit(new Scalar("size")); + emitter.Emit(new Scalar(s.StructSize.ToString())); + } + emitter.Emit(new Scalar("fields")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); + foreach (var field in s.Fields) { + ProcessedFieldConverter.Instance.WriteYaml(emitter, field, field.GetType(), serializer); + } + emitter.Emit(new SequenceEnd()); + if (s.VirtualFunctionSize != 0) { + emitter.Emit(new Scalar("vtable_size")); + emitter.Emit(new Scalar(s.VirtualFunctionSize.ToString())); + } + if (s.VirtualFunctions != null) { + emitter.Emit(new Scalar("virtual_functions")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); + foreach (var virtualFunction in s.VirtualFunctions) { + ProcessedVirtualFunctionConverter.Instance.WriteYaml(emitter, virtualFunction, virtualFunction.GetType(), serializer); + } + emitter.Emit(new SequenceEnd()); + } + if (s.MemberFunctions.Any()){ + emitter.Emit(new Scalar("member_functions")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); + foreach (var memberFunction in s.MemberFunctions) { + ProcessedMemberFunctionConverter.Instance.WriteYaml(emitter, memberFunction, memberFunction.GetType(), serializer); + } + emitter.Emit(new SequenceEnd()); + } + if (s.StaticMembers != null) { + emitter.Emit(new Scalar("static_members")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); + foreach (var staticMember in s.StaticMembers) { + ProcessedStaticMembersConverter.Instance.WriteYaml(emitter, staticMember, staticMember.GetType(), serializer); + } + emitter.Emit(new SequenceEnd()); + } + if (s.StaticMemberFunctions != null) { + emitter.Emit(new Scalar("static_member_functions")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); + foreach (var staticMemberFunction in s.StaticMemberFunctions) { + ProcessedMemberFunctionConverter.Instance.WriteYaml(emitter, staticMemberFunction, staticMemberFunction.GetType(), serializer); + } + emitter.Emit(new SequenceEnd()); + } + if (s.TemplateTypes.Any()) { + emitter.Emit(new Scalar("template_types")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Flow)); + foreach (var templateType in s.TemplateTypes) { + emitter.Emit(new Scalar(templateType)); + } + emitter.Emit(new SequenceEnd()); + } + emitter.Emit(new MappingEnd()); + } + + public static readonly IYamlTypeConverter Instance = new ProcessedStructConverter(); +} + +public class ProcessedMemberFunctionConverter : IYamlTypeConverter { + public bool Accepts(Type type) => type == typeof(ProcessedMemberFunction); + public object? ReadYaml(IParser parser, Type type, ObjectDeserializer rootDeserializer) => throw new NotImplementedException(); + public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializer serializer) { + if (value is not ProcessedMemberFunction m) return; + emitter.Emit(new MappingStart()); + emitter.Emit(new Scalar("signature")); + emitter.Emit(new Scalar(m.MemberFunctionSignature)); + emitter.Emit(new Scalar("return_type")); + emitter.Emit(new Scalar(m.MemberFunctionReturnType)); + emitter.Emit(new Scalar("name")); + emitter.Emit(new Scalar(m.MemberFunctionName)); + emitter.Emit(new Scalar("parameters")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); + foreach (var parameter in m.MemberFunctionParameters) { + ProcessedFieldConverter.Instance.WriteYaml(emitter, parameter, parameter.GetType(), serializer); + } + emitter.Emit(new SequenceEnd()); + emitter.Emit(new MappingEnd()); + } + + public static readonly IYamlTypeConverter Instance = new ProcessedMemberFunctionConverter(); +} + +public class ProcessedVirtualFunctionConverter : IYamlTypeConverter { + + public bool Accepts(Type type) => type == typeof(ProcessedVirtualFunction); + public object? ReadYaml(IParser parser, Type type, ObjectDeserializer rootDeserializer) => throw new NotImplementedException(); + public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializer serializer) { + if (value is not ProcessedVirtualFunction v) return; + emitter.Emit(new MappingStart()); + emitter.Emit(new Scalar("name")); + emitter.Emit(new Scalar(v.VirtualFunctionName)); + emitter.Emit(new Scalar("offset")); + emitter.Emit(new Scalar(v.Offset.ToString())); + if (v.VirtualFunctionReturnType != null) { + emitter.Emit(new Scalar("return_type")); + emitter.Emit(new Scalar(v.VirtualFunctionReturnTypeOverride ?? v.VirtualFunctionReturnType.FixTypeName())); + } + if (v.VirtualFunctionParameters != null) { + emitter.Emit(new Scalar("parameters")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); + foreach (var parameter in v.VirtualFunctionParameters) { + ProcessedFieldConverter.Instance.WriteYaml(emitter, parameter, parameter.GetType(), serializer); + } + emitter.Emit(new SequenceEnd()); + } + emitter.Emit(new MappingEnd()); + } + + public static readonly IYamlTypeConverter Instance = new ProcessedVirtualFunctionConverter(); +} + +public class ProcessedStaticMembersConverter : IYamlTypeConverter { + public bool Accepts(Type type) => type == typeof(ProcessedStaticMembers); + public object? ReadYaml(IParser parser, Type type, ObjectDeserializer rootDeserializer) => throw new NotImplementedException(); + public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializer serializer) { + if (value is not ProcessedStaticMembers s) return; + emitter.Emit(new MappingStart()); + emitter.Emit(new Scalar("signature")); + emitter.Emit(new Scalar(s.Signature)); + emitter.Emit(new Scalar("relative_follow_offsets")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); + foreach (var offset in s.RelativeFollowOffsets) { + emitter.Emit(new Scalar(offset.ToString())); + } + emitter.Emit(new SequenceEnd()); + emitter.Emit(new Scalar("is_pointer")); + emitter.Emit(new Scalar(s.IsPointer.ToString())); + emitter.Emit(new Scalar("return_type")); + emitter.Emit(new Scalar(s.ReturnType.FixTypeName())); + emitter.Emit(new MappingEnd()); + } + + public static readonly IYamlTypeConverter Instance = new ProcessedStaticMembersConverter(); +} From 4dda0e0b7e1922ad94e62b224b7f890666b03d9e Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:06:54 +0200 Subject: [PATCH 21/95] get template header generation working --- scripts/src_wrapper.py | 473 ++++++++++++++++++++++++++++++-------- scripts/structs_schema.py | 61 +++-- 2 files changed, 421 insertions(+), 113 deletions(-) diff --git a/scripts/src_wrapper.py b/scripts/src_wrapper.py index c3415d8ec0..9a7811ac3d 100644 --- a/scripts/src_wrapper.py +++ b/scripts/src_wrapper.py @@ -27,11 +27,11 @@ def get_size_from_string(self, type: str) -> int: """ Gets the int of a base type string. """ - if type == "int8_t" or type == "uint8_t" or type == "bool": + if type in ("int8_t", "uint8_t", "bool", "char", "unsigned char", "signed char"): return 1 - if type == "int16_t" or type == "uint16_t": + if type in ("int16_t", "uint16_t", "wchar_t"): return 2 - if type == "int32_t" or type == "uint32_t" or type == "float": + if type in ("int32_t", "uint32_t", "float"): return 4 if ( type == "int64_t" @@ -41,23 +41,289 @@ def get_size_from_string(self, type: str) -> int: ): return 8 return 0 + + def _split_template_type(self, type_name: str) -> tuple[str, list[str]]: + """Split a type's base name and top-level template arguments.""" + type_name = type_name.strip() + template_start = type_name.find("<") + if template_start < 0: + return type_name, [] + + depth = 0 + argument_start = template_start + 1 + arguments: list[str] = [] + for index in range(template_start, len(type_name)): + char = type_name[index] + if char == "<": + depth += 1 + elif char == ">": + depth -= 1 + if depth == 0: + if argument_start < index: + arguments.append(type_name[argument_start:index].strip()) + if type_name[index + 1 :].strip(): + return type_name, [] + return type_name[:template_start].strip(), arguments + elif char == "," and depth == 1: + arguments.append(type_name[argument_start:index].strip()) + argument_start = index + 1 + + return type_name, [] + + def _normalize_type(self, type_name: str) -> str: + """Normalize whitespace without losing nested template structure.""" + type_name = type_name.strip() + pointer_suffix = "" + while type_name.endswith("*"): + pointer_suffix += "*" + type_name = type_name[:-1].strip() + + base_type, template_args = self._split_template_type(type_name) + if template_args: + normalized = f"{base_type}<{','.join(self._normalize_type(arg) for arg in template_args)}>" + else: + normalized = " ".join(base_type.split()) + return normalized + pointer_suffix + + def _get_field_type( + self, + field: DefinedStructField | DefinedStructFixedField, + template_struct_lookup: dict[str, DefinedStruct], + ) -> str: + field_type = field.type.strip() + pointer_suffix = "" + while field_type.endswith("*"): + pointer_suffix += "*" + field_type = field_type[:-1].strip() + + if ( + field.template_args + and not self._split_template_type(field_type)[1] + and self._normalize_type(field_type) in template_struct_lookup + ): + field_type += f"<{', '.join(field.template_args)}>" + return field_type + pointer_suffix + + def _substitute_template_types( + self, type_name: str, substitutions: dict[str, str] + ) -> str: + type_name = type_name.strip() + pointer_suffix = "" + while type_name.endswith("*"): + pointer_suffix += "*" + type_name = type_name[:-1].strip() + + base_type, template_args = self._split_template_type(type_name) + if template_args: + substituted = ( + f"{base_type}<" + f"{', '.join(self._substitute_template_types(arg, substitutions) for arg in template_args)}>" + ) + else: + substituted = substitutions.get(base_type, base_type) + return substituted + pointer_suffix + + def get_size_from_type( + self, + type_name: str, + struct_lookup: dict[str, int], + enum_lookup: dict[str, int], + template_struct_lookup: dict[str, DefinedStruct] | None = None, + template_parameters: tuple[str, ...] = (), + ) -> int: + """Resolve a field's size using schema types and the target x64 ABI.""" + type_name = type_name.strip() + if type_name.endswith("*"): + return 8 + if type_name in template_parameters: + return 0 + + normalized_type = self._normalize_type(type_name) + if normalized_type in struct_lookup: + return struct_lookup[normalized_type] + if normalized_type in enum_lookup: + return enum_lookup[normalized_type] + + primitive_size = self.get_size_from_string(type_name) + if primitive_size: + return primitive_size + + base_type, template_args = self._split_template_type(type_name) + if not template_args: + return 0 + + stl_type_sizes = { + "std::vector": 24, + "std::set": 16, + "std::map": 16, + "std::string": 32, + "std::wstring": 32, + "std::basic_string": 32, + "std::list": 16, + "std::deque": 32, + "std::linkedlist": 24, + } + if base_type in stl_type_sizes: + return stl_type_sizes[base_type] + + if base_type == "std::pair": + if len(template_args) != 2: + raise ValueError(f"std::pair requires two type arguments: {type_name}") + first_size = self.get_size_from_type( + template_args[0], struct_lookup, enum_lookup, template_struct_lookup + ) + second_size = self.get_size_from_type( + template_args[1], struct_lookup, enum_lookup, template_struct_lookup + ) + first_alignment = self._get_type_alignment( + template_args[0], struct_lookup, enum_lookup, template_struct_lookup + ) + second_alignment = self._get_type_alignment( + template_args[1], struct_lookup, enum_lookup, template_struct_lookup + ) + alignment = max(first_alignment, second_alignment) + second_offset = self._align_size(first_size, second_alignment) + return self._align_size(second_offset + second_size, alignment) + + template_struct_lookup = template_struct_lookup or {} + template_struct = template_struct_lookup.get(self._normalize_type(base_type)) + if template_struct is not None: + return self._get_template_struct_size( + template_struct, + template_args, + struct_lookup, + enum_lookup, + template_struct_lookup, + ) + + raise ValueError(f"Cannot determine size for template type: {type_name}") + + def _get_type_alignment( + self, + type_name: str, + struct_lookup: dict[str, int], + enum_lookup: dict[str, int], + template_struct_lookup: dict[str, DefinedStruct] | None = None, + ) -> int: + type_name = type_name.strip() + if type_name.endswith("*"): + return 8 + + normalized_type = self._normalize_type(type_name) + if normalized_type in struct_lookup: + size = struct_lookup[normalized_type] + elif normalized_type in enum_lookup: + size = enum_lookup[normalized_type] + else: + size = self.get_size_from_string(type_name) + if size == 0: + base_type, template_args = self._split_template_type(type_name) + if base_type in ("std::vector", "std::set", "std::map", "std::string", "std::wstring", "std::basic_string", "std::list", "std::deque", "std::linkedlist"): + return 8 + if base_type == "std::pair": + if len(template_args) != 2: + raise ValueError(f"std::pair requires two type arguments: {type_name}") + return max( + self._get_type_alignment( + arg, struct_lookup, enum_lookup, template_struct_lookup + ) + for arg in template_args + ) + size = self.get_size_from_type( + type_name, + struct_lookup, + enum_lookup, + template_struct_lookup, + ) + return min(size & -size, 8) if size else 1 + + @staticmethod + def _align_size(size: int, alignment: int) -> int: + return (size + alignment - 1) // alignment * alignment + + def _get_template_struct_size( + self, + struct: DefinedStruct, + template_args: list[str], + struct_lookup: dict[str, int], + enum_lookup: dict[str, int], + template_struct_lookup: dict[str, DefinedStruct], + ) -> int: + if len(template_args) != len(struct.template_types): + raise ValueError( + f"Template type {struct.type} expects {len(struct.template_types)} " + f"arguments, got {len(template_args)}" + ) + if struct.size is not None: + return struct.size + + substitutions = dict(zip(struct.template_types, template_args)) + extent = 0 + for field in struct.fields: + if field.type == "__fastcall": + field_size = 8 + else: + field_type = self._get_field_type(field, template_struct_lookup) + field_type = self._substitute_template_types(field_type, substitutions) + field_size = self.get_size_from_type( + field_type, + struct_lookup, + enum_lookup, + template_struct_lookup, + ) + if field_size == 0: + raise ValueError( + f"Cannot determine size of {field_type} in template type {struct.type}" + ) + if isinstance(field, DefinedStructFixedField): + field_size *= field.size + + field_end = field.offset + field_size + if struct.union: + extent = max(extent, field_size) + else: + extent = max(extent, field_end) + return extent - def get_string_define_and_size(self, field: DefinedStructField | DefinedStructFixedField, namespace: str, struct_lookup: dict[str, int], enum_lookup: dict[str, int]) -> tuple[str, int]: - field_type = field.type - if field_type.startswith(namespace): - field_type = field_type.removeprefix(f"{namespace}::") + def get_string_define_and_size( + self, + field: DefinedStructField | DefinedStructFixedField, + namespace: str, + struct_lookup: dict[str, int], + enum_lookup: dict[str, int], + template_struct_lookup: dict[str, DefinedStruct] | None = None, + template_parameters: tuple[str, ...] = (), + ) -> tuple[str, int]: + template_struct_lookup = template_struct_lookup or {} + field_type = self._get_field_type(field, template_struct_lookup) field_name = field.name - lookup_type = self.get_short_name(field.type) - if lookup_type in struct_lookup: - return (f"{field_type} {field_name};", struct_lookup[lookup_type]) - elif lookup_type in enum_lookup: - return (f"{field_type} {field_name};", enum_lookup[lookup_type]) - else: - return (f"{field_type} {field_name};", self.get_size_from_string(field_type)) + field_size = self.get_size_from_type( + field_type, + struct_lookup, + enum_lookup, + template_struct_lookup, + template_parameters, + ) + display_type = field_type + if display_type.startswith(namespace): + display_type = display_type.removeprefix(f"{namespace}::") + return (f"{display_type} {field_name};", field_size) def get_short_name(self, full_type: str) -> str: """Extract the short name from a fully qualified type name.""" - return full_type.split("::")[-1] + depth = 0 + last_namespace_separator = -1 + index = 0 + while index < len(full_type) - 1: + if full_type[index] == "<": + depth += 1 + elif full_type[index] == ">": + depth -= 1 + elif full_type[index : index + 2] == "::" and depth == 0: + last_namespace_separator = index + 1 + index += 1 + index += 1 + return full_type[last_namespace_separator + 1 :] def build_vfunc_signature(self, vfunc: DefinedStructVFunc) -> str: """Build a function pointer signature for a virtual function. @@ -90,38 +356,19 @@ def build_export_string( output = IndentWriter() struct_lookup = { - struct.type: struct.size for struct in export.structs if struct.size is not None + self._normalize_type(struct.type): struct.size + for struct in export.structs + if struct.size is not None + } + enum_lookup = { + self._normalize_type(enum.type): self.get_size_from_string(enum.underlying) + for enum in export.enums + } + template_struct_lookup = { + self._normalize_type(struct.type): struct + for struct in export.structs + if struct.template_types } - enum_lookup = {enum.type: self.get_size_from_string(enum.underlying) for enum in export.enums} - - namespace_enum_chunks: dict[str, list[IndentWriter]] = {} - namespace_forward_chunks: dict[str, list[IndentWriter]] = {} - namespace_define_chunks: dict[str, list[IndentWriter]] = {} - namespace_order: list[str] = [] - - def add_namespace(namespace: str): - if namespace not in namespace_enum_chunks: - namespace_enum_chunks[namespace] = [] - namespace_forward_chunks[namespace] = [] - namespace_define_chunks[namespace] = [] - namespace_order.append(namespace) - - for enum in export.enums: - add_namespace(enum.namespace) - namespace_enum_chunks[enum.namespace].append(self.build_enum_string(enum)) - - vtable_names: list[str] = [] - - for struct in export.structs: - add_namespace(struct.namespace) - forward, define, vtable, vtable_full_name = self.build_struct_string( - struct, struct_lookup, enum_lookup - ) - if vtable_full_name: - vtable_names.append(vtable_full_name) - namespace_forward_chunks[struct.namespace].append(forward) - namespace_define_chunks[struct.namespace].append(define) - namespace_define_chunks[struct.namespace].append(vtable) def append_chunk(chunk: IndentWriter | str): chunk_text = str(chunk) @@ -136,53 +383,55 @@ def append_chunk(chunk: IndentWriter | str): output.append(line.lstrip(" ")) output.indent_level = prev_indent - def append_namespace_content(namespace: str): - ordered_chunks = [ - *namespace_enum_chunks[namespace], - *namespace_forward_chunks[namespace], - *namespace_define_chunks[namespace], - ] - for chunk in ordered_chunks: - append_chunk(chunk) - - # Build an ordered namespace tree so parent namespaces are emitted only once. - namespace_tree_children: dict[str, list[str]] = {"": []} - for namespace in namespace_order: - if namespace == "": - continue + def append_ordered_chunks(chunks: list[tuple[str, IndentWriter]]): + open_namespaces: list[str] = [] + for namespace, chunk in chunks: + target_namespaces = namespace.split("::") if namespace else [] + shared_count = 0 + while ( + shared_count < len(open_namespaces) + and shared_count < len(target_namespaces) + and open_namespaces[shared_count] == target_namespaces[shared_count] + ): + shared_count += 1 + + while len(open_namespaces) > shared_count: + output.unindent() + output.append("}") + open_namespaces.pop() + + for namespace_name in target_namespaces[shared_count:]: + output.append(f"namespace {namespace_name} {{") + output.indent() + open_namespaces.append(namespace_name) - parent_path = "" - current_parts: list[str] = [] - for part in namespace.split("::"): - current_parts.append(part) - current_path = "::".join(current_parts) - if parent_path not in namespace_tree_children: - namespace_tree_children[parent_path] = [] - if current_path not in namespace_tree_children[parent_path]: - namespace_tree_children[parent_path].append(current_path) - if current_path not in namespace_tree_children: - namespace_tree_children[current_path] = [] - parent_path = current_path - - # Emit global namespace content first (if any), then recurse through namespace tree. - if "" in namespace_enum_chunks: - append_namespace_content("") - - def emit_namespace_subtree(parent_path: str): - for child_path in namespace_tree_children.get(parent_path, []): - child_name = child_path.split("::")[-1] - output.append(f"namespace {child_name} {{") - output.indent() - - if child_path in namespace_enum_chunks: - append_namespace_content(child_path) - - emit_namespace_subtree(child_path) + append_chunk(chunk) + while open_namespaces: output.unindent() output.append("}") + open_namespaces.pop() + + enum_chunks = [ + (enum.namespace, self.build_enum_string(enum)) for enum in export.enums + ] + append_ordered_chunks(enum_chunks) + + forward_chunks: list[tuple[str, IndentWriter]] = [] + definition_chunks: list[tuple[str, IndentWriter]] = [] + vtable_names: list[str] = [] + for struct in export.structs: + forward, define, vtable, vtable_full_name = self.build_struct_string( + struct, struct_lookup, enum_lookup, template_struct_lookup + ) + if vtable_full_name: + vtable_names.append(vtable_full_name) + forward_chunks.append((struct.namespace, forward)) + definition_chunks.append((struct.namespace, define)) + definition_chunks.append((struct.namespace, vtable)) - emit_namespace_subtree("") + append_ordered_chunks(forward_chunks) + append_ordered_chunks(definition_chunks) return (str(output), vtable_names) @@ -219,6 +468,7 @@ def build_struct_string( struct: DefinedStruct, struct_lookup: dict[str, int], enum_lookup: dict[str, int], + template_struct_lookup: dict[str, DefinedStruct] | None = None, ) -> tuple[IndentWriter, IndentWriter, IndentWriter, str | None]: """Builds the C++ struct definition with namespace wrapping. @@ -231,16 +481,30 @@ def build_struct_string( struct_string_forward: IndentWriter = IndentWriter() struct_string_define: IndentWriter = IndentWriter() struct_string_vtable: IndentWriter = IndentWriter() + template_struct_lookup = template_struct_lookup or {} + template_declaration = "" + template_suffix = "" + if struct.template_types: + template_declaration = ( + f"template" + ) + template_suffix = f"<{', '.join(struct.template_types)}>" short_name = self.get_short_name(struct.type) vtable_full_name: str | None = None # Forward declarations + if template_declaration: + struct_string_forward += template_declaration struct_string_forward += f"struct __cppobj {short_name};" if struct.virtual_functions: + if template_declaration: + struct_string_forward += template_declaration struct_string_forward += f"struct __cppobj {short_name}_vtbl;" # Struct definition + if template_declaration: + struct_string_define += template_declaration struct_define_line = f"struct __cppobj {short_name}" struct_extend_lines = [] @@ -250,9 +514,16 @@ def build_struct_string( for base_field in base_fields: if base_field.offset == prev_size: base_fields_used.append(base_field.type) - short_base_name = self.get_short_name(base_field.type) + field_type = self._get_field_type(base_field, template_struct_lookup) + short_base_name = self.get_short_name(field_type) struct_extend_lines.append(short_base_name) - prev_size += struct_lookup[base_field.type] + prev_size += self.get_size_from_type( + field_type, + struct_lookup, + enum_lookup, + template_struct_lookup, + tuple(struct.template_types), + ) struct_extend_line = "" if len(struct_extend_lines) > 0: @@ -263,7 +534,7 @@ def build_struct_string( # Add __vtable member if this struct has virtual functions if struct.virtual_functions: - struct_string_define += f"{short_name}_vtbl* __vtable;" + struct_string_define += f"{short_name}_vtbl{template_suffix}* __vtable;" offset = 0 @@ -282,7 +553,14 @@ def build_struct_string( field_name = field.name field_type = field.type if isinstance(field, DefinedStructFixedField): - field_define, field_define_size = self.get_string_define_and_size(field, struct.namespace, struct_lookup, enum_lookup) + field_define, field_define_size = self.get_string_define_and_size( + field, + struct.namespace, + struct_lookup, + enum_lookup, + template_struct_lookup, + tuple(struct.template_types), + ) struct_string_define += f"{field_define[:-1]}[{field.size}];" prev_size += field_define_size * field.size elif field_type == "__fastcall": @@ -295,7 +573,14 @@ def build_struct_string( struct_string_define += fastcall_field_define[:-1] + ");" prev_size += 8 else: - field_define, field_define_size = self.get_string_define_and_size(field, struct.namespace, struct_lookup, enum_lookup) + field_define, field_define_size = self.get_string_define_and_size( + field, + struct.namespace, + struct_lookup, + enum_lookup, + template_struct_lookup, + tuple(struct.template_types), + ) struct_string_define += field_define prev_size += field_define_size @@ -311,6 +596,8 @@ def build_struct_string( # Virtual table struct definition if struct.virtual_functions: + if template_declaration: + struct_string_vtable += template_declaration struct_string_vtable += f"struct __cppobj {short_name}_vtbl {{" struct_string_vtable.indent() for vfunc in struct.virtual_functions: diff --git a/scripts/structs_schema.py b/scripts/structs_schema.py index ba685b800a..380e97dd15 100644 --- a/scripts/structs_schema.py +++ b/scripts/structs_schema.py @@ -7,10 +7,13 @@ except ImportError: from yaml import SafeLoader as Loader -YamlStructValuesExport: TypeAlias = list[dict[str, Union[str, int, "YamlStructValuesExport"]]] +YamlStructValuesExport: TypeAlias = list[ + dict[str, Union[str, int, "YamlStructValuesExport"]] +] YamlStructExport: TypeAlias = dict[str, YamlStructValuesExport] YamlExport: TypeAlias = dict[str, list[YamlStructExport]] + @dataclass class DefinedStructBase: name: str @@ -51,6 +54,7 @@ class DefinedStructMemFunc: class DefinedStructField(DefinedStructFuncParam): offset: int base: bool + template_args: list[str] @dataclass @@ -72,8 +76,10 @@ class DefinedStructFixedField(DefinedStructField): size: int is_string: bool + Fields: TypeAlias = list[DefinedStructField | DefinedStructFixedField] + @dataclass class DefinedStruct(DefinedStructBase): fields: Fields @@ -93,10 +99,8 @@ class DefinedStructExport: structs: list[DefinedStruct] -def get_yaml(stream, loader = Loader) -> DefinedStructExport: - dic: YamlExport = load( - stream, loader - ) +def get_yaml(stream, loader=Loader) -> DefinedStructExport: + dic: YamlExport = load(stream, loader) enums = [] structs = [] for enum in dic["enums"]: @@ -116,8 +120,13 @@ def get_yaml(stream, loader = Loader) -> DefinedStructExport: member_functions = [] static_member_functions = None static_members = None + template_types = [] for field in struct["fields"]: base = field["base"] if "base" in field else False + template_args = [] + if "template_args" in field: + for template_arg in field["template_args"]: + template_args.append(template_arg) if "size" in field: fields.append( DefinedStructFixedField( @@ -125,8 +134,9 @@ def get_yaml(stream, loader = Loader) -> DefinedStructExport: field["type"], field["offset"], base, + template_args, field["size"], - field["is_string"] + field["is_string"], ) ) elif "return_type" in field: @@ -141,6 +151,7 @@ def get_yaml(stream, loader = Loader) -> DefinedStructExport: field["type"], field["offset"], base, + template_args, field["return_type"], parameters, ) @@ -148,7 +159,11 @@ def get_yaml(stream, loader = Loader) -> DefinedStructExport: else: fields.append( DefinedStructField( - field["name"], field["type"], field["offset"], base + field["name"], + field["type"], + field["offset"], + base, + template_args, ) ) if "virtual_functions" in struct: @@ -170,18 +185,21 @@ def get_yaml(stream, loader = Loader) -> DefinedStructExport: parameters, ) ) - for memfunc in struct["member_functions"]: - parameters = [] - for param in memfunc["parameters"]: - parameters.append(DefinedStructFuncParam(param["name"], param["type"])) - member_functions.append( - DefinedStructMemFunc( - memfunc["signature"], - memfunc["return_type"], - parameters, - memfunc["name"], + if "member_functions" in struct: + for memfunc in struct["member_functions"]: + parameters = [] + for param in memfunc["parameters"]: + parameters.append( + DefinedStructFuncParam(param["name"], param["type"]) + ) + member_functions.append( + DefinedStructMemFunc( + memfunc["signature"], + memfunc["return_type"], + parameters, + memfunc["name"], + ) ) - ) if "static_member_functions" in struct: static_member_functions = [] for smemfunc in struct["static_member_functions"]: @@ -209,6 +227,9 @@ def get_yaml(stream, loader = Loader) -> DefinedStructExport: sm["is_pointer"] if "is_pointer" in sm else False, ) ) + if "template_types" in struct: + for type in struct["template_types"]: + template_types.append(type) size = None if "size" in struct: size = struct["size"] @@ -228,7 +249,7 @@ def get_yaml(stream, loader = Loader) -> DefinedStructExport: struct["union"], static_member_functions, static_members, - [] + template_types, ) ) - return DefinedStructExport(enums, structs) \ No newline at end of file + return DefinedStructExport(enums, structs) From 17a581fdd9b74a066cd663b03d32673d57bd518a Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 7 Oct 2026 05:04:59 +0200 Subject: [PATCH 22/95] fix vtable not being generated correctly --- scripts/src_wrapper.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/scripts/src_wrapper.py b/scripts/src_wrapper.py index 9a7811ac3d..28a6229e09 100644 --- a/scripts/src_wrapper.py +++ b/scripts/src_wrapper.py @@ -334,11 +334,9 @@ def build_vfunc_signature(self, vfunc: DefinedStructVFunc) -> str: Returns: Function pointer signature string """ - # Special case for Dtor - if vfunc.name == "Dtor": - # Find the struct name from context - this will be passed separately - return "DTOR" - + if not (vfunc.return_type and vfunc.parameters): + return f"void* {vfunc.name};" + return_type = vfunc.return_type if vfunc.return_type else "void" # Build parameter list @@ -533,8 +531,9 @@ def build_struct_string( struct_string_define.indent() # Add __vtable member if this struct has virtual functions - if struct.virtual_functions: + if struct.virtual_functions and prev_size == 0: struct_string_define += f"{short_name}_vtbl{template_suffix}* __vtable;" + prev_size = 8 offset = 0 @@ -586,7 +585,7 @@ def build_struct_string( if struct.size is not None and struct.size != 0: while struct.size > prev_size: - fill_size = offset - prev_size + fill_size = struct.size - prev_size line, size = self.get_struct_pad(fill_size, prev_size) struct_string_define += line prev_size += size @@ -600,13 +599,14 @@ def build_struct_string( struct_string_vtable += template_declaration struct_string_vtable += f"struct __cppobj {short_name}_vtbl {{" struct_string_vtable.indent() + previous_offset = 0 for vfunc in struct.virtual_functions: - sig = self.build_vfunc_signature(vfunc) - # Special case for DTOR macro - if sig.startswith("DTOR"): - struct_string_vtable += f"DTOR({short_name})" - else: - struct_string_vtable += sig + while previous_offset < vfunc.offset: + struct_string_vtable += f"void* vf{int(previous_offset/8)};" + previous_offset += 8 + + struct_string_vtable += self.build_vfunc_signature(vfunc) + previous_offset = vfunc.offset + 8 struct_string_vtable.unindent() struct_string_vtable += "};" vtable_full_name = f"{struct.type}_vtbl" From 078beb6cfd65a2b342421ec2f49aaa9874eb89ee Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 7 Oct 2026 05:11:16 +0200 Subject: [PATCH 23/95] fix exd declspec align --- scripts/exdschema_wrapper.py | 46 ++++++++++++++++++++---------------- scripts/ida/exdgetters.py | 2 +- scripts/lumina_wrapper.py | 2 +- scripts/src_wrapper.py | 17 +++++++++---- 4 files changed, 39 insertions(+), 28 deletions(-) diff --git a/scripts/exdschema_wrapper.py b/scripts/exdschema_wrapper.py index be300eaec2..c1078e6d48 100644 --- a/scripts/exdschema_wrapper.py +++ b/scripts/exdschema_wrapper.py @@ -40,7 +40,7 @@ class RepeatDefinition(Definition): fields: "DefinitionFields" def size(self) -> int: - return len(self.fields) * self.count + return len(self.fields) * self.count if len(self.fields) > 0 else self.count DefinitionFields: TypeAlias = list[Union[Definition, RepeatDefinition]] @@ -125,20 +125,23 @@ def map_header_and_schema( header: list[ExcelColumnDefinition], schema: DefinitionFields, header_index_offset: int = 0, + schema_index_offset: int = 0, schema_name: str = "", ): fields: Fields = [] enums: list[DefinedStructEnum] = [] enum_field_map: dict[int, int] = {} structs: list[DefinedStruct] = [] - for index in range(len(schema)): - header_index = sum(f.size() for f in schema[:index]) + header_index_offset - definition = header[header_index] - field = schema[index] + prev_offset = 0 + while prev_offset < len(schema): + header_offset = sum([f.size() if isinstance(f, RepeatDefinition) else 1 for f in schema[:prev_offset]]) + definition = header[header_offset + schema_index_offset] + field = schema[prev_offset] + definition_offset = definition.offset - header_index_offset if isinstance(field, RepeatDefinition): exported_enums, exported_structs = map_header_and_schema( - header, field.fields, header_index, schema_name + field.name + header, field.fields, definition.offset, header_offset, schema_name + field.name ) if len(exported_structs) == 0: base_type = definition.get_base_type_string() @@ -146,38 +149,37 @@ def map_header_and_schema( DefinedStructFixedField( field.name, base_type, - definition.offset, + definition_offset, False, + [], field.count, False, ) ) - pass else: fields.append( DefinedStructFixedField( field.name, exported_structs[-1].name, - definition.offset, + definition_offset, False, + [], field.count, False, ) ) structs.extend(exported_structs) enums.extend(exported_enums) - pass else: if definition.is_packed_bool(): - if definition.offset in enum_field_map: - enums[enum_field_map[definition.offset]].values[field.name] = ( + if definition_offset in enum_field_map: + enums[enum_field_map[definition_offset]].values[field.name] = ( definition.type - ExcelColumnDataType.PackedBool0 ) - pass else: enum_index = len(enums) - enum_field_map[definition.offset] = enum_index - enum_type = f"{schema_name}PackedBool{definition.offset:X}" + enum_field_map[definition_offset] = enum_index + enum_type = f"{schema_name}PackedBool{definition_offset:X}" enum_name = f"{exd_namespace}::{enum_type}" enums.append( DefinedStructEnum( @@ -194,23 +196,25 @@ def map_header_and_schema( ) fields.append( DefinedStructField( - f"PackedBool{definition.offset:X}", + f"PackedBool{definition_offset:X}", enum_name, - definition.offset, + definition_offset, False, + [] ) ) - pass else: fields.append( DefinedStructField( field.name, definition.get_base_type_string(), - definition.offset, + definition_offset, False, + [] ) ) - pass + + prev_offset += 1 size: int = 0 for field in fields: @@ -262,7 +266,7 @@ def create_struct_from_header_and_schema( index, header = headers[excel_header_name] schema = schemas[excel_header_name] excel_map[excel_header_name] = index - enums, structs = map_header_and_schema(header, schema, 0, excel_header_name) + enums, structs = map_header_and_schema(header, schema, 0, 0, excel_header_name) struct_export.enums.extend(enums) struct_export.structs.extend(structs) diff --git a/scripts/ida/exdgetters.py b/scripts/ida/exdgetters.py index a11e2644a9..289af07afc 100644 --- a/scripts/ida/exdgetters.py +++ b/scripts/ida/exdgetters.py @@ -29,7 +29,7 @@ exd_map = {excel_map[k]: k for k in excel_map} -header, vtables = src.build_export_string(struct_export) +header, vtables = src.build_export_string(struct_export, 4) ida = IdaInterface() diff --git a/scripts/lumina_wrapper.py b/scripts/lumina_wrapper.py index 6353fe470c..9acdbf87d2 100644 --- a/scripts/lumina_wrapper.py +++ b/scripts/lumina_wrapper.py @@ -141,7 +141,7 @@ def get_nugpkg(package: str, version: str = ""): return get_nugpkg_deps(join(tempdirname, f"{package}.nuspec")) -def get_excel_header_files() -> dict[str, list[ExcelColumnDefinition]]: +def get_excel_header_files() -> dict[str, tuple[int, list[ExcelColumnDefinition]]]: deps = get_nugpkg("Lumina") deps_gotten: dict[str, str] = {} diff --git a/scripts/src_wrapper.py b/scripts/src_wrapper.py index 28a6229e09..e62afe97df 100644 --- a/scripts/src_wrapper.py +++ b/scripts/src_wrapper.py @@ -349,7 +349,8 @@ def build_vfunc_signature(self, vfunc: DefinedStructVFunc) -> str: def build_export_string( self, - export: DefinedStructExport + export: DefinedStructExport, + full_declspec_align: int = 0 ): output = IndentWriter() @@ -420,7 +421,7 @@ def append_ordered_chunks(chunks: list[tuple[str, IndentWriter]]): vtable_names: list[str] = [] for struct in export.structs: forward, define, vtable, vtable_full_name = self.build_struct_string( - struct, struct_lookup, enum_lookup, template_struct_lookup + struct, struct_lookup, enum_lookup, template_struct_lookup, full_declspec_align ) if vtable_full_name: vtable_names.append(vtable_full_name) @@ -467,6 +468,7 @@ def build_struct_string( struct_lookup: dict[str, int], enum_lookup: dict[str, int], template_struct_lookup: dict[str, DefinedStruct] | None = None, + declspec_align: int = 0 ) -> tuple[IndentWriter, IndentWriter, IndentWriter, str | None]: """Builds the C++ struct definition with namespace wrapping. @@ -491,19 +493,24 @@ def build_struct_string( short_name = self.get_short_name(struct.type) vtable_full_name: str | None = None + if declspec_align > 0: + declspec = f"__declspec(align({declspec_align})) " + else: + declspec = "" + # Forward declarations if template_declaration: struct_string_forward += template_declaration - struct_string_forward += f"struct __cppobj {short_name};" + struct_string_forward += f"struct {short_name};" if struct.virtual_functions: if template_declaration: struct_string_forward += template_declaration - struct_string_forward += f"struct __cppobj {short_name}_vtbl;" + struct_string_forward += f"struct {short_name}_vtbl;" # Struct definition if template_declaration: struct_string_define += template_declaration - struct_define_line = f"struct __cppobj {short_name}" + struct_define_line = f"struct {declspec}{short_name}" struct_extend_lines = [] base_fields = [field for field in struct.fields if field.base] From 70e7515046cbc665a869c778c0fc576dbdc7ce05 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 7 Oct 2026 05:12:03 +0200 Subject: [PATCH 24/95] try to free dotnet from python runtime --- scripts/lumina_wrapper.py | 89 ++++++++++++++++++++++++--------------- 1 file changed, 54 insertions(+), 35 deletions(-) diff --git a/scripts/lumina_wrapper.py b/scripts/lumina_wrapper.py index 9acdbf87d2..34cd373bca 100644 --- a/scripts/lumina_wrapper.py +++ b/scripts/lumina_wrapper.py @@ -10,6 +10,7 @@ from json import load as jsonload from enum import IntEnum from dataclasses import dataclass +import gc dotnetload("coreclr") import clr @@ -111,6 +112,9 @@ def get_base_type_string(self): def __lt__(self, other): return self.offset < other.offset + def __repr__(self): + return self.get_base_type_string() + def get_nugpkg_and_extract(package: str, version: str = ""): with urlopen( @@ -156,43 +160,58 @@ def get_excel_header_files() -> dict[str, tuple[int, list[ExcelColumnDefinition] path.append(tempdirname) from System.Runtime.Loader import AssemblyLoadContext # type: ignore - from System import Activator # type: ignore + from System import Activator, GC # type: ignore ctx = AssemblyLoadContext("temp", isCollectible=True) - assemblies: dict = {} - - for file in listdir(tempdirname): - if file.endswith(".dll"): - assemblies[file] = ctx.LoadFromAssemblyPath( - abspath(join(tempdirname, file)) - ) - - assembly = assemblies["Lumina.dll"] - game_data_type = assembly.GetType("Lumina.GameData") - game_data_instance = Activator.CreateInstance( - game_data_type, [join(game_path, "game", "sqpack"), None] - ) - - excel_list_type = assembly.GetType("Lumina.Data.Files.Excel.ExcelListFile") - excel_header_type = assembly.GetType("Lumina.Data.Files.Excel.ExcelHeaderFile") - - root_exl = game_data_instance.GetFile[excel_list_type]("exd/root.exl") - - excel_header_files: dict[str, tuple[int, list[ExcelColumnDefinition]]] = {} - - for kvp in root_exl.ExdMap: - key = kvp.Key - value = kvp.Value - - if value != -1: - defs: list[ExcelColumnDefinition] = [] - header = game_data_instance.GetFile[excel_header_type](f"exd/{key}.exh") - for column_definition in header.ColumnDefinitions: - defs.append( - ExcelColumnDefinition( - int(column_definition.Type), column_definition.Offset - ) + + def _isolated_extract(context): + assemblies: dict = {} + + for file in listdir(tempdirname): + if file.endswith(".dll"): + assemblies[file] = context.LoadFromAssemblyPath( + abspath(join(tempdirname, file)) ) - excel_header_files[key] = (value, sorted(defs)) + + assembly = assemblies["Lumina.dll"] + game_data_type = assembly.GetType("Lumina.GameData") + game_data_instance = Activator.CreateInstance( + game_data_type, [join(game_path, "game", "sqpack"), None] + ) + + excel_list_type = assembly.GetType("Lumina.Data.Files.Excel.ExcelListFile") + excel_header_type = assembly.GetType("Lumina.Data.Files.Excel.ExcelHeaderFile") + + root_exl = game_data_instance.GetFile[excel_list_type]("exd/root.exl") + + excel_header_files: dict[str, tuple[int, list[ExcelColumnDefinition]]] = {} + + for kvp in root_exl.ExdMap: + key = kvp.Key + value = kvp.Value + + if value != -1: + defs: list[ExcelColumnDefinition] = [] + header = game_data_instance.GetFile[excel_header_type](f"exd/{key}.exh") + for column_definition in header.ColumnDefinitions: + defs.append( + ExcelColumnDefinition( + int(column_definition.Type), column_definition.Offset + ) + ) + excel_header_files[key] = (value, sorted(defs)) + + + return excel_header_files + + excel_header_files = _isolated_extract(ctx) + + ctx.Unload() + + del ctx + + gc.collect() + GC.Collect() + GC.WaitForPendingFinalizers() return excel_header_files From 59252cf0a3262ca4175a69dd31622022cfed21de Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 7 Oct 2026 05:13:08 +0200 Subject: [PATCH 25/95] fix namespace decl --- scripts/src_wrapper.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/src_wrapper.py b/scripts/src_wrapper.py index e62afe97df..b8fa9247cf 100644 --- a/scripts/src_wrapper.py +++ b/scripts/src_wrapper.py @@ -400,7 +400,8 @@ def append_ordered_chunks(chunks: list[tuple[str, IndentWriter]]): open_namespaces.pop() for namespace_name in target_namespaces[shared_count:]: - output.append(f"namespace {namespace_name} {{") + formatted_name = namespace_name.replace(".", "::") + output.append(f"namespace {formatted_name} {{") output.indent() open_namespaces.append(namespace_name) From 05a028ca92edfc03db9af74ef26611754d4d15c3 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 7 Oct 2026 05:13:28 +0200 Subject: [PATCH 26/95] get correct struct size --- scripts/src_wrapper.py | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/scripts/src_wrapper.py b/scripts/src_wrapper.py index b8fa9247cf..35552c936b 100644 --- a/scripts/src_wrapper.py +++ b/scripts/src_wrapper.py @@ -85,6 +85,17 @@ def _normalize_type(self, type_name: str) -> str: normalized = " ".join(base_type.split()) return normalized + pointer_suffix + def _get_struct_size( + self, + type_name: str, + struct_lookup: dict[str, int], + ) -> int | None: + normalized_type = self._normalize_type(type_name) + if normalized_type in struct_lookup: + return struct_lookup[normalized_type] + + return struct_lookup.get(self.get_short_name(normalized_type)) + def _get_field_type( self, field: DefinedStructField | DefinedStructFixedField, @@ -139,8 +150,9 @@ def get_size_from_type( return 0 normalized_type = self._normalize_type(type_name) - if normalized_type in struct_lookup: - return struct_lookup[normalized_type] + struct_size = self._get_struct_size(normalized_type, struct_lookup) + if struct_size is not None: + return struct_size if normalized_type in enum_lookup: return enum_lookup[normalized_type] @@ -210,8 +222,9 @@ def _get_type_alignment( return 8 normalized_type = self._normalize_type(type_name) - if normalized_type in struct_lookup: - size = struct_lookup[normalized_type] + struct_size = self._get_struct_size(normalized_type, struct_lookup) + if struct_size is not None: + size = struct_size elif normalized_type in enum_lookup: size = enum_lookup[normalized_type] else: @@ -354,11 +367,13 @@ def build_export_string( ): output = IndentWriter() - struct_lookup = { - self._normalize_type(struct.type): struct.size - for struct in export.structs - if struct.size is not None - } + struct_lookup: dict[str, int] = {} + for struct in export.structs: + if struct.size is None: + continue + normalized_type = self._normalize_type(struct.type) + struct_lookup[normalized_type] = struct.size + struct_lookup.setdefault(self.get_short_name(normalized_type), struct.size) enum_lookup = { self._normalize_type(enum.type): self.get_size_from_string(enum.underlying) for enum in export.enums From 4aa94e7bcd5371093df0c8a1cc4526a3e3353102 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 7 Oct 2026 05:13:42 +0200 Subject: [PATCH 27/95] unfuck wrapper --- scripts/ida/ida_wrapper.py | 173 +------------------------------------ 1 file changed, 2 insertions(+), 171 deletions(-) diff --git a/scripts/ida/ida_wrapper.py b/scripts/ida/ida_wrapper.py index 95eb49c41c..e7ceb6d2db 100644 --- a/scripts/ida/ida_wrapper.py +++ b/scripts/ida/ida_wrapper.py @@ -145,7 +145,7 @@ def select_parser(self): """Sets the parser used for building struct data.""" ida_srclang.set_parser_argv( "clang", - "-x c++ -target x86_64-pc-windows-msvc -fms-compatibility -fms-extensions -fdelayed-template-parsing", + "-x c++ -target x86_64-pc-windows-msvc -fms-compatibility -fms-extensions -fdelayed-template-parsing -std=c++17", ) def parse_string(self, decl: str, vtables: list[str]): @@ -153,6 +153,7 @@ def parse_string(self, decl: str, vtables: list[str]): ida_srclang.parse_decls_with_parser("clang", None, decl, False) for vtable in vtables: self.mark_as_vtable(vtable) + def mark_as_vtable(vtbl_name: str) -> str | None: """ Mark a struct as a vtable (TAUDT_VFTABLE). @@ -373,176 +374,6 @@ def build_struct_string(self, struct: DefinedStruct, struct_lookup: dict[str, in return (struct_string_forward, struct_string_define, struct_string_vtables) - def parse_string(self, decl: str, vtables: list[str]): - """""" - ida_srclang.parse_decls_with_parser("clang", None, decl, False) - for vtable in vtables: - self.mark_as_vtable(vtable) - -class BaseIdaInterface(SrcInterface): - @abstractmethod - def get_struct_id(self, name): - pass - - @abstractmethod - def get_enum_id(self, name): - pass - - @abstractmethod - def delete_enum_members(self, eid: int): - """Remove all enum members - - Args: - eid (int): The id of the enum - """ - pass - - def mark_as_vtable(vtbl_name: str) -> str | None: - """ - Mark a struct as a vtable (TAUDT_VFTABLE). - Compatible with IDA 8.x and 9.x. - Returns an error string on failure, None on success. - """ - tif = ida_typeinf.tinfo_t() - - if not tif.get_named_type(None, vtbl_name): - return f"Type '{vtbl_name}' not found in type library" - - if not tif.is_struct(): - actual = "union" if tif.is_union() else "enum" if tif.is_enum() else "unknown" - return f"'{vtbl_name}' is not a struct (got {actual})" - - udt = ida_typeinf.udt_type_data_t() - if not tif.get_udt_details(udt): - return f"Failed to retrieve UDT details for '{vtbl_name}'" - - if udt.taudt_bits & ida_typeinf.TAUDT_VFTABLE: - return f"'{vtbl_name}' is already marked as a vtable" - - udt.taudt_bits |= ida_typeinf.TAUDT_VFTABLE - - if not tif.create_udt(udt, ida_typeinf.BTF_STRUCT): - return f"Failed to reconstruct UDT for '{vtbl_name}' after setting vtable flag" - - if _get_ida_major() >= 9: - result = tif.set_named_type(None, vtbl_name) - else: - result = tif.set_named_type( - ida_typeinf.get_idati(), - vtbl_name, - ida_typeinf.NTF_REPLACE - ) - - if not result: - return f"Failed to write back '{vtbl_name}' to type library" - - return None - - def enum_exists(self, name: str): - return self.get_enum_id(name) != idaapi.BADADDR - - def clean_name(self, name: str): - """Clean a name - - Args: - name (str): The name - - Returns: - str: The cleaned name - """ - return name - - def clean_struct_name(self, name: str): - """Clean a struct name - - Args: - name (str): The struct name - - Returns: - str: The cleaned struct name - """ - - if name == "Tm": - return "tm" # tm is a keyword in IDA for the time struct but C# exports it as Tm - return ( - name.replace(" ", "") - .replace("unsigned", "u") - .replace("__int64", "long") - .replace("__int32", "int") - .replace("__int16", "short") - .replace("__int8", "char") - ) - - def get_named_type(self, name: str): - """Retrieve a tinfo_t from the named type. - - Args: - name (str): Name of the type. - """ - - tinfo = ida_typeinf.tinfo_t() - clean_name = self.clean_struct_name(name) - if ( - self.get_struct_id(clean_name) != idaapi.BADADDR - or self.get_enum_id(clean_name) != idaapi.BADADDR - ): - if not tinfo.get_named_type(idaapi.get_idati(), clean_name): - raise ValueError("{0} not found in IDA database".format(clean_name)) - return tinfo - - if name == "void": - idaapi.parse_decl( - tinfo, idaapi.get_idati(), "void (__fastcall)();", idaapi.PT_SIL - ) - return tinfo.get_rettype() - - terminated = name + ";" - idaapi.parse_decl(tinfo, idaapi.get_idati(), terminated, idaapi.PT_SIL) - - tinfo_str = tinfo.dstr() - if tinfo_str == name or tinfo_str == clean_name: - return tinfo - - terminated = clean_name + ";" - idaapi.parse_decl(tinfo, idaapi.get_idati(), terminated, idaapi.PT_SIL) - - return tinfo - - def get_tinfo_from_type(self, raw_type: str, array_size=0): - """Retrieve a tinfo_t from a raw type string. - - Args: - raw_type (str): Raw type string. - array_size (int, optional): Size of the array. Defaults to 0. - """ - - type = raw_type.rstrip("*") - ptr_count = len(raw_type) - len(type) - - type_tinfo = self.get_named_type(type) - - ptr_tinfo = None - if ptr_count > 0: - for i in range(ptr_count): - ptr_tinfo = idaapi.tinfo_t() - if not ptr_tinfo.create_ptr(type_tinfo): - print("! failed to create pointer") - return None - type_tinfo = ptr_tinfo - else: - ptr_tinfo = type_tinfo - - if array_size > 0: - array_tinfo = idaapi.tinfo_t() - if not array_tinfo.create_array(ptr_tinfo, array_size): - print("! failed to create array") - return None - - ptr_tinfo = array_tinfo - - return ptr_tinfo - - def rename_struct(self, sid: int, new_name: str): """Rename a struct in IDA. From 70bb0f07478b0c6274877b13088cc9754934fde4 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 7 Oct 2026 05:14:00 +0200 Subject: [PATCH 28/95] cleanup and prepare importer --- scripts/ffxiv_structimporter.py | 685 +++----------------------------- 1 file changed, 60 insertions(+), 625 deletions(-) diff --git a/scripts/ffxiv_structimporter.py b/scripts/ffxiv_structimporter.py index eb5f64c01b..571d55b03d 100644 --- a/scripts/ffxiv_structimporter.py +++ b/scripts/ffxiv_structimporter.py @@ -13,25 +13,21 @@ from abc import abstractmethod from time import time from structs_schema import * +from src_wrapper import SrcInterface -predefine = """ -typedef char size8_st; -typedef unsigned char size8_t; -typedef short size16_st; -typedef unsigned short size16_t; -typedef int size32_st; -typedef unsigned int size32_t; -typedef long long size64_st; -typedef unsigned long long size64_t; +predefine = """#define _HAS_ITERATOR_DEBUGGING 0 +#define _ITERATOR_DEBUG_LEVEL 0 +#include +#include +#include +#include +#include +#include +#include -#define DTOR(type) type *(__fastcall *Dtor)(type *__hidden self, size8_t freeFlags); """ -struct_string_forward = "" -struct_string_define = "" - - class BaseApi: @abstractmethod def can_run(self): @@ -431,7 +427,7 @@ def get_vfunc_ea_from_data(self, class_name, offset): def delete_struct_members(self, fullname): # type: (str) -> None - self.remove_struct_members(self.get_struct_id(fullname)) + pass @property def get_file_path(self): @@ -440,626 +436,56 @@ def get_file_path(self): ) def generate_hashed_type_name(self, name: str) -> str: - name = self.clean_struct_name(name) - - name = hashlib.sha1(name.encode()).hexdigest() - - return "struc_" + name + return "" def get_srclang_type_name(self, name: str) -> str: - if not self.srclang_importer: - return name - - ptr_count = 0 - i = len(name) - 1 - while i >= 0 and name[i] == '*': - ptr_count += 1 - i -= 1 - - full_name = name - if ptr_count > 0: - name = name.strip("*") - - cname = self.srclang_types.get(name) - if cname is None: - cname = self.srclang_types.get(self.clean_struct_name(name)) - - if cname is not None: - return cname + "*" * ptr_count - - return full_name + return "" def can_run(self): return self.enum_exists("Component::Exd::SheetsEnum") def create_enum_struct(self, enum): # type: (DefinedStructEnum) -> None - fullname = enum.type - - e = self.get_enum_id(fullname) - if e == idaapi.BADADDR: - e = self.create_enum(fullname) - - self.set_enum_width(e, self.get_size_from_ida_type(enum.underlying)) - if self.is_signed(enum.underlying): - self.set_enum_flag(e, 0x20000) - if enum.flags: - if idaapi.IDA_SDK_VERSION < 900: - self.add_enum_member(e, "{0}.{1}".format(enum.name, "tmp"), self.get_enum_default_mask(e)) - self.set_enum_as_bf(e) - for value in enum.values: - self.add_enum_member( - e, "{0}.{1}".format(enum.name, value), enum.values[value] - ) - if enum.flags and idaapi.IDA_SDK_VERSION < 900: - self.remove_enum_member(e, "tmp", enum.name) + return def delete_enum(self, enum): # type: (DefinedStructEnum) -> None - eid = idc.get_enum(enum.type) - if eid != idaapi.BADADDR: - self.delete_enum_members(eid) - idc.set_enum_bf(eid, False) + return def delete_struct(self, struct): # type: (DefinedStruct) -> None - idaapi.begin_type_updating(idaapi.UTP_STRUCT) - fullname = self.clean_struct_name(struct.type) - self.delete_struct_members(fullname) - self.delete_struct_members(fullname + "_vtbl") - - # also check C types in case of an incomplete C run - if self.srclang_importer: - cname = self.get_srclang_type_name(fullname) - self.delete_struct_members(cname) - self.delete_struct_members(cname + "_vtbl") - - idaapi.end_type_updating(idaapi.UTP_STRUCT) + return def create_struct(self, struct): # type: (DefinedStruct) -> None - - # TODO: change this over to ida_srclang - fullname = self.clean_struct_name(struct.type) - - if self.srclang_importer: - # rename C++ -> C or create new C struct - cname = self.generate_hashed_type_name(fullname) - self.srclang_types[fullname] = cname - - sid = self.get_struct_id(fullname) - if sid == idaapi.BADADDR: - if self.get_struct_id(cname) == idaapi.BADADDR: - self.create_struct_type(cname, struct.union) - else: - self.rename_struct(sid, cname) - - if not struct.virtual_functions: - return - - sid = self.get_struct_id(fullname + "_vtbl") - if sid == idaapi.BADADDR: - if self.get_struct_id(cname + "_vtbl") == idaapi.BADADDR: - self.create_struct_type(cname + "_vtbl") - else: - self.rename_struct(sid, cname + "_vtbl") - - return - - if self.get_struct_id(fullname) == idaapi.BADADDR: - self.create_struct_type(fullname, struct.union) - - if struct.virtual_functions: - self.create_struct_type(fullname + "_vtbl") + return def validate_srclang_struct(self, struct: DefinedStruct): - cname = self.get_srclang_type_name(self.clean_struct_name(struct.type)) - sid = self.get_struct_id(cname) - if sid == idaapi.BADADDR: - ida_kernwin.warning(f"Struct {cname} ({struct.type}) not found during validation") - exit() - - ti = self.get_struct(sid) - - expected_base_offsets = {} - for field in struct.fields: - if not field.srclang_is_baseclass: - continue - expected_base_offsets[field.offset] = ( - expected_base_offsets.get(field.offset, 0) + 1 - ) - - if expected_base_offsets: - udt = ida_typeinf.udt_type_data_t() - if not ti.get_udt_details(udt): - ida_kernwin.warning( - f"Could not find baseclass for {cname} ({struct.type}) during validation" - ) - exit() - - actual_base_offsets = {} - for udm in udt: - if not udm.is_baseclass(): - continue - offset = int(udm.offset / 8) - actual_base_offsets[offset] = actual_base_offsets.get(offset, 0) + 1 - - for offset, expected_count in expected_base_offsets.items(): - actual_count = actual_base_offsets.get(offset, 0) - if actual_count < expected_count: - ida_kernwin.warning( - f"Baseclass offset mismatch in struct {cname} ({struct.type}).\n" - f"Expected {expected_count} member(s) for baseclass at {offset}, got {actual_count}" - ) - exit() - - last_offset = -1 - for field in struct.fields: - if field.offset == last_offset: - continue - - last_offset = field.offset - - if field.offset is None or field.srclang_is_baseclass: - continue - - if field.offset == 0 and field.name == "_vtable": - continue - - # ugly hack to workaround duplicate field names in structs - field_name = field.srclang_field_name - - (idx, udm) = ti.get_udm(field_name) - if idx == -1: - ida_kernwin.warning(f"Field {field_name} not found in struct {cname} ({struct.type}) during validation") - exit() - - if (udm.offset / 8) != field.offset: - ida_kernwin.warning(f"Field \"{field_name}\" offset mismatch in struct {cname} ({struct.type}) during validation.\nExpected {field.offset}, got {(udm.offset/8)}") - exit() + pass def get_srclang_fill_type(self, available_bytes: int, current_offset: int) -> tuple[str, int]: - if available_bytes >= 8 and (current_offset % 8) == 0: - return ("__int64", 8) - elif available_bytes >= 4 and (current_offset % 4) == 0: - return ("__int32", 4) - elif available_bytes >= 2 and (current_offset % 2) == 0: - return ("__int16", 2) - else: - return ("char", 1) + return ("", 0) def append_srclang_padding(self, decl: list[str], current_size: int, target_size: int) -> int: - while current_size < target_size: - if self.full_padding: - (fill_type, fill_size) = self.get_srclang_fill_type( - target_size - current_size, current_size - ) - decl.append(f"{fill_type} field_{current_size:X};") - current_size += fill_size - else: - arr_size = target_size - current_size - decl.append(f"char field_{current_size:X}[{arr_size}];") - current_size += arr_size - - return current_size + return 0 def create_srclang_decl(self, struct: DefinedStruct) -> str: - fullname = self.get_srclang_type_name(self.clean_struct_name(struct.type)) - - decl = [ "_" ] # placeholder, filled after we determine base classes - - cur_size = 0 - - contiguous_fields = True - - seen_fields = {} - - inherits_from = [] - - has_explicit_vtable = (len(struct.fields) != 0 and struct.fields[0].name == "_vtable") - - if struct.virtual_functions != None or has_explicit_vtable: - # the placeholder will force IDA to mark the _vtbl struct as a VFT - # and attach it to this struct - decl.append("virtual void _placeholder();") - - # offset for vfptr if needed - if struct.srclang_needs_vfptr or has_explicit_vtable: - cur_size += 8 - - last_field_offset = -1 - for field in struct.fields: - offset = field.offset - - # skip explicit vtable fields - if offset == 0 and field.name == "_vtable": - continue - - if offset == last_field_offset and not struct.union: - # NOTE In IDA versions < 9.0 you could have overlapping fields or an automatically created union. - # We're not able to support this for srclang, so overlapping fields should ideally be - # treated as a layout error which requires a union to resolve. - # - # I've made the decision here to drop these with a warning, but it is probably worth evaluating - # whether it's worthwhile to make this an error later. - print(f"Skipping {struct.type}.{field.name} as it is at a duplicate offset.") - continue - - last_field_offset = offset - - if offset > cur_size: - contiguous_fields = False - cur_size = self.append_srclang_padding(decl, cur_size, offset) - - field_is_base = field.base and contiguous_fields - field_name = ( - field.name - if not field_is_base - else f"baseclass_{offset:X}" - ) - field.srclang_is_baseclass = field_is_base - - # ugly hack to workaround duplicate field names in structs - if field_name in seen_fields: - next_index = seen_fields[field_name] + 1 - seen_fields[field_name] = next_index - - field_name += f"_{next_index}" - else: - seen_fields[field_name] = 1 - - field.srclang_field_name = field_name - - array_size = field.size if hasattr(field, "size") else 0 - - field_type = self.clean_name(field.type) - if field_type == "__fastcall": - field_decl = self.get_srclang_type_name(self.clean_name(field.return_type)) - field_decl = field_decl + "(__fastcall* " + field_name + ")(" - for param in field.parameters: - field_decl = field_decl + self.get_srclang_type_name(self.clean_name(param.type)) + "" - field_decl = field_decl + param.name + "," - field_decl = field_decl[:-2] + ")" - - decl.append(field_decl) - cur_size += 8 - - continue - - field_size = 0 - - # struct type - if self.get_idc_type_from_ida_type( - self.get_srclang_type_name(self.clean_struct_name(field_type)) - ) == self.get_struct_flag(): - field_type = self.get_srclang_type_name(self.clean_struct_name(field_type)) - - tinfo = self.get_tinfo_from_type(field_type) - field_size = tinfo.get_size() - - # enum type - elif ( - self.get_idc_type_from_ida_type(field_type) - == self.get_enum_flag() - ): - field_size = idc.get_enum_width(self.get_enum_id(field_type)) - - # primitive type - else: - field_size = self.get_size_from_ida_type(field_type) - - if field_type.endswith("*"): - field_type = self.get_srclang_type_name(field_type) - - field_decl = f"{field_type} {field_name}" - if array_size > 0: - field_size *= array_size - field_decl += f"[{array_size}];" - else: - field_decl += ";" - - if field_is_base: - inherits_from.append(field_type) - else: - decl.append(field_decl) - - cur_size += field_size - - if struct.size is not None and struct.size != 0: - cur_size = self.append_srclang_padding(decl, cur_size, struct.size) - - decl.append("};") - - # set struct type - if struct.union: - decl[0] = f"union {fullname} " - else: - decl[0] = f"struct __attribute__((packed)) {fullname} " - if len(inherits_from) > 0: - inheritances = ", ".join(inherits_from) - decl[0] += f": {inheritances}" - - decl[0] += " {" - - return "\n".join(decl) + return "" def create_struct_member_fill(self, struct_name, offset): # type: (str, int) -> None - s = self.get_struct(self.get_struct_id(struct_name)) - prev_size = self.get_struct_size(s) - if self.full_padding: - flag = self.get_idc_type_from_size(prev_size) - size = self.get_size_from_idc_type(flag) - if size > offset - prev_size: - flag = self.get_idc_type_from_size( - offset - prev_size, prev_size - ) - size = self.get_size_from_idc_type(flag) - - self.create_struct_member( - s, "field_{0:X}".format(prev_size), prev_size, flag, None, size - ) - else: - self.create_struct_member( - s, - "field_{0:X}".format(prev_size), - prev_size, - ida_bytes.byte_flag(), - None, - offset - prev_size, - ) + pass def create_struct_members(self, struct): # type: (DefinedStruct) -> None - idaapi.begin_type_updating(idaapi.UTP_STRUCT) - - if self.srclang_importer: - idaapi.begin_type_updating(idaapi.UTP_STRUCT) - - decl = self.create_srclang_decl(struct) - num_errors = ida_srclang.parse_decls_for_srclang( - ida_srclang.SRCLANG_C, - None, - decl, - False - ) - - if num_errors != 0: - # show messagebox - print(f"above errors occurred while parsing the following:\n---\n{decl}\n---") - ida_kernwin.warning(f"Error parsing srclang decl for {struct.type}, please see errors in Output window.") - exit() - - if struct.virtual_functions is not None: - # delete the _placeholder function from the VFT - cname = self.srclang_types[self.clean_struct_name(struct.type)] - sid = self.get_struct_id(f"{cname}_vtbl") - tinfo = self.get_struct(sid) - tinfo.del_udm(0) - - idaapi.end_type_updating(idaapi.UTP_STRUCT) - - self.validate_srclang_struct(struct) - return - - fullname = self.clean_struct_name(struct.type) - - tid = self.get_struct_id(fullname) - if tid == idaapi.BADADDR: - print("Error: Struct {0} not found when trying to create members".format(fullname)) - return - - s = self.get_struct(tid) - - if struct.virtual_functions != None and ( - struct.fields == [] or struct.fields[0].offset > 0 - ): - self.create_struct_member( - s, "__vftable", 0, ida_bytes.qword_flag(), None, 8 - ) - type = fullname + "_vtbl*" if struct.virtual_functions else "void**" - meminfo = self.get_struct_member_by_name(s, "__vftable") - self.set_struct_member_info( - s, meminfo, 0, self.get_tinfo_from_type(type), 0, False - ) - - contiguous_fields = True - for field in struct.fields: - offset = field.offset - - prev_size = self.get_struct_size(s) - while offset > prev_size: - contiguous_fields = False - self.create_struct_member_fill(fullname, offset) - prev_size = self.get_struct_size(s) - - field_is_base = field.base and contiguous_fields - field_name = ( - field.name - if not field_is_base - else "baseclass_{0:X}".format(offset) - ) - field_type = self.clean_name(field.type) - if field_type == "__fastcall": - self.create_struct_member( - s, - field_name, - offset, - self.get_idc_type_from_ida_type("__int64"), - None, - self.get_size_from_ida_type("__int64"), - ) - field_type = self.clean_name(field.return_type) - field_type = field_type + "(__fastcall* " + field_name + ")(" - for param in field.parameters: - field_type = field_type + self.clean_name(param.type) + "" - field_type = field_type + param.name + "," - field_type = field_type[:-2] + ")" - elif ( - self.get_idc_type_from_ida_type( - self.clean_struct_name(field_type) - ) - == self.get_struct_flag() - ): - field_type = self.clean_struct_name(field_type) - self.create_struct_member( - s, - field_name, - offset, - self.get_idc_type_from_ida_type(field_type), - self.get_struct_opinfo_from_type(field_type), - self.get_size_from_ida_type(field_type), - ) - elif ( - self.get_idc_type_from_ida_type(field_type) - == self.get_enum_flag() - ): - self.create_struct_member( - s, - field_name, - offset, - self.get_idc_type_from_ida_type(field_type), - self.get_enum_opinfo_from_type(field_type), - self.get_size_from_ida_type(field_type), - ) - else: - self.create_struct_member( - s, - field_name, - offset, - self.get_idc_type_from_ida_type(field_type), - None, - self.get_size_from_ida_type(field_type), - ) - - meminfo = self.get_struct_member_by_name(s, field_name) - if meminfo is not None: - if field_is_base: - if idaapi.IDA_SDK_VERSION >= 900: - meminfo.set_baseclass() - else: - meminfo.props |= self.get_base_class_flag() - - array_size = field.size if hasattr(field, "size") else 0 - self.set_struct_member_info( - s, - meminfo, - 0, - self.get_tinfo_from_type(field_type, array_size), - 0, - field.is_string if hasattr(field, "is_string") and (field_type == "char" or field_type == "wchar_t") else False - ) - - if struct.size is not None and struct.size != 0: - prev_size = self.get_struct_size(s) - while struct.size > prev_size: - self.create_struct_member_fill(fullname, struct.size) - prev_size = self.get_struct_size(s) - - idaapi.end_type_updating(idaapi.UTP_STRUCT) + pass def create_vtable(self, struct): # type: (DefinedStruct) -> None - fullname = self.clean_name(struct.type) - s = self.get_struct(self.get_struct_id(fullname + "_vtbl")) - for virt_func in struct.virtual_functions: - if virt_func is None: - continue - - offset = virt_func.offset - field_name = virt_func.name - self.create_struct_member( - s, - field_name, - offset, - self.get_idc_type_from_ida_type("__int64"), - None, - self.get_size_from_ida_type("__int64"), - ) - if virt_func.return_type == None or virt_func.parameters == None: - continue - - meminfo = self.get_struct_member_by_name(s, field_name) - if meminfo is None: - raise RuntimeError("Failed to find member {0} in struct {1}".format(field_name, fullname)) - - field_type = self.clean_name(virt_func.return_type) - field_type = field_type + "(__fastcall* " + field_name + ")(" - for param in virt_func.parameters: - field_type = field_type + self.clean_name(param.type) + " " - field_type = field_type + param.name + "," - field_type = field_type[:-1] + ")" - - self.set_struct_member_info( - s, meminfo, 0, self.get_tinfo_from_type(field_type), 0, False - ) - if struct.vtable_size: - size = int(struct.vtable_size / 8) - else: - size = int(self.get_struct_size(s) / 8) - - for i in range(size): - offset = i * 8 - member_id = self.get_struct_member_id(s, offset) - if idaapi.IDA_SDK_VERSION >= 900: - udm_idx, udm = s.get_udm_by_offset(offset * 8) - - # srclang ends up with gap UDMs in VFTs that need to be deleted and filled back out as vf## placeholders - is_vft_gap = ( - udm is not None - and udm.name.startswith("gap") - and udm.type.dstr().startswith("_BYTE[") - ) - if is_vft_gap: - result = s.del_udm(udm_idx) - if result == ida_typeinf.TERR_OK: - member_id = idc.BADADDR - else: - print( - f"Error: failed to delete VFT gap UDM {fullname}.{udm.name} " - f"at slot {i}: {ida_typeinf.tinfo_errstr(result)}" - ) - if member_id == idc.BADADDR: - name = "vf{0}".format(i) - fallback_name = self.get_fallback_vfunc_name(struct.type, i) - if fallback_name: - name = fallback_name - - self.create_struct_member( - s, - name, - i * 8, - self.get_idc_type_from_ida_type("__int64"), - None, - self.get_size_from_ida_type("__int64"), - ) - meminfo = self.get_struct_member_by_name(s, name) - self.set_struct_member_info( - s, meminfo, 0, self.get_tinfo_from_type("__int64"), 0, False - ) + pass def finalise_struct(self, struct: DefinedStruct): - if not self.srclang_importer: - return - - fullname = self.clean_struct_name(struct.type) - cname = self.get_srclang_type_name(fullname) - - sid = self.get_struct_id(cname) - if sid == idaapi.BADADDR: - ida_kernwin.warning(f"Failed to find and finalise struct {cname}") - - self.rename_struct(sid, fullname) - - if not struct.virtual_functions: - return - - sid = self.get_struct_id(cname + "_vtbl") - if sid == idaapi.BADADDR: - ida_kernwin.warning(f"Failed to find and finalise vtable for struct {cname}") - return - - self.rename_struct(sid, fullname + "_vtbl") + pass def create_union(self, struct): # type: (DefinedStruct) -> None @@ -2119,37 +1545,46 @@ def run(): print("{0} Preprocessing yaml".format(get_time())) api.preprocess_yaml(yaml) - print("{0} Deleting old structs".format(get_time())) - for struct in yaml.structs[::-1]: - api.delete_struct(struct) - - print("{0} Deleting old enums and creating new ones".format(get_time())) - for enum in yaml.enums: - api.delete_enum(enum) - - for enum in yaml.enums: - api.create_enum_struct(enum) + if isinstance(api, IdaApi): + print("{0} Generating src file in memory".format(get_time())) + src_interface = SrcInterface() + header, vtables = src_interface.build_export_string(yaml) - print("{0} Creating new structs".format(get_time())) - for struct in yaml.structs: - api.create_struct(struct) + print("{0} Importing src file".format(get_time())) + api.select_parser() + api.parse_string(header, vtables) + else: + print("{0} Deleting old structs".format(get_time())) + for struct in yaml.structs[::-1]: + api.delete_struct(struct) + + print("{0} Deleting old enums and creating new ones".format(get_time())) + for enum in yaml.enums: + api.delete_enum(enum) + + for enum in yaml.enums: + api.create_enum_struct(enum) + + print("{0} Creating new structs".format(get_time())) + for struct in yaml.structs: + api.create_struct(struct) - print("{0} Creating members for structs".format(get_time())) - for struct in yaml.structs: - api.create_struct_members(struct) + print("{0} Creating members for structs".format(get_time())) + for struct in yaml.structs: + api.create_struct_members(struct) - print("{0} Finalising structs".format(get_time())) - for struct in yaml.structs: - api.finalise_struct(struct) + print("{0} Finalising structs".format(get_time())) + for struct in yaml.structs: + api.finalise_struct(struct) - print("{0} Creating vtables for structs".format(get_time())) - for struct in yaml.structs: - if struct.virtual_functions: - api.create_vtable(struct) + print("{0} Creating vtables for structs".format(get_time())) + for struct in yaml.structs: + if struct.virtual_functions: + api.create_vtable(struct) - print("{0} Mapping unions/vtables for structs".format(get_time())) - for struct in yaml.structs: - api.create_union(struct) + print("{0} Mapping unions/vtables for structs".format(get_time())) + for struct in yaml.structs: + api.create_union(struct) if update_virt_func: for struct in yaml.structs: From 6f98421ffa6cd4b8e01ead0379264af5dfca89e7 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 7 Oct 2026 05:15:30 +0200 Subject: [PATCH 29/95] save test write --- scripts/test_execute.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/scripts/test_execute.py b/scripts/test_execute.py index 2c404534e1..0cb90a56a3 100644 --- a/scripts/test_execute.py +++ b/scripts/test_execute.py @@ -4,10 +4,11 @@ from structs_schema import get_yaml from src_wrapper import SrcInterface +from exdschema_wrapper import get_exdschema_data, create_struct_from_header_and_schema +from lumina_wrapper import get_excel_header_files from io import open -predef_header = """ -#define _HAS_ITERATOR_DEBUGGING 0 +predef_header = """#define _HAS_ITERATOR_DEBUGGING 0 #define _ITERATOR_DEBUG_LEVEL 0 #include #include @@ -17,7 +18,6 @@ #include #include -#define DTOR(type) type *(__fastcall *Dtor)(type *__hidden self, uint8_t freeFlags); """ src = SrcInterface() @@ -27,7 +27,18 @@ header, vtables = src.build_export_string(struct_export) +excel_header_files = get_excel_header_files() +exdschema = get_exdschema_data("latest") + +struct_export_excel, excel_map = create_struct_from_header_and_schema( + excel_header_files, exdschema +) + +header_excel, vtables_excel = src.build_export_string(struct_export_excel, 4) + with open("test.h", "w") as f: + f.write(predef_header) + f.write(header_excel) f.write(header) running_time = time() - current_time From 85a18c5fb1fbb6f3ee7370ac03be9bffc82a3586 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 7 Oct 2026 15:51:14 +0200 Subject: [PATCH 30/95] fix python data not being correctly set --- scripts/ffxiv_structimporter.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/ffxiv_structimporter.py b/scripts/ffxiv_structimporter.py index 571d55b03d..e60b09be7d 100644 --- a/scripts/ffxiv_structimporter.py +++ b/scripts/ffxiv_structimporter.py @@ -159,6 +159,9 @@ def get_yaml(self): static_members = None for field in struct["fields"]: base = field["base"] if "base" in field else False + template_types = [] + if "template_args" in field: + template_types = field["template_args"] if "size" in field: fields.append( DefinedStructFixedField( @@ -166,6 +169,7 @@ def get_yaml(self): field["type"], field["offset"], base, + template_types, field["size"], field["is_string"] ) @@ -182,6 +186,7 @@ def get_yaml(self): field["type"], field["offset"], base, + template_types, field["return_type"], parameters, ) @@ -189,7 +194,8 @@ def get_yaml(self): else: fields.append( DefinedStructField( - field["name"], field["type"], field["offset"], base + field["name"], field["type"], field["offset"], base, + template_types ) ) if "virtual_functions" in struct: From d2dd5e2d43035df11affeb9d6480e1bbf3795c53 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:14:31 +0200 Subject: [PATCH 31/95] correct check for what is in yml --- scripts/ffxiv_structimporter.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/scripts/ffxiv_structimporter.py b/scripts/ffxiv_structimporter.py index e60b09be7d..c6c9c20769 100644 --- a/scripts/ffxiv_structimporter.py +++ b/scripts/ffxiv_structimporter.py @@ -217,18 +217,19 @@ def get_yaml(self): parameters, ) ) - for memfunc in struct["member_functions"]: - parameters = [] - for param in memfunc["parameters"]: - parameters.append(DefinedStructFuncParam(param["name"], param["type"])) - member_functions.append( - DefinedStructMemFunc( - memfunc["signature"], - memfunc["return_type"], - parameters, - memfunc["name"], + if "member_functions" in struct: + for memfunc in struct["member_functions"]: + parameters = [] + for param in memfunc["parameters"]: + parameters.append(DefinedStructFuncParam(param["name"], param["type"])) + member_functions.append( + DefinedStructMemFunc( + memfunc["signature"], + memfunc["return_type"], + parameters, + memfunc["name"], + ) ) - ) if "static_member_functions" in struct: static_member_functions = [] for smemfunc in struct["static_member_functions"]: From 0ba3bc9f77427a61bbb498938350a4f540735b4d Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:19:02 +0200 Subject: [PATCH 32/95] actually clean up and use struct_schema get_yaml function --- scripts/ffxiv_structimporter.py | 145 +------------------------------- 1 file changed, 1 insertion(+), 144 deletions(-) diff --git a/scripts/ffxiv_structimporter.py b/scripts/ffxiv_structimporter.py index c6c9c20769..c21eee5b84 100644 --- a/scripts/ffxiv_structimporter.py +++ b/scripts/ffxiv_structimporter.py @@ -137,150 +137,7 @@ def get_file_path(self): def get_yaml(self): # type: () -> DefinedStructExport with open(self.get_file_path, "r") as fd: - dic = load(fd, Loader=Loader) # type: dict[str, dict[str, list[dict[str, str | int | list[dict[str, str | int]]]]]] - enums = [] - structs = [] - for enum in dic["enums"]: - enums.append( - DefinedStructEnum( - enum["name"], - enum["type"], - enum["namespace"], - enum["underlying"], - enum["flags"], - enum["values"], - ) - ) - for struct in dic["structs"]: - fields = [] - virtual_functions = None - member_functions = [] - static_member_functions = None - static_members = None - for field in struct["fields"]: - base = field["base"] if "base" in field else False - template_types = [] - if "template_args" in field: - template_types = field["template_args"] - if "size" in field: - fields.append( - DefinedStructFixedField( - field["name"], - field["type"], - field["offset"], - base, - template_types, - field["size"], - field["is_string"] - ) - ) - elif "return_type" in field: - parameters = [] - for param in field["parameters"]: - parameters.append( - DefinedStructFuncParam(param["name"], param["type"]) - ) - fields.append( - DefinedStructFuncField( - field["name"], - field["type"], - field["offset"], - base, - template_types, - field["return_type"], - parameters, - ) - ) - else: - fields.append( - DefinedStructField( - field["name"], field["type"], field["offset"], base, - template_types - ) - ) - if "virtual_functions" in struct: - virtual_functions = [] - for vfunc in struct["virtual_functions"]: - parameters = ( - [ - DefinedStructFuncParam(param["name"], param["type"]) - for param in vfunc["parameters"] - ] - if "parameters" in vfunc - else None - ) - virtual_functions.append( - DefinedStructVFunc( - vfunc["name"], - vfunc["return_type"] if "return_type" in vfunc else None, - vfunc["offset"], - parameters, - ) - ) - if "member_functions" in struct: - for memfunc in struct["member_functions"]: - parameters = [] - for param in memfunc["parameters"]: - parameters.append(DefinedStructFuncParam(param["name"], param["type"])) - member_functions.append( - DefinedStructMemFunc( - memfunc["signature"], - memfunc["return_type"], - parameters, - memfunc["name"], - ) - ) - if "static_member_functions" in struct: - static_member_functions = [] - for smemfunc in struct["static_member_functions"]: - parameters = [] - for param in smemfunc["parameters"]: - parameters.append( - DefinedStructFuncParam(param["name"], param["type"]) - ) - static_member_functions.append( - DefinedStructMemFunc( - smemfunc["signature"], - smemfunc["return_type"], - parameters, - smemfunc["name"], - ) - ) - if "static_members" in struct: - static_members = [] - for sm in struct["static_members"]: - static_members.append( - DefinedStructStaticMember( - sm["name"], - sm["signature"], - sm["relative_follow_offsets"], - sm["return_type"], - sm["is_pointer"] if "is_pointer" in sm else False, - ) - ) - size = None - if "size" in struct: - size = struct["size"] - vtable_size = None - if "vtable_size" in struct: - vtable_size = struct["vtable_size"] - structs.append( - DefinedStruct( - struct["name"], - struct["type"], - struct["namespace"], - fields, - size, - vtable_size, - virtual_functions, - member_functions, - struct["union"], - static_member_functions, - static_members, - [] - ) - ) - return DefinedStructExport(enums, structs) + return get_yaml(fd) @abstractmethod def preprocess_yaml(self, yml: DefinedStructExport): From 1618e545494af7f6c4989915e0224cad0b8791a6 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:32:11 +0200 Subject: [PATCH 33/95] fix improper function definition --- scripts/ida/ida_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ida/ida_wrapper.py b/scripts/ida/ida_wrapper.py index e7ceb6d2db..b2b785b3ec 100644 --- a/scripts/ida/ida_wrapper.py +++ b/scripts/ida/ida_wrapper.py @@ -154,7 +154,7 @@ def parse_string(self, decl: str, vtables: list[str]): for vtable in vtables: self.mark_as_vtable(vtable) - def mark_as_vtable(vtbl_name: str) -> str | None: + def mark_as_vtable(self, vtbl_name: str) -> str | None: """ Mark a struct as a vtable (TAUDT_VFTABLE). From 4acfc79caec90fc882e5573aeb9757f32f29775e Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:33:03 +0200 Subject: [PATCH 34/95] change this to self for first arg of member and vtable functions --- CExporter/Exporter.TypeProcessing.cs | 4 ++-- CExporter/Exporter.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CExporter/Exporter.TypeProcessing.cs b/CExporter/Exporter.TypeProcessing.cs index de1c68d81b..b27828e896 100644 --- a/CExporter/Exporter.TypeProcessing.cs +++ b/CExporter/Exporter.TypeProcessing.cs @@ -310,7 +310,7 @@ private static ProcessedField ProcessParameter(ParameterInfo parameter) { private static ProcessedField ProcessVirtualParameter(Type parameter, int i, ParameterInfo[]? parameters) { var parameterInfo = i == 0 ? null : parameters?[i - 1]; - var name = i == 0 ? "this" : parameterInfo?.Name ?? $"a{i + 1}"; + var name = i == 0 ? "self" : parameterInfo?.Name ?? $"a{i + 1}"; return ProcessParameter( parameter, name, @@ -457,7 +457,7 @@ private static ProcessedField ProcessParameter( FieldTypeOverride = memberFunctionClass.FixTypeName() + "*", FieldType = memberFunctionClass, FieldOffset = -1, - FieldName = "this", + FieldName = "self", Bits = [], IsUnion = false, UnionValues = [] diff --git a/CExporter/Exporter.cs b/CExporter/Exporter.cs index 6a84ee3c0d..90ab52e922 100644 --- a/CExporter/Exporter.cs +++ b/CExporter/Exporter.cs @@ -248,7 +248,7 @@ public static void ProcessDefinedVTables(DataDefinition def) { vparams = [ new ProcessedField{ FieldType = vreturnType, - FieldName = "this", + FieldName = "self", Bits = [], IsUnion = false, UnionValues = [], From 63142e0c0f0493c76add00c47dfcb4c830d99082 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:34:13 +0200 Subject: [PATCH 35/95] try to isolate dotnet runtime in python --- scripts/lumina_wrapper.py | 112 ++++++++++++++++++++------------------ 1 file changed, 59 insertions(+), 53 deletions(-) diff --git a/scripts/lumina_wrapper.py b/scripts/lumina_wrapper.py index 34cd373bca..f58ce9dd62 100644 --- a/scripts/lumina_wrapper.py +++ b/scripts/lumina_wrapper.py @@ -6,15 +6,12 @@ from sys import platform, path from shutil import rmtree from xml.dom import pulldom -from pythonnet import load as dotnetload +from pythonnet import load as dotnetload, unload from json import load as jsonload from enum import IntEnum from dataclasses import dataclass import gc -dotnetload("coreclr") -import clr - tempdirname = join(getcwd(), "luminatmp") try: rmtree(tempdirname) @@ -146,72 +143,81 @@ def get_nugpkg(package: str, version: str = ""): def get_excel_header_files() -> dict[str, tuple[int, list[ExcelColumnDefinition]]]: - deps = get_nugpkg("Lumina") - deps_gotten: dict[str, str] = {} + dotnetload("coreclr") + def _isolated_clr(): + import clr + + deps = get_nugpkg("Lumina") + deps_gotten: dict[str, str] = {} - while len(deps) > 0: - local_deps = [] - for package, version in deps: - if package not in deps_gotten or deps_gotten[package] != version: - local_deps.extend(get_nugpkg(package, version)) - deps_gotten[package] = version - deps = local_deps + while len(deps) > 0: + local_deps = [] + for package, version in deps: + if package not in deps_gotten or deps_gotten[package] != version: + local_deps.extend(get_nugpkg(package, version)) + deps_gotten[package] = version + deps = local_deps - path.append(tempdirname) + path.append(tempdirname) - from System.Runtime.Loader import AssemblyLoadContext # type: ignore - from System import Activator, GC # type: ignore + from System.Runtime.Loader import AssemblyLoadContext # type: ignore + from System import Activator, GC # type: ignore - ctx = AssemblyLoadContext("temp", isCollectible=True) + ctx = AssemblyLoadContext("temp", isCollectible=True) - def _isolated_extract(context): - assemblies: dict = {} + def _isolated_extract(context): + assemblies: dict = {} - for file in listdir(tempdirname): - if file.endswith(".dll"): - assemblies[file] = context.LoadFromAssemblyPath( - abspath(join(tempdirname, file)) - ) + for file in listdir(tempdirname): + if file.endswith(".dll"): + assemblies[file] = context.LoadFromAssemblyPath( + abspath(join(tempdirname, file)) + ) - assembly = assemblies["Lumina.dll"] - game_data_type = assembly.GetType("Lumina.GameData") - game_data_instance = Activator.CreateInstance( - game_data_type, [join(game_path, "game", "sqpack"), None] - ) + assembly = assemblies["Lumina.dll"] + game_data_type = assembly.GetType("Lumina.GameData") + game_data_instance = Activator.CreateInstance( + game_data_type, [join(game_path, "game", "sqpack"), None] + ) - excel_list_type = assembly.GetType("Lumina.Data.Files.Excel.ExcelListFile") - excel_header_type = assembly.GetType("Lumina.Data.Files.Excel.ExcelHeaderFile") + excel_list_type = assembly.GetType("Lumina.Data.Files.Excel.ExcelListFile") + excel_header_type = assembly.GetType("Lumina.Data.Files.Excel.ExcelHeaderFile") - root_exl = game_data_instance.GetFile[excel_list_type]("exd/root.exl") + root_exl = game_data_instance.GetFile[excel_list_type]("exd/root.exl") - excel_header_files: dict[str, tuple[int, list[ExcelColumnDefinition]]] = {} + excel_header_files: dict[str, tuple[int, list[ExcelColumnDefinition]]] = {} - for kvp in root_exl.ExdMap: - key = kvp.Key - value = kvp.Value + for kvp in root_exl.ExdMap: + key = kvp.Key + value = kvp.Value - if value != -1: - defs: list[ExcelColumnDefinition] = [] - header = game_data_instance.GetFile[excel_header_type](f"exd/{key}.exh") - for column_definition in header.ColumnDefinitions: - defs.append( - ExcelColumnDefinition( - int(column_definition.Type), column_definition.Offset + if value != -1: + defs: list[ExcelColumnDefinition] = [] + header = game_data_instance.GetFile[excel_header_type](f"exd/{key}.exh") + for column_definition in header.ColumnDefinitions: + defs.append( + ExcelColumnDefinition( + int(column_definition.Type), column_definition.Offset + ) ) - ) - excel_header_files[key] = (value, sorted(defs)) - + excel_header_files[key] = (value, sorted(defs)) + - return excel_header_files + return excel_header_files + + excel_header_files = _isolated_extract(ctx) - excel_header_files = _isolated_extract(ctx) + ctx.Unload() - ctx.Unload() + del ctx + + gc.collect() + GC.Collect() + GC.WaitForPendingFinalizers() + return excel_header_files - del ctx + excel_header_files = _isolated_clr() - gc.collect() - GC.Collect() - GC.WaitForPendingFinalizers() + unload() return excel_header_files From e20dca024defd827148bcf558ae2ec86a3f2f4e6 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:53:39 +0200 Subject: [PATCH 36/95] remove a bunch of unneded stuff --- scripts/ffxiv_structimporter.py | 471 +------------------------------- 1 file changed, 14 insertions(+), 457 deletions(-) diff --git a/scripts/ffxiv_structimporter.py b/scripts/ffxiv_structimporter.py index c21eee5b84..1aa02be419 100644 --- a/scripts/ffxiv_structimporter.py +++ b/scripts/ffxiv_structimporter.py @@ -176,13 +176,6 @@ def load_data_yaml(self): else: # noinspection PyUnresolvedReferences class IdaApi(BaseApi, IdaInterface): - def __init__(self, full_padding, srclang_importer): - # type: (bool, bool) -> None - self.full_padding = full_padding - self.srclang_importer = srclang_importer - if self.srclang_importer: - self.srclang_types = {} - def validate_name_cfg(self): """Verifies that the user's IDA config allows template characters for type names""" temporary_name = "struc_name_test" @@ -405,10 +398,6 @@ def update_virt_func(self, virt_func, struct): ea = self.get_func_ea_by_name(func_name) if ea == idc.BADADDR: - # ignore if added during preprocess - if virt_func.inherited_from_preprocess: - return - ea = self.get_vfunc_ea_from_data(struct.type, virt_func.offset) if ea in (0, idc.BADADDR): print(f"Error: {func_name} not found and its VFT slot could not be resolved") @@ -514,448 +503,7 @@ def should_update_virt_func(self): == ida_kernwin.ASKBTN_YES ) - def preprocess_yaml(self, yaml: DefinedStructExport) -> None: - # Attempts to identify and fix missing baseclass flags and VFT inconsistencies, checks - # for issues that were previously 'hidden' by the IDA API, then builds an inheritance - # graph to propagate virtual functions and validate layouts & inheritance chains - - if not self.srclang_importer: - return - - data_classes = {} - if hasattr(self, "data_yaml") and self.data_yaml: - data_classes = self.data_yaml.get("classes", {}) or {} - - def get_data_vtables(class_name: str) -> list[dict]: - class_data = data_classes.get(class_name) or {} - vtables = class_data.get("vtbls", []) - return vtables if isinstance(vtables, list) else [] - - class BaseEdge: - def __init__(self, derived: 'Node', base: 'Node', field: DefinedStructField): - self.derived = derived - self.base = base - self.field = field - self.offset = field.offset - - class Node: - def __init__(self, struct: DefinedStruct): - self.struct: DefinedStruct = struct - self.declared_vtable_size = struct.vtable_size - self.bases: list['BaseEdge'] = [] - self.children: list['BaseEdge'] = [] - - def is_virtual(self) -> bool: - return self.struct.virtual_functions is not None - - def primary_base(self) -> 'BaseEdge | None': - primary_bases = [edge for edge in self.bases if edge.offset == 0] - if len(primary_bases) > 1: - base_names = ", ".join(edge.base.struct.type for edge in primary_bases) - raise RuntimeError( - f"Multiple baseclasses defined at offset 0 for '{self.struct.type}': {base_names}" - ) - - return primary_bases[0] if primary_bases else None - - nodes: dict[str, Node] = {} - for struct in yaml.structs: - if struct.type in nodes: - raise RuntimeError(f"Duplicate type '{struct.type}'") - - nodes[struct.type] = Node(struct) - - for node in nodes.values(): - declared_vtable_size = node.declared_vtable_size - if declared_vtable_size is not None: - if type(declared_vtable_size) is not int or declared_vtable_size < 0: - raise RuntimeError( - f"Invalid VFT size {declared_vtable_size} " - f"for '{node.struct.type}'" - ) - - if declared_vtable_size % 8 != 0: - raise RuntimeError( - f"Invalid VFT size {declared_vtable_size:#x} " - f"for '{node.struct.type}'" - ) - - vfuncs_by_offset = {} - for vfunc in node.struct.virtual_functions or []: - if vfunc is None: - continue - - if type(vfunc.offset) is not int or vfunc.offset < 0: - raise RuntimeError( - f"Invalid VFT offset {vfunc.offset} for " - f"'{node.struct.type}.{vfunc.name}'" - ) - - if vfunc.offset % 8 != 0: - raise RuntimeError( - f"Unaligned VFT offset {vfunc.offset:#x} for " - f"'{node.struct.type}.{vfunc.name}'" - ) - - previous_vfunc = vfuncs_by_offset.get(vfunc.offset) - if previous_vfunc is not None: - raise RuntimeError( - f"Duplicate VFT slot {vfunc.offset // 8} " - f"for '{node.struct.type}': " - f"'{previous_vfunc.name}' and '{vfunc.name}'" - ) - - vfuncs_by_offset[vfunc.offset] = vfunc - - # normalize - for node in nodes.values(): - if node.struct.fields and node.struct.fields[0].name == "_vtable": - node.struct.fields.pop(0) - if not node.is_virtual(): - node.struct.virtual_functions = [] - node.struct.vtable_size = 0 - - if not node.is_virtual() and get_data_vtables(node.struct.type): - node.struct.virtual_functions = [] - node.struct.vtable_size = 0 - - def add_base_edge(derived: Node, field: DefinedStructField) -> BaseEdge: - parent = nodes.get(field.type) - if parent is None: - raise RuntimeError( - f"Could not resolve baseclass '{field.type}' " - f"for '{derived.struct.type}'" - ) - - edge = BaseEdge(derived, parent, field) - derived.bases.append(edge) - parent.children.append(edge) - return edge - - # add every explicit baseclass - for node in nodes.values(): - for field in node.struct.fields: - if field.base: - add_base_edge(node, field) - - def is_or_has_primary(node: Node, class_name: str, visited=None) -> bool: - if node.struct.type == class_name: - return True - - if visited is None: - visited = set() - - if node.struct.type in visited: - return False - - visited.add(node.struct.type) - - edge = node.primary_base() - return ( - edge is not None - and is_or_has_primary(edge.base, class_name, visited) - ) - - for node in nodes.values(): - if node.primary_base() is not None: - continue - - vtables = get_data_vtables(node.struct.type) - primary_vtable = vtables[0] if vtables else None - primary_vtable_base = ( - primary_vtable.get("base") - if isinstance(primary_vtable, dict) - else None - ) - - if not primary_vtable_base: - continue - - candidates = [] - for field in node.struct.fields: - candidate = nodes.get(field.type) - if ( - field.offset == 0 - and not field.base - and candidate is not None - and is_or_has_primary(candidate, primary_vtable_base) - ): - candidates.append((field, candidate)) - - if len(candidates) != 1: - continue - - field, _ = candidates[0] - - print( - f"Warning: Fixing missing baseclass '{field.type}' " - f"for '{node.struct.type}'" - ) - - field.base = True - add_base_edge(node, field) - - # validate inheritance graph to make sure we don't have any circular dependants - visit_state: dict[str, int] = {} - visit_stack: list[str] = [] - - def validate_acyclic(node: Node) -> None: - state = visit_state.get(node.struct.type, 0) - if state == 2: - return - - if state == 1: - cycle_start = visit_stack.index(node.struct.type) - cycle = visit_stack[cycle_start:] + [node.struct.type] - cycle_path = " -> ".join(cycle) - raise RuntimeError(f"Circular inheritance detected: {cycle_path}") - - visit_state[node.struct.type] = 1 - - visit_stack.append(node.struct.type) - for edge in node.bases: - validate_acyclic(edge.base) - visit_stack.pop() - - visit_state[node.struct.type] = 2 - - for node in nodes.values(): - validate_acyclic(node) - - - inherited_vfts_visited: set[str] = set() - - def inherit_vft(node: Node) -> None: - if node.struct.type in inherited_vfts_visited: - return - - edge = node.primary_base() - if edge is not None: - inherit_vft(edge.base) - if edge.base.is_virtual() and not node.is_virtual(): - node.struct.virtual_functions = [] - node.struct.vtable_size = 0 - - inherited_vfts_visited.add(node.struct.type) - - for node in nodes.values(): - inherit_vft(node) - - for node in nodes.values(): - primary_base = node.primary_base() - primary_base_has_vft = ( - primary_base is not None and primary_base.base.is_virtual() - ) - - node.struct.srclang_needs_vfptr = ( - node.is_virtual() and not primary_base_has_vft - ) - - if ( - node.struct.srclang_needs_vfptr - and primary_base is not None - ): - # TODO(Caitlyn): maybe this should be a hard error here - print( - f"Warning: Class '{node.struct.type}' is polymorphic, " - f"but its non-polymorphic base class " - f"'{primary_base.base.struct.type}' is placed at offset zero" - ) - - # normalize VFTs into indexable lists - normalized_vfts: set[str] = set() - - def normalize_vft(node: Node) -> None: - if node.struct.type in normalized_vfts: - return - - primary_base = node.primary_base() - if primary_base is not None: - normalize_vft(primary_base.base) - - if not node.is_virtual(): - normalized_vfts.add(node.struct.type) - return - - vfuncs = node.struct.virtual_functions or [] - - last_vfunc = max( - (vf for vf in vfuncs if vf is not None), - key=lambda vf: vf.offset, - default=None, - ) - - needed_size = ( - last_vfunc.offset + 8 - if last_vfunc is not None - else 0 - ) - vft_size = needed_size - - if node.struct.vtable_size and node.struct.vtable_size > vft_size: - vft_size = node.struct.vtable_size - - needed_base_size = 0 - if ( - primary_base is not None - and primary_base.base.is_virtual() - and (primary_base.base.struct.vtable_size or 0) > vft_size - ): - needed_base_size = primary_base.base.struct.vtable_size - vft_size = needed_base_size - - defined_size = node.declared_vtable_size - if ( - defined_size is not None - and vft_size > defined_size - ): - if needed_base_size > needed_size: - reason = f"baseclass '{primary_base.base.struct.type}'" - else: - reason = ( - f"virtual function '{last_vfunc.name}' " - f"at offset {last_vfunc.offset:#x}" - ) - - print( - f"Warning: Expanding VFT for '{node.struct.type}' from " - f"{defined_size:#x} to {vft_size:#x} to fit {reason}" - ) - - new_vft = [None] * (vft_size // 8) - for vf in vfuncs: - if vf is not None: - new_vft[vf.offset // 8] = vf - - node.struct.virtual_functions = new_vft - node.struct.vtable_size = vft_size - normalized_vfts.add(node.struct.type) - - for node in nodes.values(): - normalize_vft(node) - - def primary_children(node: Node): - for edge in node.children: - if edge.derived.primary_base() is edge: - yield edge.derived - - def copy_vfunc_for_type(vf: DefinedStructVFunc, struct_type: str): - new_vf = copy.deepcopy(vf) - new_vf.inherited_from_preprocess = True - if hasattr(new_vf, 'parameters') and new_vf.parameters: - new_vf.parameters[0].type = struct_type + "*" - return new_vf - - # propagate known VFs down into subclasses - def propagate_vft_down(node: Node, visited=None) -> None: - if visited is None: - visited = set() - - if node.struct.type in visited: - return - - visited.add(node.struct.type) - - if not node.is_virtual(): - return - - parent_vfs = node.struct.virtual_functions or [] - - for child in primary_children(node): - if parent_vfs: - if not child.is_virtual(): - child.struct.virtual_functions = [None] * len(parent_vfs) - child.struct.vtable_size = node.struct.vtable_size - elif len(child.struct.virtual_functions) < len(parent_vfs): - extra_slots = len(parent_vfs) - len(child.struct.virtual_functions) - child.struct.virtual_functions += [None] * extra_slots - child.struct.vtable_size = max(child.struct.vtable_size or 0, node.struct.vtable_size) - - for i, vf in enumerate(parent_vfs): - if vf is None or child.struct.virtual_functions[i] is not None: - continue - - child.struct.virtual_functions[i] = copy_vfunc_for_type(vf, child.struct.type) - - propagate_vft_down(child, visited) - - visited_nodes: set[str] = set() - def propagate_vft_up(node: Node) -> None: - if node.struct.type in visited_nodes: - return - - visited_nodes.add(node.struct.type) - - if not node.is_virtual(): - return - - for child in primary_children(node): - # propagate from bottom up - propagate_vft_up(child) - - child_vfs = child.struct.virtual_functions - parent_vfs = node.struct.virtual_functions - if not child_vfs or not parent_vfs: - continue - - for i, vf in enumerate(child_vfs): - if i >= len(parent_vfs): - break - - if (vf is None or parent_vfs[i] is not None): - continue - - parent_vfs[i] = copy_vfunc_for_type(vf, node.struct.type) - - roots = [ - node - for node in nodes.values() - if node.is_virtual() - and ( - node.primary_base() is None - or not node.primary_base().base.is_virtual() - ) - ] - - for root in roots: - propagate_vft_down(root) - - for root in roots: - propagate_vft_up(root) - - for root in roots: - propagate_vft_down(root) - - full_padding = False - srclang_importer = False - if idaapi.IDA_SDK_VERSION >= 900: - srclang_importer = ( - ida_kernwin.ask_buttons( - "SrcLang Importer", - "Legacy Importer", - "", - ida_kernwin.ASKBTN_YES, - "HIDECANCEL\nWhich importer should be used?\n\nSrcLang Importer: Experimental - faster importer with full padding, and improved support for inheritance and virtual function calls.\n\nLegacy Importer: The original, battle-tested importer. Full Padding on the Legacy Importer on IDA 9+ can take 8 hours or longer.", - ) - == ida_kernwin.ASKBTN_YES - ) - - if srclang_importer: - full_padding = True - - if not srclang_importer: - full_padding = ( - ida_kernwin.ask_buttons( - "Full Padding", - "Array Padding", - "", - ida_kernwin.ASKBTN_YES, - "HIDECANCEL\nWhat padding style to use?\n\nFull Padding: Adds padding based on allignment of 1,2,4,8\nArray Padding: Adds padding based on the size between fields with byte arrays\n\nFull Padding will take longer to add padding between fields but is recommended for quick struct modifications.", - ) - == ida_kernwin.ASKBTN_YES - ) - - api = IdaApi(full_padding, srclang_importer) + api = IdaApi() if not api.validate_name_cfg(): ida_kernwin.warning( "Type name validation failed.\n" @@ -1406,17 +954,26 @@ def run(): print("{0} Loading data yaml".format(get_time())) api.data_yaml = api.load_data_yaml() - print("{0} Preprocessing yaml".format(get_time())) - api.preprocess_yaml(yaml) - if isinstance(api, IdaApi): print("{0} Generating src file in memory".format(get_time())) src_interface = SrcInterface() header, vtables = src_interface.build_export_string(yaml) + predef_header = """#define _HAS_ITERATOR_DEBUGGING 0 +#define _ITERATOR_DEBUG_LEVEL 0 +#include +#include +#include +#include +#include +#include +#include + +""" + print("{0} Importing src file".format(get_time())) api.select_parser() - api.parse_string(header, vtables) + api.parse_string(predef_header + header, vtables) else: print("{0} Deleting old structs".format(get_time())) for struct in yaml.structs[::-1]: From fdf07d802bc2f5403d876f4d91bbe521e35c6303 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Sun, 11 Oct 2026 13:15:28 +0200 Subject: [PATCH 37/95] correct CExporter for inline structs and enums dependency chain resolving is still borked --- CExporter/Exporter.TypeProcessing.cs | 64 ++++++++-- CExporter/Exporter.cs | 168 +++++++++++++++------------ CExporter/ExporterStatics.cs | 2 +- CExporter/Extensions.cs | 45 +++++-- CExporter/ProcessedData.cs | 18 ++- CExporter/YamlConverters.cs | 33 +++++- 6 files changed, 232 insertions(+), 98 deletions(-) diff --git a/CExporter/Exporter.TypeProcessing.cs b/CExporter/Exporter.TypeProcessing.cs index b27828e896..99a2a0be9a 100644 --- a/CExporter/Exporter.TypeProcessing.cs +++ b/CExporter/Exporter.TypeProcessing.cs @@ -1,11 +1,14 @@ using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Reflection; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using FFXIVClientStructs.Attributes; +using FFXIVClientStructs.Havok.Common.Base.Types; +using FFXIVClientStructs.STD; using InteropGenerator.Runtime.Attributes; namespace CExporter; @@ -27,7 +30,7 @@ private static void ProcessType(ProcessingType type, bool quiet) { private static (string typeOverride, Type[] typeOverrideTemplates) ProcessStdType(Type fieldType) { const string basicString = "BasicString"; - string[] pointerValuesType = ["Vector", "Deque", "Span"]; + string[] pointerValuesType = ["Vector", "Span"]; var pointerCount = 0; while (fieldType.IsPointer()) { fieldType = fieldType.GetPointerType(); @@ -85,14 +88,42 @@ private static (string typeOverride, Type[] typeOverrideTemplates) ProcessStdTyp private static ProcessedField ProcessField(FieldInfo field, int offset) { if (field.FieldType is { Namespace: ExporterStatics.StdNamespacePrefix } && !field.FieldType.IsGenericTypeParameterOrPointer()) { - var (typeOverride, typeOverrideTemplates) = ProcessStdType(field.FieldType); + if (!field.FieldType.Name.StartsWith("StdLinkedList") && !(field.DeclaringType?.FullName!.Contains("StdLinkedList") ?? false)){ + var (typeOverride, typeOverrideTemplates) = ProcessStdType(field.FieldType); + return new ProcessedField { + FieldType = field.FieldType, + FieldOffset = field.GetFieldOffset() - offset, + FieldName = field.Name, + IsBase = field.IsDirectBase(), + FieldTypeOverride = typeOverride, + FieldTypeOverrideTemplates = typeOverrideTemplates, + Bits = [], + IsUnion = false, + UnionValues = [] + }; + } + if(!(field.DeclaringType?.FullName!.Contains("StdLinkedList") ?? false)) { + _processType.Add(field.FieldType); + return new ProcessedField { + FieldType = field.FieldType, + FieldOffset = field.GetFieldOffset() - offset, + FieldName = field.Name, + IsBase = field.IsDirectBase(), + FieldTypeOverride = "StdLinkedList", + FieldTypeOverrideTemplates = field.FieldType.GetGenericArguments(), + Bits = [], + IsUnion = false, + UnionValues = [] + }; + } + _processType.Add(field.FieldType); return new ProcessedField { FieldType = field.FieldType, FieldOffset = field.GetFieldOffset() - offset, FieldName = field.Name, IsBase = field.IsDirectBase(), - FieldTypeOverride = typeOverride, - FieldTypeOverrideTemplates = typeOverrideTemplates, + FieldTypeOverride = null, + FieldTypeOverrideTemplates = null, Bits = [], IsUnion = false, UnionValues = [] @@ -352,7 +383,7 @@ private static ProcessedField ProcessParameter( private static object? PreProcessType(ProcessingType processingType, bool quiet) { var (type, overrideType) = processingType; - if (type.IsFixedBuffer() || type.IsBaseType()) return null; + if (type.IsFixedBuffer() || type.IsBaseType() || type.IsGenericTypeParameterOrPointer()) return null; while (type.IsPointer) type = type.GetElementType()!; while (type.IsGenericPointer()) type = type.GenericTypeArguments[0]; @@ -367,7 +398,7 @@ private static ProcessedField ProcessParameter( }; if (!quiet) Console.WriteLine($"Processed {processedEnum.EnumNamespace}::{processedEnum.EnumName} with {processedEnum.EnumValues.Count} fields"); - _enums.Add(processedEnum); + return processedEnum; } else if (type.IsStruct()) { while (type.IsPointer) type = type.GetElementType()!; while (type.IsGenericPointer()) type = type.GenericTypeArguments[0]; @@ -473,10 +504,12 @@ .. memberFunctionParameters.Select(ProcessParameter).ToArray()] .GroupBy(t => t.GetCustomAttribute()!.Union) .ToFrozenDictionary(t => t.Key, t => t.GroupBy(k => k.GetCustomAttribute()!.Struct).OrderBy(k => k.Key).ToFrozenDictionary(k => k.Key, k => k.ToArray())); + var nestedTypes = ((TypeInfo)type).DeclaredNestedTypes.Where(t => !ExporterStatics.IgnoredTypeNames.Contains(t.Name) && !t.Name.EndsWith("VirtualTable", StringComparison.Ordinal) && t.GetCustomAttribute() == null && t.GetCustomAttribute() == null).ToArray(); + var processedStruct = new ProcessedStruct { StructType = type, IsUnion = type.GetCustomAttribute() != null, - StructName = type.Name, + StructName = type.Name.IndexOf('`') != -1 ? type.Name[..type.Name.IndexOf('`')] : type.Name, StructNamespace = type.GetNamespace(), StructSize = size, VirtualFunctionSize = vtableSize, @@ -484,7 +517,12 @@ .. memberFunctionParameters.Select(ProcessParameter).ToArray()] VirtualFunctions = virtualFunctions, MemberFunctions = memberFunctionsArray, StructTypeOverride = overrideType, - TemplateTypes = type.IsGenericTypeDefinition ? type.GetGenericArguments().Select(t => t.Name).ToArray() : [] + TemplateTypes = type.IsGenericTypeDefinition && !type.GetGenericArguments().GenericSequenceEqual(type.DeclaringType?.GetGenericArguments() ?? []) ? type.GetGenericArguments().Select(t => t.Name).ToArray() : [], + InlineStructs = nestedTypes.Where(t => t.IsStruct()).Select(t => (ProcessedStruct?)PreProcessType(t, quiet)).Where(t => t != null).ToArray()!, + InlineEnums = nestedTypes.Where(t => t.IsEnum).Select(t => (ProcessedEnum?)PreProcessType(t, quiet)).Where(t => t != null).ToArray()!, + IsSequential = type.StructLayoutAttribute?.Value == LayoutKind.Sequential, + StaticMemberFunctions = null, + StaticMembers = null, }; foreach (var (fieldName, unionInfos) in unionInfoFields) { @@ -505,7 +543,13 @@ .. memberFunctionParameters.Select(ProcessParameter).ToArray()] VirtualFunctionSize = 0, StructTypeOverride = unionStructType, Fields = unionFields.Select(t => ProcessField(t, offsetStart)).ToArray(), - IsUnion = false + IsUnion = false, + InlineStructs = null, + InlineEnums = null, + IsSequential = false, + StaticMemberFunctions = null, + StaticMembers = null, + VirtualFunctions = null, }); processedUnionStructFields.Add(new ProcessedFieldUnion { FieldType = unionFields[0].FieldType, @@ -541,7 +585,7 @@ .. memberFunctionParameters.Select(ProcessParameter).ToArray()] } if (!quiet) Console.WriteLine($"Processed {processedStruct.StructTypeName} with {processedStruct.Fields.Length + (processedStruct.VirtualFunctions?.Length ?? 0) + processedStruct.MemberFunctions.Length} fields and methods"); - _structs.Add(processedStruct); + return processedStruct; } return null; } diff --git a/CExporter/Exporter.cs b/CExporter/Exporter.cs index 90ab52e922..c50b3bed70 100644 --- a/CExporter/Exporter.cs +++ b/CExporter/Exporter.cs @@ -82,7 +82,7 @@ public static void ProcessTypes(bool quiet) { while (_processType.Count > 0) { if (!quiet) Console.WriteLine($"{PassString(count)} with {_processType.Count} structs and enum types"); var tmp = _processType - .Where(t => t.Type is { IsUnmanagedFunctionPointer: false, IsFunctionPointer: false } and not { Namespace: ExporterStatics.StdNamespacePrefix or ExporterStatics.StdHelperNamespacePrefix }) + .Where(t => t.Type is { IsUnmanagedFunctionPointer: false, IsFunctionPointer: false } and not { Namespace: ExporterStatics.StdNamespacePrefix or ExporterStatics.StdHelperNamespacePrefix } || t.Type.Name.StartsWith("StdLinkedList")) .ToArray(); _processType.Clear(); foreach (var @struct in tmp) { @@ -103,70 +103,89 @@ public static void ProcessTypes(bool quiet) { } - public static void ProcessStaticFunctions(bool quiet) { - Type[] types = [.. ExporterStatics.GetHavokTypes(), .. ExporterStatics.GetXIVTypes()]; - types = types.Where(t => t.IsStruct()).Where(type => type.GetMethods(ExporterStatics.StaticBindingFlags).Length != 0).ToArray(); - if (!quiet) Console.WriteLine("::group::Processed Struct Static Members"); - var typeAndMembers = types.Select(t => (t, t.GetMethods(ExporterStatics.StaticBindingFlags))).ToArray(); - foreach (var (type, methods) in typeAndMembers) { - if (!quiet) Console.WriteLine($"Processing {type} with {methods.Length} methods"); - var sanitizedName = type.FixTypeName(); - var currentStructIndex = _structs.FindIndex(s => s.StructTypeName == sanitizedName); - if (currentStructIndex == -1) { - if (!quiet) Console.WriteLine($"Error in struct {type} please fix"); - continue; + public static (ProcessedStruct s, int c) ProcessStaticFunction(ProcessedStruct currentStruct) + { + var methods = currentStruct.StructType.GetMethods(ExporterStatics.StaticBindingFlags); + foreach (var methodInfo in methods) { + var staticAddress = methodInfo.GetCustomAttribute(); + if (staticAddress != null) { + currentStruct.StaticMembers ??= []; + currentStruct.StaticMembers = [.. currentStruct.StaticMembers, + new ProcessedStaticMembers { + Signature = staticAddress.Signature, + RelativeFollowOffsets = staticAddress.RelativeFollowOffsets, + IsPointer = staticAddress.IsPointer, + ReturnType = methodInfo.ReturnType.GetPointerType() + }]; } - var currentStruct = _structs[currentStructIndex]; - foreach (var methodInfo in methods) { - var staticAddress = methodInfo.GetCustomAttribute(); - if (staticAddress != null) { - var name = methodInfo.Name; - if (name.StartsWith("Get")) { - name = name[3..]; - } - currentStruct.StaticMembers ??= []; - currentStruct.StaticMembers = [.. currentStruct.StaticMembers, - new ProcessedStaticMembers { - Name = name, - Signature = staticAddress.Signature, - RelativeFollowOffsets = staticAddress.RelativeFollowOffsets, - IsPointer = staticAddress.IsPointer, - ReturnType = methodInfo.ReturnType.GetPointerType() - }]; - } - var memberFunction = methodInfo.GetCustomAttribute(); - if (memberFunction != null || staticAddress != null) - _processType.Add(methodInfo.ReturnType); - if (memberFunction == null) continue; - var returnType = methodInfo.ReturnType; - var pointerCount = 0; - while (returnType.IsPointer()) { - pointerCount++; - returnType = returnType.GetPointerType(); - } - string? overrideType = null; - if (returnType is { Namespace: ExporterStatics.StdNamespacePrefix }) { - (overrideType, var requiredTypes) = ProcessStdType(returnType); - foreach (var requiredType in requiredTypes) { - _processType.Add(requiredType); - } - overrideType += new string('*', pointerCount); + var memberFunction = methodInfo.GetCustomAttribute(); + if (memberFunction != null || staticAddress != null) + _processType.Add(methodInfo.ReturnType); + if (memberFunction == null) continue; + var returnType = methodInfo.ReturnType; + var pointerCount = 0; + while (returnType.IsPointer()) { + pointerCount++; + returnType = returnType.GetPointerType(); + } + string? overrideType = null; + if (returnType is { Namespace: ExporterStatics.StdNamespacePrefix }) { + (overrideType, var requiredTypes) = ProcessStdType(returnType); + foreach (var requiredType in requiredTypes) { + _processType.Add(requiredType); } - var memberFunctionReturnType = new ProcessedMemberFunctionReturn(methodInfo.ReturnType, overrideType); - if (methodInfo.GetCustomAttribute() is not { } excelAttribute) - _processType.Add(memberFunctionReturnType.Type); - else - memberFunctionReturnType.OverrideType = $"Component::Exd::Sheets::{excelAttribute.SheetName}*"; - currentStruct.StaticMemberFunctions ??= []; - currentStruct.StaticMemberFunctions = [.. currentStruct.StaticMemberFunctions, - new ProcessedMemberFunction { - MemberFunctionSignature = memberFunction.Signature, - MemberFunctionName = methodInfo.Name, - MemberFunctionReturnType = memberFunctionReturnType, - MemberFunctionParameters = methodInfo.GetParameters().Select(ProcessParameter).ToArray() - }]; + overrideType += new string('*', pointerCount); + } + var memberFunctionReturnType = new ProcessedMemberFunctionReturn(methodInfo.ReturnType, overrideType); + if (methodInfo.GetCustomAttribute() is not { } excelAttribute) + _processType.Add(memberFunctionReturnType.Type); + else + memberFunctionReturnType.OverrideType = $"Component::Exd::Sheets::{excelAttribute.SheetName}*"; + currentStruct.StaticMemberFunctions ??= []; + currentStruct.StaticMemberFunctions = [.. currentStruct.StaticMemberFunctions, + new ProcessedMemberFunction { + MemberFunctionSignature = memberFunction.Signature, + MemberFunctionName = methodInfo.Name, + MemberFunctionReturnType = memberFunctionReturnType, + MemberFunctionParameters = methodInfo.GetParameters().Select(ProcessParameter).ToArray() + }]; + } + return (currentStruct, methods.Length); + } + + public static (ProcessedStruct[] inlineStruct, (uint s, uint c)) ProcessStaticFunctionsInline(ProcessedStruct[] inlineStruct) + { + var membersCount = 0u; + var structCount = 0u; + for (var currentStructIndex = 0; currentStructIndex < inlineStruct.Length; currentStructIndex++) { + var (s, c) = ProcessStaticFunction(inlineStruct[currentStructIndex]); + inlineStruct[currentStructIndex] = s; + structCount++; + membersCount += (uint)c; + if(inlineStruct[currentStructIndex].InlineStructs != null) { + var (inline, (inlineCount, memberCount)) = ProcessStaticFunctionsInline(_structs[currentStructIndex].InlineStructs!); + _structs[currentStructIndex].InlineStructs = inline; + structCount += inlineCount; + membersCount += memberCount; + } + } + return (inlineStruct, (structCount, membersCount)); + } + + public static void ProcessStaticFunctions(bool quiet) { + if (!quiet) Console.WriteLine("::group::Processed Struct Static Members"); + uint structCount = 0, members = 0; + for (var currentStructIndex = 0; currentStructIndex < _structs.Count; currentStructIndex++) { + var (s, c) = ProcessStaticFunction(_structs[currentStructIndex]); + _structs[currentStructIndex] = s; + structCount++; + members += (uint)c; + if(_structs[currentStructIndex].InlineStructs != null) { + var (inline, (inlineCount, memberCount)) = ProcessStaticFunctionsInline(_structs[currentStructIndex].InlineStructs!); + _structs[currentStructIndex].InlineStructs = inline; + structCount += inlineCount; + members += memberCount; } - _structs[currentStructIndex] = currentStruct; } if (!quiet) { @@ -196,7 +215,7 @@ public static void ProcessStaticFunctions(bool quiet) { Console.WriteLine("::endgroup::"); Console.WriteLine(); Console.WriteLine($"Processed {_enums.Count - enumsCount} enums and {_structs.Count - structsCount} structs"); - Console.WriteLine($"Processed {typeAndMembers.Length} structs with {typeAndMembers.Sum(t => t.Item2.Length)} members"); + Console.WriteLine($"Processed {structCount} structs with {members} members"); Console.WriteLine(); Console.WriteLine($"Processed total {_enums.Count} enums and {_structs.Count} structs"); } @@ -274,7 +293,7 @@ public static void ProcessDefinedVTables(DataDefinition def) { public static void Write(DirectoryInfo dir) { // make sure we have all the dependencies for each struct before we write them - var structs = _structs.ToFrozenDictionary(x => x.StructTypeName, x => x); + var structs = _structs.SelectMany(t => t.ChildrenAndSelf).DistinctBy(x => x.StructTypeName).ToFrozenDictionary(x => x.StructTypeName, x => x); var structToFullDeps = new Dictionary>(); var structDeps = structs.SelectMany(t => ResolveFullDependencies(t.Value)).Distinct().ToArray(); _structs.Sort((a, b) => { @@ -282,15 +301,6 @@ public static void Write(DirectoryInfo dir) { return cmp != 0 ? cmp : string.Compare(a.StructTypeName, b.StructTypeName, StringComparison.Ordinal); }); -#if DEBUG - for (var i = 0; i < _structs.Count; i++) { - foreach (var dep in _structs[i].DependencyNames) { - if (_structs.FindIndex(0, i, x => x.StructTypeName == dep) == -1) - throw new InvalidOperationException(); - } - } -#endif - HashSet ResolveFullDependencies(ProcessedStruct s) { if (structToFullDeps.TryGetValue(s.StructTypeName, out var deps)) return deps; @@ -300,6 +310,18 @@ HashSet ResolveFullDependencies(ProcessedStruct s) { return structToFullDeps[s.StructTypeName] = fullDeps; } + void OrderInlineStructs(ProcessedStruct parent) { + if (parent.InlineStructs == null) return; + parent.InlineStructs = [.. parent.InlineStructs + .OrderBy(t => ResolveFullDependencies(t).Count) + .ThenBy(t => t.StructTypeName, StringComparer.Ordinal)]; + foreach (var inlineStruct in parent.InlineStructs) + OrderInlineStructs(inlineStruct); + } + + foreach (var s in _structs) + OrderInlineStructs(s); + var serializer = new SerializerBuilder() .WithNamingConvention(UnderscoredNamingConvention.Instance) .WithTypeConverter(ProcessedStructConverter.Instance) diff --git a/CExporter/ExporterStatics.cs b/CExporter/ExporterStatics.cs index d4375661bf..03d2bd2fc4 100644 --- a/CExporter/ExporterStatics.cs +++ b/CExporter/ExporterStatics.cs @@ -35,7 +35,7 @@ private static Type[] GetTypes(string namespacePrefix) { Type[] definedTypes; try { - definedTypes = assembly.DefinedTypes.Select(ti => ti.AsType()).ToArray(); + definedTypes = assembly.DefinedTypes.Where(ti => ti.IsValueType && !ti.IsEnum && !ti.IsNested).Select(ti => ti.AsType()).ToArray(); } catch (ReflectionTypeLoadException ex) { definedTypes = ex.Types.Where(t => t != null).ToArray()!; } diff --git a/CExporter/Extensions.cs b/CExporter/Extensions.cs index 29b2758f21..90dd3fbf5c 100644 --- a/CExporter/Extensions.cs +++ b/CExporter/Extensions.cs @@ -127,7 +127,13 @@ private int GetSizeOf() { public string GetNamespace() { var ns = type.Namespace!; - var offset = ns.IndexOf('.', ns.IndexOf('.') + 1) + 1; + if (ns.StartsWith("System")) { + return ns; + } + var offset = ns.IndexOf('.') + 1; + ns = offset == 0 ? "" : ns[offset..]; + if (ns.StartsWith("SQEX")) return ns; + offset = ns.IndexOf('.') + 1; return offset == 0 ? "" : ns[offset..]; } @@ -147,6 +153,16 @@ private bool IsHavokStdXivOrInterop() { || ns.StartsWith("Interop") || ns.StartsWith("Havok"); } + + private bool IsSQEX() { + var ns = type.Namespace.AsSpan(); + const string cs = "FFXIVClientStructs."; + if (!ns.StartsWith(cs)) + return false; + ns = ns[cs.Length..]; + + return ns.StartsWith("SQEX"); + } public Type GetPointerType() { while (type.IsPointer()) { @@ -172,19 +188,28 @@ public string SanitizeName(bool ignoreGenericTypes = false) { name = name[(name.IndexOf('.') + 1)..]; name = name[(name.IndexOf('.') + 1)..]; } + if (type.IsSQEX()) { + name = name[(name.IndexOf('.') + 1)..]; + } if (!type.IsGenericType) return name.ToString(); if (type.IsGenericPointer()) return type.GenericTypeArguments[0].FixTypeName() + "*"; - var afterBacktick = name[name.IndexOf('`')..]; - name = name[..^afterBacktick.Length]; using var builderPooled = StringBuilderPool.Get(500); var builder = builderPooled.Builder; - builder.Append(name); + if (name.IndexOf('`') != -1){ + + var afterBacktick = name[name.IndexOf('`')..]; + name = name[..^afterBacktick.Length]; + + builder.Append(name); - var me = StdNodeRegex().EnumerateMatches(afterBacktick); - if (me.MoveNext()) - builder.Append('.').Append(afterBacktick.Slice(me.Current.Index + 3, me.Current.Length - 3)); + var me = StdNodeRegex().EnumerateMatches(afterBacktick); + if (me.MoveNext()) + builder.Append('.').Append(afterBacktick.Slice(me.Current.Index + 3, me.Current.Length - 3)); + } + else + builder.Append(name); if (!ignoreGenericTypes) { @@ -282,4 +307,10 @@ public bool IsInStructList(List structs) { public string GetCorrectedName() => name is "Dtor" or "Ctor" ? name.ToLower() : name; } + + extension(Type[] span){ + public bool GenericSequenceEqual(Type[] other){ + return span.Select(t => t.Name).SequenceEqual(other.Select(t => t.Name)); + } + } } diff --git a/CExporter/ProcessedData.cs b/CExporter/ProcessedData.cs index b11dc7bd3d..c0d04a6054 100644 --- a/CExporter/ProcessedData.cs +++ b/CExporter/ProcessedData.cs @@ -56,7 +56,6 @@ public class ProcessedStaticMembers { public required Type ReturnType; } -// TODO: Export if struct is generic with types public class ProcessedStruct { public required string? StructTypeOverride; public required bool IsUnion; @@ -66,11 +65,14 @@ public class ProcessedStruct { public required int StructSize; public required ProcessedField[] Fields; public required int VirtualFunctionSize; - public ProcessedVirtualFunction[]? VirtualFunctions; // null if there are no virtual functions, empty if there's a vtable with unknown contents + public required ProcessedVirtualFunction[]? VirtualFunctions; public required ProcessedMemberFunction[] MemberFunctions; - public ProcessedMemberFunction[]? StaticMemberFunctions; - public ProcessedStaticMembers[]? StaticMembers; + public required ProcessedMemberFunction[]? StaticMemberFunctions; + public required ProcessedStaticMembers[]? StaticMembers; public required string[] TemplateTypes; + public required ProcessedStruct[]? InlineStructs; + public required ProcessedEnum[]? InlineEnums; + public required bool IsSequential; [YamlIgnore] public string StructTypeName => StructTypeOverride ?? StructType.FixTypeName(); [YamlIgnore] @@ -88,13 +90,19 @@ public string[] DependencyNames { .SelectMany(t => { if (t.FieldTypeOverride?.StartsWith(ExporterStatics.StdCppNamespace) ?? false) return t.FieldTypeOverrideTemplates!.Where(k => !k.ShouldNotExportType()).Select(k => k.FixTypeName()).Where(k => !ExporterStatics.BaseTypeNames.Contains(k)); return [t.FieldTypeOverride ?? t.FieldType.FixTypeName()]; - }).Distinct().ToArray(); + }) + .Concat(InlineStructs?.SelectMany(t => t.DependencyNames) ?? Enumerable.Empty()) + .Distinct() + .ToArray(); } return _dependencyNames; } } + + public ProcessedStruct[] ChildrenAndSelf => InlineStructs != null && InlineStructs.Any() ? [this, .. InlineStructs.SelectMany(t => t.ChildrenAndSelf)] : [this]; public ProcessedStruct FixOrder() { Fields = [.. Fields.OrderBy(t => t.FieldOffset)]; + if (InlineStructs != null) InlineStructs = [.. InlineStructs.Select(t => t.FixOrder())]; return this; } } diff --git a/CExporter/YamlConverters.cs b/CExporter/YamlConverters.cs index 9aa8ac16cd..272ac13e22 100644 --- a/CExporter/YamlConverters.cs +++ b/CExporter/YamlConverters.cs @@ -122,6 +122,15 @@ public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializ } emitter.Emit(new SequenceEnd()); } + else if (f.FieldTypeOverrideTemplates != null && f.FieldTypeOverrideTemplates.Any()) + { + emitter.Emit(new Scalar("template_args")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Flow)); + foreach (var templateArg in f.FieldTypeOverrideTemplates){ + emitter.Emit(new Scalar(templateArg.FixTypeName())); + } + emitter.Emit(new SequenceEnd()); + } emitter.Emit(new MappingEnd()); } @@ -142,7 +151,7 @@ public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializ emitter.Emit(new Scalar(s.StructNamespace)); emitter.Emit(new Scalar("union")); emitter.Emit(new Scalar(s.IsUnion.ToString())); - if (s.StructType.StructLayoutAttribute?.Value == LayoutKind.Explicit && s.StructSize != 0) { + if (s.StructSize > 1 || s.Fields.Any()) { emitter.Emit(new Scalar("size")); emitter.Emit(new Scalar(s.StructSize.ToString())); } @@ -159,7 +168,7 @@ public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializ if (s.VirtualFunctions != null) { emitter.Emit(new Scalar("virtual_functions")); emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); - foreach (var virtualFunction in s.VirtualFunctions) { + foreach (var virtualFunction in s.VirtualFunctions.OrderBy(t => t.Offset)) { ProcessedVirtualFunctionConverter.Instance.WriteYaml(emitter, virtualFunction, virtualFunction.GetType(), serializer); } emitter.Emit(new SequenceEnd()); @@ -196,6 +205,26 @@ public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializ } emitter.Emit(new SequenceEnd()); } + if (s.InlineStructs != null) { + emitter.Emit(new Scalar("inline_structs")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); + foreach (var inlineStruct in s.InlineStructs) { + Instance.WriteYaml(emitter, inlineStruct, inlineStruct.GetType(), serializer); + } + emitter.Emit(new SequenceEnd()); + } + if (s.InlineEnums != null) { + emitter.Emit(new Scalar("inline_enums")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); + foreach (var inlineEnum in s.InlineEnums) { + ProcessedEnumConverter.Instance.WriteYaml(emitter, inlineEnum, inlineEnum.GetType(), serializer); + } + emitter.Emit(new SequenceEnd()); + } + if (s.IsSequential) { + emitter.Emit(new Scalar("is_sequential")); + emitter.Emit(new Scalar(s.IsSequential.ToString())); + } emitter.Emit(new MappingEnd()); } From 4ae8c8f9f0fff7f86b53d5102d03eaebb13fdccb Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Sun, 11 Oct 2026 13:15:54 +0200 Subject: [PATCH 38/95] do not export the enumerator --- FFXIVClientStructs/STD/StdLinkedList.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/FFXIVClientStructs/STD/StdLinkedList.cs b/FFXIVClientStructs/STD/StdLinkedList.cs index 90901a7c76..40d72892cb 100644 --- a/FFXIVClientStructs/STD/StdLinkedList.cs +++ b/FFXIVClientStructs/STD/StdLinkedList.cs @@ -15,6 +15,7 @@ public struct Node { public Enumerator GetEnumerator() => new(this); + [CExporterIgnore] public struct Enumerator(StdLinkedList list) { private Node* _current = list.Start; From 0ce498f2d76b809eaf043c30f20178af092481d0 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Sun, 11 Oct 2026 13:17:27 +0200 Subject: [PATCH 39/95] get inline struct and enums working with the importer --- scripts/exdschema_wrapper.py | 15 +- scripts/ffxiv_structimporter.py | 2 +- scripts/ida/form_builder.py | 107 ++++++++++++ scripts/ida/ida_wrapper.py | 45 ++++- scripts/src_wrapper.py | 300 +++++++++++++++++++++++++++----- scripts/structs_schema.py | 281 ++++++++++++++++-------------- scripts/test_execute.py | 33 ++-- 7 files changed, 588 insertions(+), 195 deletions(-) create mode 100644 scripts/ida/form_builder.py diff --git a/scripts/exdschema_wrapper.py b/scripts/exdschema_wrapper.py index c1078e6d48..b8970bf641 100644 --- a/scripts/exdschema_wrapper.py +++ b/scripts/exdschema_wrapper.py @@ -47,6 +47,7 @@ def size(self) -> int: def get_exdschema_versions(): + print("Fetching https://api.github.com/repos/xivdev/exdschema/branches") with urlopen("https://api.github.com/repos/xivdev/exdschema/branches") as response: vers: list[str] = [] for branch in jsonload(response): @@ -80,6 +81,7 @@ def process_exdschema_definition(fields: ExdSchemaYaml): def get_exdschema_data(version: str): + print(f"Downloading https://github.com/xivdev/EXDSchema/archive/refs/heads/{version}.zip") with urlopen( f"https://github.com/xivdev/EXDSchema/archive/refs/heads/{version}.zip" ) as response: @@ -160,7 +162,7 @@ def map_header_and_schema( fields.append( DefinedStructFixedField( field.name, - exported_structs[-1].name, + exported_structs[-1].type, definition_offset, False, [], @@ -179,8 +181,8 @@ def map_header_and_schema( else: enum_index = len(enums) enum_field_map[definition_offset] = enum_index - enum_type = f"{schema_name}PackedBool{definition_offset:X}" - enum_name = f"{exd_namespace}::{enum_type}" + enum_name = f"{schema_name}PackedBool{definition_offset:X}" + enum_type = f"{exd_namespace}::{enum_name}" enums.append( DefinedStructEnum( enum_name, @@ -197,7 +199,7 @@ def map_header_and_schema( fields.append( DefinedStructField( f"PackedBool{definition_offset:X}", - enum_name, + enum_type, definition_offset, False, [] @@ -238,8 +240,8 @@ def map_header_and_schema( if len(fields) != 0: structs.append( DefinedStruct( - f"{exd_namespace}::{schema_name}", schema_name, + f"{exd_namespace}::{schema_name}", exd_namespace, fields, size, @@ -250,6 +252,9 @@ def map_header_and_schema( None, None, [], + [], + [], + False ) ) diff --git a/scripts/ffxiv_structimporter.py b/scripts/ffxiv_structimporter.py index 1aa02be419..89edeca06f 100644 --- a/scripts/ffxiv_structimporter.py +++ b/scripts/ffxiv_structimporter.py @@ -1019,7 +1019,7 @@ def run(): if virt_func is None: continue - if virt_func.return_type != None and virt_func.parameters != None: + if virt_func.return_type != None or virt_func.parameters != None: api.update_virt_func(virt_func, struct) if update_member_func: diff --git a/scripts/ida/form_builder.py b/scripts/ida/form_builder.py new file mode 100644 index 0000000000..4b1a091758 --- /dev/null +++ b/scripts/ida/form_builder.py @@ -0,0 +1,107 @@ +import ida_kernwin +from webbrowser import open + +F = ida_kernwin.Form + +def build_spec(fields, change_cb): + pad = max((len(f.get("label", "")) for f in fields), default=0) + lines = ["STARTITEM 0", "BUTTON YES* Run", "BUTTON CANCEL Cancel"] + need_first_space = True + ctrls = {"FormChangeCb": F.FormChangeCb(change_cb)} + + prepend_line = "" + for fld in fields: + t, key = fld["type"], fld.get("key") + label = fld.get("label", "").ljust(pad) + tip = f"#{fld['tip']}#" if fld.get("tip") else "" + + if t == "space": + lines.append("") + if need_first_space: + need_first_space = False + lines.append("{FormChangeCb}") + elif t == "label": + label_text = fld["text"] + if "prepend" in fld and fld["prepend"]: + prepend_line = f"{prepend_line}{label_text}" + else: + lines.append(label_text) + elif t == "folder": + lines.append(f"{prepend_line}<{tip}{label}:{{{key}}}>") + ctrls[key] = F.DirInput(swidth=fld.get("width", 60), + value=fld.get("value", "")) + elif t == "text": + lines.append(f"{prepend_line}<{tip}{label}:{{{key}}}>") + ctrls[key] = F.StringInput(swidth=fld.get("width", 30), + value=fld.get("value", "")) + elif t == "number": + lines.append(f"{prepend_line}<{tip}{label}:{{{key}}}>") + ctrls[key] = F.NumericInput(tp=F.FT_DEC, value=fld.get("value", 0)) + elif t == "dropdown": + lines.append(f"{prepend_line}<{tip}{label}:{{{key}}}>") + ctrls[key] = F.DropdownListControl(items=fld["items"], readonly=True, + selval=fld.get("selval", 0)) + elif t == "button": + on_click = fld["on_click"] + lines.append(f"{prepend_line}<{tip}{label}:{{{key}}}>") + ctrls[key] = F.ButtonInput(lambda code, fn=on_click: fn()) + + elif t in ("checks", "radios"): + names = [] + items = fld["items"] + for i, (ikey, ilabel) in enumerate(items): + name = f"{key}__{ikey}" + names.append(name) + last = (i == len(items) - 1) + if last: + end = f">{{{key}}}>" if t == "checks" else f">>{{{key}}}>" + else: + end = ">" + lines.append(f"{prepend_line}<{ilabel}:{{{name}}}{end}") + ctrls[name] = F.Control() if False else None + for n in names: + ctrls.pop(n, None) + if t == "checks": + ctrls[key] = F.ChkGroupControl(tuple(names)) + else: + ctrls[key] = F.RadGroupControl(tuple(names)) + else: + raise ValueError(f"Unknown field type: {t}") + + if t != "label": + prepend_line = "" + + if need_first_space: + raise ValueError(f"Did not receive a line of type space") + + return "\n".join(lines) + "\n", ctrls + +class DynamicForm(F): + def __init__(self, fields, on_change=None): + self.fields = fields + self._on_change_cb = on_change + template, ctrls = build_spec(fields, self._on_change) + F.__init__(self, template, ctrls) + + def _on_change(self, fid): + if self._on_change_cb: + self._on_change_cb(self, fid) + return 1 + + def results(self): + """Return all values as a plain dict keyed by each field's 'key'.""" + out = {} + for fld in self.fields: + t, key = fld["type"], fld.get("key") + if t in ("folder", "text", "number"): + out[key] = getattr(self, key).value + elif t == "dropdown": + v = getattr(self, key).value + out[key] = fld["items"][v] if isinstance(v, int) else v + elif t == "checks": + mask = getattr(self, key).value + out[key] = {ik: bool(mask & (1 << i)) + for i, (ik, _) in enumerate(fld["items"])} + elif t == "radios": + out[key] = fld["items"][getattr(self, key).value][0] + return out \ No newline at end of file diff --git a/scripts/ida/ida_wrapper.py b/scripts/ida/ida_wrapper.py index b2b785b3ec..d8115456d0 100644 --- a/scripts/ida/ida_wrapper.py +++ b/scripts/ida/ida_wrapper.py @@ -9,8 +9,37 @@ import ida_funcs import ida_srclang from structs_schema import * +from form_builder import DynamicForm from abc import abstractmethod +from platform import system +from pathlib import Path +from webbrowser import open + +def pick_folder(form_content: list[dict[str, any]]) -> str | None: + f = DynamicForm(form_content) + f.Compile() + try: + if f.Execute() == 1: # 1 = OK, 0 = Cancel + return f.folder.value or None + return None + finally: + f.Free() + +fields = [ + { "type": "label", "text": "Select msvc"}, + { "type": "space" }, + { "type": "label", "text": "It should be located where you used:", "prepend": True}, + { "type": "button", "key": "msvcdownload", "label": "mstorsjo/msvc-wine", "on_click": lambda: open("https://github.com/mstorsjo/msvc-wine")}, + { "type": "space" }, + { "type": "folder", "key": "folder", "label": "Folder", "value": "", "tip": "Pick the folder where ./vsdownload.sh was run" } +] + +def first_subfolder(base: Path) -> Path | None: + for p in sorted(base.iterdir()): + if p.is_dir(): + return p + return None # For more information about TERR_ constants, see: # https://docs.hex-rays.com/developer-guide/idapython/idapython-porting-guide-ida-9#type-information-error-codes @@ -143,9 +172,23 @@ def get_tinfo_from_type(self, raw_type: str, array_size=0): def select_parser(self): """Sets the parser used for building struct data.""" + includes = "" + if system() != "Windows": + path = pick_folder(fields) + if path == None: + raise "Has to have msvc includes for the parsers to work with the project" + path = Path(path) + vc_path = path / "vc" / "tools" / "msvc" + vc_path = first_subfolder(vc_path) / "include" + window_kit = first_subfolder(path / "Windows Kits" / "10" / "Include") + winsdk_ucrt_path = window_kit / "ucrt" + winsdk_um_path = window_kit / "um" + winsdk_shared_path = window_kit / "shared" + includes = f" -I{vc_path.as_posix()} -I\"{winsdk_ucrt_path.as_posix()}\" -I\"{winsdk_um_path.as_posix()}\" -I\"{winsdk_shared_path.as_posix()}\"" + ida_srclang.set_parser_argv( "clang", - "-x c++ -target x86_64-pc-windows-msvc -fms-compatibility -fms-extensions -fdelayed-template-parsing -std=c++17", + f"-x c++ -target x86_64-pc-windows-msvc -fms-compatibility -fms-extensions -fdelayed-template-parsing -U__int64 -std=c++17{includes}", ) def parse_string(self, decl: str, vtables: list[str]): diff --git a/scripts/src_wrapper.py b/scripts/src_wrapper.py index 35552c936b..a9d2bc0995 100644 --- a/scripts/src_wrapper.py +++ b/scripts/src_wrapper.py @@ -42,6 +42,10 @@ def get_size_from_string(self, type: str) -> int: return 8 return 0 + def is_base_type(self, type: str) -> bool: + type = type.rstrip('*') + return type in ("byte", "wchar_t", "char", "float", "double", "bool", "int8_t", "uint8_t", "int16_t", "uint16_t", "int32_t", "uint32_t", "int64_t", "uint64_t", "void") + def _split_template_type(self, type_name: str) -> tuple[str, list[str]]: """Split a type's base name and top-level template arguments.""" type_name = type_name.strip() @@ -112,7 +116,8 @@ def _get_field_type( and not self._split_template_type(field_type)[1] and self._normalize_type(field_type) in template_struct_lookup ): - field_type += f"<{', '.join(field.template_args)}>" + template_args = [f"::{template_arg}" if not self.is_base_type(template_arg) else template_arg for template_arg in field.template_args] + field_type += f"<{', '.join(template_args)}>" return field_type + pointer_suffix def _substitute_template_types( @@ -141,6 +146,7 @@ def get_size_from_type( enum_lookup: dict[str, int], template_struct_lookup: dict[str, DefinedStruct] | None = None, template_parameters: tuple[str, ...] = (), + struct_definition_lookup: dict[str, DefinedStruct] | None = None, ) -> int: """Resolve a field's size using schema types and the target x64 ABI.""" type_name = type_name.strip() @@ -162,6 +168,17 @@ def get_size_from_type( base_type, template_args = self._split_template_type(type_name) if not template_args: + struct_definition_lookup = struct_definition_lookup or {} + struct_definition = struct_definition_lookup.get(normalized_type) + if struct_definition is not None and not struct_definition.template_types: + return self._get_template_struct_size( + struct_definition, + [], + struct_lookup, + enum_lookup, + template_struct_lookup or {}, + struct_definition_lookup, + ) return 0 stl_type_sizes = { @@ -173,7 +190,6 @@ def get_size_from_type( "std::basic_string": 32, "std::list": 16, "std::deque": 32, - "std::linkedlist": 24, } if base_type in stl_type_sizes: return stl_type_sizes[base_type] @@ -182,16 +198,32 @@ def get_size_from_type( if len(template_args) != 2: raise ValueError(f"std::pair requires two type arguments: {type_name}") first_size = self.get_size_from_type( - template_args[0], struct_lookup, enum_lookup, template_struct_lookup + template_args[0], + struct_lookup, + enum_lookup, + template_struct_lookup, + struct_definition_lookup=struct_definition_lookup, ) second_size = self.get_size_from_type( - template_args[1], struct_lookup, enum_lookup, template_struct_lookup + template_args[1], + struct_lookup, + enum_lookup, + template_struct_lookup, + struct_definition_lookup=struct_definition_lookup, ) first_alignment = self._get_type_alignment( - template_args[0], struct_lookup, enum_lookup, template_struct_lookup + template_args[0], + struct_lookup, + enum_lookup, + template_struct_lookup, + struct_definition_lookup, ) second_alignment = self._get_type_alignment( - template_args[1], struct_lookup, enum_lookup, template_struct_lookup + template_args[1], + struct_lookup, + enum_lookup, + template_struct_lookup, + struct_definition_lookup, ) alignment = max(first_alignment, second_alignment) second_offset = self._align_size(first_size, second_alignment) @@ -206,6 +238,7 @@ def get_size_from_type( struct_lookup, enum_lookup, template_struct_lookup, + struct_definition_lookup, ) raise ValueError(f"Cannot determine size for template type: {type_name}") @@ -216,6 +249,7 @@ def _get_type_alignment( struct_lookup: dict[str, int], enum_lookup: dict[str, int], template_struct_lookup: dict[str, DefinedStruct] | None = None, + struct_definition_lookup: dict[str, DefinedStruct] | None = None, ) -> int: type_name = type_name.strip() if type_name.endswith("*"): @@ -238,7 +272,11 @@ def _get_type_alignment( raise ValueError(f"std::pair requires two type arguments: {type_name}") return max( self._get_type_alignment( - arg, struct_lookup, enum_lookup, template_struct_lookup + arg, + struct_lookup, + enum_lookup, + template_struct_lookup, + struct_definition_lookup, ) for arg in template_args ) @@ -247,6 +285,7 @@ def _get_type_alignment( struct_lookup, enum_lookup, template_struct_lookup, + struct_definition_lookup=struct_definition_lookup, ) return min(size & -size, 8) if size else 1 @@ -261,6 +300,7 @@ def _get_template_struct_size( struct_lookup: dict[str, int], enum_lookup: dict[str, int], template_struct_lookup: dict[str, DefinedStruct], + struct_definition_lookup: dict[str, DefinedStruct] | None = None, ) -> int: if len(template_args) != len(struct.template_types): raise ValueError( @@ -271,10 +311,16 @@ def _get_template_struct_size( return struct.size substitutions = dict(zip(struct.template_types, template_args)) - extent = 0 + has_base_at_zero = any( + field.base and field.offset == 0 for field in struct.fields + ) + has_implicit_vtable = bool(struct.virtual_functions) and not has_base_at_zero + extent = 8 if has_implicit_vtable else 1 + alignment = 8 if has_implicit_vtable else 1 for field in struct.fields: if field.type == "__fastcall": field_size = 8 + field_alignment = 8 else: field_type = self._get_field_type(field, template_struct_lookup) field_type = self._substitute_template_types(field_type, substitutions) @@ -283,20 +329,29 @@ def _get_template_struct_size( struct_lookup, enum_lookup, template_struct_lookup, + struct_definition_lookup=struct_definition_lookup, ) if field_size == 0: raise ValueError( f"Cannot determine size of {field_type} in template type {struct.type}" ) + field_alignment = self._get_type_alignment( + field_type, + struct_lookup, + enum_lookup, + template_struct_lookup, + struct_definition_lookup, + ) if isinstance(field, DefinedStructFixedField): field_size *= field.size + alignment = max(alignment, field_alignment) field_end = field.offset + field_size if struct.union: extent = max(extent, field_size) else: extent = max(extent, field_end) - return extent + return self._align_size(extent, alignment) def get_string_define_and_size( self, @@ -305,7 +360,8 @@ def get_string_define_and_size( struct_lookup: dict[str, int], enum_lookup: dict[str, int], template_struct_lookup: dict[str, DefinedStruct] | None = None, - template_parameters: tuple[str, ...] = (), + template_parameters: list[str] = [], + struct_definition_lookup: dict[str, DefinedStruct] | None = None, ) -> tuple[str, int]: template_struct_lookup = template_struct_lookup or {} field_type = self._get_field_type(field, template_struct_lookup) @@ -316,10 +372,13 @@ def get_string_define_and_size( enum_lookup, template_struct_lookup, template_parameters, + struct_definition_lookup, ) display_type = field_type if display_type.startswith(namespace): display_type = display_type.removeprefix(f"{namespace}::") + elif not self.is_base_type(display_type) and not display_type.rstrip('*') in template_parameters: + display_type = f"::{display_type}" return (f"{display_type} {field_name};", field_size) def get_short_name(self, full_type: str) -> str: @@ -351,11 +410,20 @@ def build_vfunc_signature(self, vfunc: DefinedStructVFunc) -> str: return f"void* {vfunc.name};" return_type = vfunc.return_type if vfunc.return_type else "void" + + if not self.is_base_type(return_type): + return_type = f"::{return_type}" # Build parameter list param_list = "" if vfunc.parameters: - params = [f"{param.type} {param.name}" for param in vfunc.parameters] + params = [] + for param in vfunc.parameters: + if not self.is_base_type(param.type): + params.append(f"::{param.type} {param.name}") + else: + params.append(f"{param.type} {param.name}") + param_list = ", ".join(params) return f"{return_type} (__fastcall *{vfunc.name})({param_list});" @@ -367,22 +435,48 @@ def build_export_string( ): output = IndentWriter() - struct_lookup: dict[str, int] = {} + all_structs: list[DefinedStruct] = [] + inline_structs: list[DefinedStruct] = [] + inline_enums: list[DefinedStructEnum] = [] + + def collect_inline_types(struct: DefinedStruct): + for inline_enum in struct.inline_enums: + inline_enums.append(inline_enum) + for inline_struct in struct.inline_structs: + inline_structs.append(inline_struct) + all_structs.append(inline_struct) + collect_inline_types(inline_struct) + for struct in export.structs: + all_structs.append(struct) + collect_inline_types(struct) + + inline_struct_types = { + self._normalize_type(struct.type) for struct in inline_structs + } + inline_enum_types = { + self._normalize_type(enum.type) for enum in inline_enums + } + + struct_lookup: dict[str, int] = {} + for struct in all_structs: if struct.size is None: continue normalized_type = self._normalize_type(struct.type) struct_lookup[normalized_type] = struct.size struct_lookup.setdefault(self.get_short_name(normalized_type), struct.size) - enum_lookup = { - self._normalize_type(enum.type): self.get_size_from_string(enum.underlying) - for enum in export.enums - } - template_struct_lookup = { - self._normalize_type(struct.type): struct - for struct in export.structs - if struct.template_types - } + enum_lookup: dict[str, int] = {} + for enum in [*export.enums, *inline_enums]: + enum_lookup[self._normalize_type(enum.type)] = self.get_size_from_string( + enum.underlying + ) + template_struct_lookup: dict[str, DefinedStruct] = {} + struct_definition_lookup: dict[str, DefinedStruct] = {} + for struct in all_structs: + normalized_type = self._normalize_type(struct.type) + struct_definition_lookup[normalized_type] = struct + if struct.template_types: + template_struct_lookup[normalized_type] = struct def append_chunk(chunk: IndentWriter | str): chunk_text = str(chunk) @@ -427,37 +521,95 @@ def append_ordered_chunks(chunks: list[tuple[str, IndentWriter]]): output.append("}") open_namespaces.pop() - enum_chunks = [ - (enum.namespace, self.build_enum_string(enum)) for enum in export.enums - ] - append_ordered_chunks(enum_chunks) - + enum_chunks = [] forward_chunks: list[tuple[str, IndentWriter]] = [] + + for enum in export.enums: + if self._normalize_type(enum.type) in inline_enum_types: + continue + enum_forward = IndentWriter() + enum_forward += f"enum {enum.name} : {enum.underlying};" + forward_chunks.append((enum.namespace, enum_forward)) + enum_chunks.append((enum.namespace, self.build_enum_string(enum))) + definition_chunks: list[tuple[str, IndentWriter]] = [] + inline_enum_definition_chunks: list[tuple[str, IndentWriter]] = [] vtable_names: list[str] = [] + seen_vtable_names: set[str] = set() + + def collect_inline_enum_definitions(struct: DefinedStruct): + namespace_prefix = struct.namespace.replace(".", "::") + owner_type = struct.type + if namespace_prefix and owner_type.startswith(f"{namespace_prefix}::"): + owner_type = owner_type[len(namespace_prefix) + 2 :] + if struct.template_types: + owner_type += f"<{', '.join(struct.template_types)}>" + + for inline_enum in struct.inline_enums: + inline_enum_definition = self.build_enum_string( + inline_enum, owner_type, tuple(struct.template_types) + ) + inline_enum_definition_chunks.append( + (inline_enum.namespace, inline_enum_definition) + ) + for inline_struct in struct.inline_structs: + collect_inline_enum_definitions(inline_struct) + + def collect_vtable_names(struct: DefinedStruct): + if struct.virtual_functions: + vtable_name = f"{struct.type}_vtbl" + if vtable_name not in seen_vtable_names: + seen_vtable_names.add(vtable_name) + vtable_names.append(vtable_name) + for inline_struct in struct.inline_structs: + collect_vtable_names(inline_struct) + for struct in export.structs: + if self._normalize_type(struct.type) in inline_struct_types: + continue forward, define, vtable, vtable_full_name = self.build_struct_string( - struct, struct_lookup, enum_lookup, template_struct_lookup, full_declspec_align + struct, + struct_lookup, + enum_lookup, + template_struct_lookup, + full_declspec_align, + struct_definition_lookup, ) - if vtable_full_name: + if vtable_full_name and vtable_full_name not in seen_vtable_names: + seen_vtable_names.add(vtable_full_name) vtable_names.append(vtable_full_name) + collect_vtable_names(struct) + collect_inline_enum_definitions(struct) forward_chunks.append((struct.namespace, forward)) definition_chunks.append((struct.namespace, define)) definition_chunks.append((struct.namespace, vtable)) + def forward_chunk_key(key: tuple[str, IndentWriter]) -> str: + (namespace, _) = key + return namespace + + forward_chunks = sorted(forward_chunks, key=forward_chunk_key) + append_ordered_chunks(forward_chunks) + append_ordered_chunks(enum_chunks) append_ordered_chunks(definition_chunks) + append_ordered_chunks(inline_enum_definition_chunks) return (str(output), vtable_names) def build_enum_string( self, enum: DefinedStructEnum, + enum_scope: str | None = None, + template_types: tuple[str, ...] = (), ): output = IndentWriter() - enum_name = self.get_short_name(enum.type) - output += f"enum {enum_name} : {enum.underlying} {{" + enum_name = enum.name + if template_types: + output += f"template" + enum_type = f"{enum_scope}::{enum_name}" if enum_scope != None else enum_name + output += f"enum {enum_type} : {enum.underlying} {{" output.indent() for value_name, value in enum.values.items(): @@ -467,7 +619,7 @@ def build_enum_string( output += "};" return output - + def get_struct_pad(self, fill_size: int, prev_size: int): if self.full_padding: type, size = self.get_type_and_type_size_from_gap(prev_size) @@ -484,7 +636,8 @@ def build_struct_string( struct_lookup: dict[str, int], enum_lookup: dict[str, int], template_struct_lookup: dict[str, DefinedStruct] | None = None, - declspec_align: int = 0 + declspec_align: int = 0, + struct_definition_lookup: dict[str, DefinedStruct] | None = None, ) -> tuple[IndentWriter, IndentWriter, IndentWriter, str | None]: """Builds the C++ struct definition with namespace wrapping. @@ -517,7 +670,8 @@ def build_struct_string( # Forward declarations if template_declaration: struct_string_forward += template_declaration - struct_string_forward += f"struct {short_name};" + struct_string_forward += f"struct {struct.name};" + if struct.virtual_functions: if template_declaration: struct_string_forward += template_declaration @@ -536,14 +690,14 @@ def build_struct_string( if base_field.offset == prev_size: base_fields_used.append(base_field.type) field_type = self._get_field_type(base_field, template_struct_lookup) - short_base_name = self.get_short_name(field_type) - struct_extend_lines.append(short_base_name) + struct_extend_lines.append(f"::{field_type}" if not self.is_base_type(field_type) else field_type) prev_size += self.get_size_from_type( field_type, struct_lookup, enum_lookup, template_struct_lookup, tuple(struct.template_types), + struct_definition_lookup, ) struct_extend_line = "" @@ -553,6 +707,37 @@ def build_struct_string( struct_string_define += f"{struct_define_line}{struct_extend_line} {{" struct_string_define.indent() + for inline_enum in struct.inline_enums: + struct_string_define += f"enum {self.get_short_name(inline_enum.name)} : {inline_enum.underlying};" + + def append_chunk(chunk: IndentWriter | str): + for line in str(chunk).split("\n"): + if not line.strip(): + continue + leading = len(line) - len(line.lstrip(" ")) + previous_indent = struct_string_define.indent_level + struct_string_define.indent_level = previous_indent + leading // 2 + struct_string_define.append(line.lstrip(" ")) + struct_string_define.indent_level = previous_indent + + inline_struct_chunks = [ + self.build_struct_string( + inline_struct, + struct_lookup, + enum_lookup, + template_struct_lookup, + declspec_align, + struct_definition_lookup, + ) + for inline_struct in struct.inline_structs + ] + + for inline_forward, _, _, _ in inline_struct_chunks: + append_chunk(inline_forward) + for _, inline_define, inline_vtable, _ in inline_struct_chunks: + append_chunk(inline_define) + append_chunk(inline_vtable) + # Add __vtable member if this struct has virtual functions if struct.virtual_functions and prev_size == 0: struct_string_define += f"{short_name}_vtbl{template_suffix}* __vtable;" @@ -566,11 +751,12 @@ def build_struct_string( offset = field.offset - while offset > prev_size: - fill_size = offset - prev_size - line, size = self.get_struct_pad(fill_size, prev_size) - struct_string_define += line - prev_size += size + if not struct.is_sequential: + while offset > prev_size: + fill_size = offset - prev_size + line, size = self.get_struct_pad(fill_size, prev_size) + struct_string_define += line + prev_size += size field_name = field.name field_type = field.type @@ -581,7 +767,8 @@ def build_struct_string( struct_lookup, enum_lookup, template_struct_lookup, - tuple(struct.template_types), + struct.template_types, + struct_definition_lookup, ) struct_string_define += f"{field_define[:-1]}[{field.size}];" prev_size += field_define_size * field.size @@ -592,7 +779,9 @@ def build_struct_string( ) for param in field.parameters: fastcall_field_define += param.type + " " + param.name + ", " - struct_string_define += fastcall_field_define[:-1] + ");" + if len(field.parameters) > 0: + fastcall_field_define = fastcall_field_define[:-2] + struct_string_define += fastcall_field_define + ");" prev_size += 8 else: field_define, field_define_size = self.get_string_define_and_size( @@ -601,7 +790,8 @@ def build_struct_string( struct_lookup, enum_lookup, template_struct_lookup, - tuple(struct.template_types), + struct.template_types, + struct_definition_lookup, ) struct_string_define += field_define prev_size += field_define_size @@ -634,4 +824,26 @@ def build_struct_string( struct_string_vtable += "};" vtable_full_name = f"{struct.type}_vtbl" - return (struct_string_forward, struct_string_define, struct_string_vtable, vtable_full_name) \ No newline at end of file + return (struct_string_forward, struct_string_define, struct_string_vtable, vtable_full_name) + + def get_predef_header(self): + return """#define _HAS_ITERATOR_DEBUGGING 0 +#define _ITERATOR_DEBUG_LEVEL 0 +#define _AMD64_ +#undef __unaligned +#define __unaligned +#define __IZoneIdentifier2_FWD_DEFINED__ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +""" \ No newline at end of file diff --git a/scripts/structs_schema.py b/scripts/structs_schema.py index 380e97dd15..c4ac535836 100644 --- a/scripts/structs_schema.py +++ b/scripts/structs_schema.py @@ -85,12 +85,15 @@ class DefinedStruct(DefinedStructBase): fields: Fields size: int vtable_size: int | None - virtual_functions: list[DefinedStructVFunc] | None + virtual_functions: list[DefinedStructVFunc] member_functions: list[DefinedStructMemFunc] union: bool static_member_functions: list[DefinedStructMemFunc] | None static_members: list[DefinedStructStaticMember] | None template_types: list[str] + inline_structs: list["DefinedStruct"] + inline_enums: list[DefinedStructEnum] + is_sequential: bool @dataclass @@ -115,141 +118,161 @@ def get_yaml(stream, loader=Loader) -> DefinedStructExport: ) ) for struct in dic["structs"]: - fields = [] - virtual_functions = None - member_functions = [] - static_member_functions = None - static_members = None - template_types = [] - for field in struct["fields"]: - base = field["base"] if "base" in field else False - template_args = [] - if "template_args" in field: - for template_arg in field["template_args"]: - template_args.append(template_arg) - if "size" in field: - fields.append( - DefinedStructFixedField( - field["name"], - field["type"], - field["offset"], - base, - template_args, - field["size"], - field["is_string"], - ) + structs.append(parse_struct(struct)) + return DefinedStructExport(enums, structs) + +def parse_struct(struct: YamlStructExport) -> DefinedStruct: + fields = [] + virtual_functions = [] + member_functions = [] + static_member_functions = None + static_members = None + template_types = [] + inline_structs = [] + inline_enums = [] + for field in struct["fields"]: + base = field["base"] if "base" in field else False + template_args = [] + if "template_args" in field: + for template_arg in field["template_args"]: + template_args.append(template_arg) + if "size" in field: + fields.append( + DefinedStructFixedField( + field["name"], + field["type"], + field["offset"], + base, + template_args, + field["size"], + field["is_string"], + ) + ) + elif "return_type" in field: + parameters = [] + for param in field["parameters"]: + parameters.append( + DefinedStructFuncParam(param["name"], param["type"]) + ) + fields.append( + DefinedStructFuncField( + field["name"], + field["type"], + field["offset"], + base, + template_args, + field["return_type"], + parameters, + ) + ) + else: + fields.append( + DefinedStructField( + field["name"], + field["type"], + field["offset"], + base, + template_args, ) - elif "return_type" in field: - parameters = [] - for param in field["parameters"]: - parameters.append( - DefinedStructFuncParam(param["name"], param["type"]) - ) - fields.append( - DefinedStructFuncField( - field["name"], - field["type"], - field["offset"], - base, - template_args, - field["return_type"], - parameters, - ) + ) + if "virtual_functions" in struct: + for vfunc in struct["virtual_functions"]: + parameters = ( + [ + DefinedStructFuncParam(param["name"], param["type"]) + for param in vfunc["parameters"] + ] + if "parameters" in vfunc + else None + ) + virtual_functions.append( + DefinedStructVFunc( + vfunc["name"], + vfunc["return_type"] if "return_type" in vfunc else None, + vfunc["offset"], + parameters, ) - else: - fields.append( - DefinedStructField( - field["name"], - field["type"], - field["offset"], - base, - template_args, - ) + ) + if "member_functions" in struct: + for memfunc in struct["member_functions"]: + parameters = [] + for param in memfunc["parameters"]: + parameters.append( + DefinedStructFuncParam(param["name"], param["type"]) ) - if "virtual_functions" in struct: - virtual_functions = [] - for vfunc in struct["virtual_functions"]: - parameters = ( - [ - DefinedStructFuncParam(param["name"], param["type"]) - for param in vfunc["parameters"] - ] - if "parameters" in vfunc - else None + member_functions.append( + DefinedStructMemFunc( + memfunc["signature"], + memfunc["return_type"], + parameters, + memfunc["name"], ) - virtual_functions.append( - DefinedStructVFunc( - vfunc["name"], - vfunc["return_type"] if "return_type" in vfunc else None, - vfunc["offset"], - parameters, - ) + ) + if "static_member_functions" in struct: + static_member_functions = [] + for smemfunc in struct["static_member_functions"]: + parameters = [] + for param in smemfunc["parameters"]: + parameters.append( + DefinedStructFuncParam(param["name"], param["type"]) ) - if "member_functions" in struct: - for memfunc in struct["member_functions"]: - parameters = [] - for param in memfunc["parameters"]: - parameters.append( - DefinedStructFuncParam(param["name"], param["type"]) - ) - member_functions.append( - DefinedStructMemFunc( - memfunc["signature"], - memfunc["return_type"], - parameters, - memfunc["name"], - ) + static_member_functions.append( + DefinedStructMemFunc( + smemfunc["signature"], + smemfunc["return_type"], + parameters, + smemfunc["name"], ) - if "static_member_functions" in struct: - static_member_functions = [] - for smemfunc in struct["static_member_functions"]: - parameters = [] - for param in smemfunc["parameters"]: - parameters.append( - DefinedStructFuncParam(param["name"], param["type"]) - ) - static_member_functions.append( - DefinedStructMemFunc( - smemfunc["signature"], - smemfunc["return_type"], - parameters, - smemfunc["name"], - ) + ) + if "static_members" in struct: + static_members = [] + for sm in struct["static_members"]: + static_members.append( + DefinedStructStaticMember( + sm["signature"], + sm["relative_follow_offsets"], + sm["return_type"], + sm["is_pointer"] if "is_pointer" in sm else False, ) - if "static_members" in struct: - static_members = [] - for sm in struct["static_members"]: - static_members.append( - DefinedStructStaticMember( - sm["signature"], - sm["relative_follow_offsets"], - sm["return_type"], - sm["is_pointer"] if "is_pointer" in sm else False, - ) + ) + if "template_types" in struct: + for type in struct["template_types"]: + template_types.append(type) + if "inline_structs" in struct: + for inline_struct in struct["inline_structs"]: + inline_structs.append(parse_struct(inline_struct)) + if "inline_enums" in struct: + for enum in struct["inline_enums"]: + inline_enums.append( + DefinedStructEnum( + enum["name"], + enum["type"], + enum["namespace"], + enum["underlying"], + enum["flags"], + enum["values"], ) - if "template_types" in struct: - for type in struct["template_types"]: - template_types.append(type) - size = None - if "size" in struct: - size = struct["size"] - vtable_size = None - if "vtable_size" in struct: - vtable_size = struct["vtable_size"] - structs.append( - DefinedStruct( - struct["name"], - struct["type"], - struct["namespace"], - fields, - size, - vtable_size, - virtual_functions, - member_functions, - struct["union"], - static_member_functions, - static_members, - template_types, ) + size = None + if "size" in struct: + size = struct["size"] + vtable_size = None + if "vtable_size" in struct: + vtable_size = struct["vtable_size"] + return DefinedStruct( + struct["name"], + struct["type"], + struct["namespace"], + fields, + size, + vtable_size, + virtual_functions, + member_functions, + struct["union"], + static_member_functions, + static_members, + template_types, + inline_structs, + inline_enums, + struct["is_sequential"] if "is_sequential" in struct else False ) - return DefinedStructExport(enums, structs) diff --git a/scripts/test_execute.py b/scripts/test_execute.py index 0cb90a56a3..34a7da9815 100644 --- a/scripts/test_execute.py +++ b/scripts/test_execute.py @@ -8,23 +8,24 @@ from lumina_wrapper import get_excel_header_files from io import open -predef_header = """#define _HAS_ITERATOR_DEBUGGING 0 -#define _ITERATOR_DEBUG_LEVEL 0 -#include -#include -#include -#include -#include -#include -#include - -""" - src = SrcInterface() with open("ffxiv_structs.yml") as f: struct_export = get_yaml(f) +# for struct in struct_export.structs: +# for virtual_function in struct.virtual_functions: +# not_gui = not struct.namespace.endswith("GUI") +# not_agent = not struct.namespace.endswith("Agent") +# not_gui_or_agent = not_gui or not_agent +# not_receive_event = virtual_function.name != "ReceiveEvent" +# not_gui_or_agent_and_receive_event = not_gui_or_agent and not_receive_event +# not_dtor = virtual_function.name != "dtor" +# not_dtor_and_gui_or_agent_and_receive_event = not_dtor and not_gui_or_agent_and_receive_event +# not_offset = not virtual_function.offset +# if not_offset and not_dtor_and_gui_or_agent_and_receive_event: +# print(f"{struct.namespace}.{struct.name}_vtbl.{virtual_function.name}", not_offset, not_dtor_and_gui_or_agent_and_receive_event, not_dtor, not_gui_or_agent_and_receive_event, not_receive_event, not_gui_or_agent, not_agent, not_gui) + header, vtables = src.build_export_string(struct_export) excel_header_files = get_excel_header_files() @@ -37,9 +38,11 @@ header_excel, vtables_excel = src.build_export_string(struct_export_excel, 4) with open("test.h", "w") as f: - f.write(predef_header) - f.write(header_excel) - f.write(header) + f.write(src.get_predef_header()) + f.write("\n") + f.write(header_excel) + f.write("\n") + f.write(header) running_time = time() - current_time print(f"execution took {running_time}s") \ No newline at end of file From 6422130c4c72a0e1cb729b9c4921d1222d42d0f1 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Sun, 11 Oct 2026 13:30:28 +0200 Subject: [PATCH 40/95] get proper inline struct and enum dependency chain resolver working --- CExporter/Exporter.cs | 67 ++++++++++++++++++++++++++++++-------- CExporter/ProcessedData.cs | 19 ++++++++++- CExporter/Program.cs | 1 + 3 files changed, 72 insertions(+), 15 deletions(-) diff --git a/CExporter/Exporter.cs b/CExporter/Exporter.cs index c50b3bed70..d60413eac8 100644 --- a/CExporter/Exporter.cs +++ b/CExporter/Exporter.cs @@ -291,36 +291,75 @@ public static void ProcessDefinedVTables(DataDefinition def) { } } - public static void Write(DirectoryInfo dir) { - // make sure we have all the dependencies for each struct before we write them + public static void VerifyDependencies() => ResolveFullDependencies(); + + private static Dictionary> ResolveFullDependencies() { var structs = _structs.SelectMany(t => t.ChildrenAndSelf).DistinctBy(x => x.StructTypeName).ToFrozenDictionary(x => x.StructTypeName, x => x); var structToFullDeps = new Dictionary>(); - var structDeps = structs.SelectMany(t => ResolveFullDependencies(t.Value)).Distinct().ToArray(); - _structs.Sort((a, b) => { - var cmp = ResolveFullDependencies(a).Count.CompareTo(ResolveFullDependencies(b).Count); - return cmp != 0 ? cmp : string.Compare(a.StructTypeName, b.StructTypeName, StringComparison.Ordinal); - }); + var resolving = new List(); - HashSet ResolveFullDependencies(ProcessedStruct s) { + foreach (var processedStruct in structs.Values) + resolve(processedStruct); + + return structToFullDeps; + + HashSet resolve(ProcessedStruct s) { if (structToFullDeps.TryGetValue(s.StructTypeName, out var deps)) return deps; + var cycleStart = resolving.IndexOf(s.StructTypeName); + if (cycleStart >= 0) { + var cycle = resolving.Skip(cycleStart).Append(s.StructTypeName).ToArray(); + var error = $"Cyclic type dependency detected: {string.Join(" -> ", cycle)}"; + if (!ExporterStatics.ErrorList.Contains(error)) ExporterStatics.ErrorList.Add(error); + return []; + } + + resolving.Add(s.StructTypeName); var fullDeps = s.DependencyNames.ToHashSet(); - foreach (var dn in s.DependencyNames) - fullDeps.UnionWith(ResolveFullDependencies(structs[dn])); + foreach (var dependencyName in s.DependencyNames) { + if (!structs.TryGetValue(dependencyName, out var dependency)) { + var error = $"Unable to resolve type dependency {dependencyName} referenced by {s.StructTypeName}"; + if (!ExporterStatics.ErrorList.Contains(error)) ExporterStatics.ErrorList.Add(error); + continue; + } + + fullDeps.UnionWith(resolve(dependency)); + if (dependency.StructType.DeclaringType is not null) { + var owner = dependency.StructType; + while (owner.DeclaringType is not null) owner = owner.DeclaringType; + var ownerName = owner.FixTypeName(); + var sourceOwner = s.StructType; + while (sourceOwner.DeclaringType is not null) sourceOwner = sourceOwner.DeclaringType; + if (ownerName != sourceOwner.FixTypeName() && structs.TryGetValue(ownerName, out var ownerStruct)) { + fullDeps.Add(ownerName); + fullDeps.UnionWith(resolve(ownerStruct)); + } + } + } + resolving.RemoveAt(resolving.Count - 1); return structToFullDeps[s.StructTypeName] = fullDeps; } + } + + public static void Write(DirectoryInfo dir) { + // make sure we have all the dependencies for each struct before we write them + var structToFullDeps = ResolveFullDependencies(); + _structs.Sort((a, b) => { + var cmp = structToFullDeps[a.StructTypeName].Count.CompareTo(structToFullDeps[b.StructTypeName].Count); + return cmp != 0 ? cmp : string.Compare(a.StructTypeName, b.StructTypeName, StringComparison.Ordinal); + }); - void OrderInlineStructs(ProcessedStruct parent) { + void orderInlineStructs(ProcessedStruct parent) { if (parent.InlineStructs == null) return; parent.InlineStructs = [.. parent.InlineStructs - .OrderBy(t => ResolveFullDependencies(t).Count) + .OrderBy(t => structToFullDeps[t.StructTypeName].Count) .ThenBy(t => t.StructTypeName, StringComparer.Ordinal)]; foreach (var inlineStruct in parent.InlineStructs) - OrderInlineStructs(inlineStruct); + orderInlineStructs(inlineStruct); } foreach (var s in _structs) - OrderInlineStructs(s); + orderInlineStructs(s); var serializer = new SerializerBuilder() .WithNamingConvention(UnderscoredNamingConvention.Instance) diff --git a/CExporter/ProcessedData.cs b/CExporter/ProcessedData.cs index c0d04a6054..bbd6729ff3 100644 --- a/CExporter/ProcessedData.cs +++ b/CExporter/ProcessedData.cs @@ -83,15 +83,24 @@ public string[] DependencyNames { if (_dependencyNames.Length == 0) { _dependencyNames = Fields .Where(t => { - if (t.FieldTypeOverride == null) return (t.GetType() == typeof(ProcessedField) || t.GetType() == typeof(ProcessedFixedField)) && !t.FieldType.ShouldNotExportType() && !ExporterStatics.BaseTypeNames.Contains(t.FieldType.FixTypeName()) && !t.FieldType.ContainsGenericParameters; + if (t.FieldTypeOverride == null) { + if (t.FieldType.IsEnum) { + var enumOwner = t.FieldType.DeclaringType; + return enumOwner != null && !IsNestedWithin(StructType, enumOwner); + } + return (t.GetType() == typeof(ProcessedField) || t.GetType() == typeof(ProcessedFixedField)) && !t.FieldType.ShouldNotExportType() && !ExporterStatics.BaseTypeNames.Contains(t.FieldType.FixTypeName()) && !t.FieldType.ContainsGenericParameters; + } if (t.FieldTypeOverride.EndsWith('*') || t.FieldTypeOverride.StartsWith("Component::Exd::Sheets::") || ExporterStatics.BaseTypeNames.Contains(t.FieldTypeOverride)) return false; return !t.FieldTypeOverrideCheck.HasValue || !t.FieldTypeOverrideCheck.Value; }) .SelectMany(t => { + if (t.FieldTypeOverride == null && t.FieldType.IsEnum) + return [t.FieldType.DeclaringType!.FixTypeName()]; if (t.FieldTypeOverride?.StartsWith(ExporterStatics.StdCppNamespace) ?? false) return t.FieldTypeOverrideTemplates!.Where(k => !k.ShouldNotExportType()).Select(k => k.FixTypeName()).Where(k => !ExporterStatics.BaseTypeNames.Contains(k)); return [t.FieldTypeOverride ?? t.FieldType.FixTypeName()]; }) .Concat(InlineStructs?.SelectMany(t => t.DependencyNames) ?? Enumerable.Empty()) + .Concat(InlineStructs?.Select(t => t.StructTypeName) ?? Enumerable.Empty()) .Distinct() .ToArray(); } @@ -99,6 +108,14 @@ public string[] DependencyNames { } } + private static bool IsNestedWithin(Type type, Type possibleOwner) { + if (type == possibleOwner) return true; + for (var declaringType = type.DeclaringType; declaringType != null; declaringType = declaringType.DeclaringType) { + if (declaringType == possibleOwner) return true; + } + return false; + } + public ProcessedStruct[] ChildrenAndSelf => InlineStructs != null && InlineStructs.Any() ? [this, .. InlineStructs.SelectMany(t => t.ChildrenAndSelf)] : [this]; public ProcessedStruct FixOrder() { Fields = [.. Fields.OrderBy(t => t.FieldOffset)]; diff --git a/CExporter/Program.cs b/CExporter/Program.cs index e60adc05a8..020dab46e8 100644 --- a/CExporter/Program.cs +++ b/CExporter/Program.cs @@ -37,6 +37,7 @@ public static void Main(string[] args) { Exporter.VerifyNoNameOverlap(dataCheck); Exporter.ProcessDefinedVTables(data); + Exporter.VerifyDependencies(); if (!quiet) Console.WriteLine($"Processed all types in: {DateTime.Now - timeStart}"); timeStart = DateTime.Now; From f2f6a3dd9cf4495aea30345edb24be1a6dd67dc4 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Sun, 11 Oct 2026 13:30:54 +0200 Subject: [PATCH 41/95] fix cyclic error reported by CExporter --- .../FFXIV/Client/Game/UI/ContentsFinder.cs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/FFXIVClientStructs/FFXIV/Client/Game/UI/ContentsFinder.cs b/FFXIVClientStructs/FFXIV/Client/Game/UI/ContentsFinder.cs index 0e43258bc3..4df138be04 100644 --- a/FFXIVClientStructs/FFXIV/Client/Game/UI/ContentsFinder.cs +++ b/FFXIVClientStructs/FFXIV/Client/Game/UI/ContentsFinder.cs @@ -10,7 +10,7 @@ public unsafe partial struct ContentsFinder { [StaticAddress("0F B6 FA 48 8D 0D ?? ?? ?? ?? E8 ?? ?? ?? ?? 80 78 59 02", 6)] public static partial ContentsFinder* Instance(); - [FieldOffset(0x18)] public LootRule LootRules; + [FieldOffset(0x18)] public ContentsFinderQueueInfo.LootRule LootRules; [FieldOffset(0x19)] public bool IsUnrestrictedParty; [FieldOffset(0x1A)] public bool IsMinimalIL; @@ -22,12 +22,6 @@ public unsafe partial struct ContentsFinder { [MemberFunction("E8 ?? ?? ?? ?? 33 FF 0F B6 48")] public partial ContentsFinderQueueInfo* GetQueueInfo(); - - public enum LootRule : byte { - Normal = 0, - GreedOnly = 1, - Lootmaster = 2 - } } [GenerateInterop] @@ -48,7 +42,7 @@ public unsafe partial struct ContentsFinderQueueInfo { [FieldOffset(0x5A)] public byte QueuedContentRouletteId; [FieldOffset(0x5B)] public sbyte ClampedPositionInQueue; [FieldOffset(0x5C)] public sbyte PositionInQueue; - [FieldOffset(0x5D)] public ContentsFinder.LootRule PoppedContentLootRule; + [FieldOffset(0x5D)] public LootRule PoppedContentLootRule; [FieldOffset(0x60)] public bool PoppedContentIsInProgress; @@ -101,7 +95,7 @@ public partial void OnQueuePop( uint contentId, nint a4, bool isInProgressParty, - ContentsFinder.LootRule lootRule, + LootRule lootRule, ulong inProgressPartyStartTimestamp, nint a8, bool isUnrestrictedParty, @@ -110,6 +104,12 @@ public partial void OnQueuePop( bool isExplorerMode, bool isLevelSync, bool isLimitedLeveling); + + public enum LootRule : byte { + Normal = 0, + GreedOnly = 1, + Lootmaster = 2 + } } [StructLayout(LayoutKind.Explicit, Size = 0x10)] From 82a877eaf171c3f10f8ce024c6bde599a86e5e82 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Sun, 11 Oct 2026 14:54:58 +0200 Subject: [PATCH 42/95] get some type collisions fixed C# generics workaround fixing --- CExporter/Exporter.TypeProcessing.cs | 53 +++++- scripts/src_wrapper.py | 256 +++++++++++++++++++++++++-- 2 files changed, 290 insertions(+), 19 deletions(-) diff --git a/CExporter/Exporter.TypeProcessing.cs b/CExporter/Exporter.TypeProcessing.cs index 99a2a0be9a..bbb8ef6c48 100644 --- a/CExporter/Exporter.TypeProcessing.cs +++ b/CExporter/Exporter.TypeProcessing.cs @@ -1,14 +1,11 @@ using System; using System.Collections.Frozen; using System.Collections.Generic; -using System.Diagnostics; using System.Linq; using System.Reflection; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using FFXIVClientStructs.Attributes; -using FFXIVClientStructs.Havok.Common.Base.Types; -using FFXIVClientStructs.STD; using InteropGenerator.Runtime.Attributes; namespace CExporter; @@ -20,7 +17,40 @@ private static void ProcessType(ProcessingType type, bool quiet) { case null: return; case ProcessedStruct s: - _structs.Add(s); + var existingIndex = _structs.FindIndex(existing => + existing.StructTypeName == s.StructTypeName); + if (existingIndex < 0) { + _structs.Add(s); + } else if (s.TemplateTypes.Length > 0 && _structs[existingIndex].TemplateTypes.Length == 0) { + var nonGeneric = _structs[existingIndex]; + s.Fields = [.. s.Fields.Where(field => + !field.IsBase || field.FieldType != nonGeneric.StructType)]; + s.VirtualFunctions ??= nonGeneric.VirtualFunctions; + s.VirtualFunctionSize = Math.Max(s.VirtualFunctionSize, nonGeneric.VirtualFunctionSize); + s.MemberFunctions = [.. nonGeneric.MemberFunctions, .. s.MemberFunctions]; + var genericSelfType = $"{nonGeneric.StructTypeName}<{string.Join(", ", s.TemplateTypes)}>*"; + foreach (var virtualFunction in s.VirtualFunctions ?? []) { + foreach (var parameter in virtualFunction.VirtualFunctionParameters ?? []) { + if (parameter.FieldName == "self" + && parameter.FieldType.IsPointer() + && parameter.FieldType.GetPointerType() == nonGeneric.StructType) + parameter.FieldTypeOverride = genericSelfType; + } + if ((virtualFunction.VirtualFunctionReturnType?.IsPointer() ?? false) && virtualFunction.VirtualFunctionReturnType?.GetPointerType() == nonGeneric.StructType) + virtualFunction.VirtualFunctionReturnTypeOverride = genericSelfType; + } + foreach (var memberFunction in s.MemberFunctions) { + foreach (var parameter in memberFunction.MemberFunctionParameters) { + if (parameter.FieldName == "self" + && parameter.FieldType.IsPointer() + && parameter.FieldType.GetPointerType() == nonGeneric.StructType) + parameter.FieldTypeOverride = genericSelfType; + } + if (memberFunction.MemberFunctionReturnType.Type.IsPointer() && memberFunction.MemberFunctionReturnType.Type.GetPointerType() == nonGeneric.StructType) + memberFunction.MemberFunctionReturnType.OverrideType = genericSelfType; + } + _structs[existingIndex] = s; + } break; case ProcessedEnum e: _enums.Add(e); @@ -413,7 +443,16 @@ private static ProcessedField ProcessParameter( if (type.Name.Contains('`')) { type = type.GetGenericTypeDefinition(); } - if (type.IsInStructList(_structs) || (overrideType?.IsInStructList(_structs) ?? false)) return null; + var typeName = type.FixTypeName(); + var existingIndex = _structs.FindIndex(existing => + existing.StructTypeName == typeName); + var templateTypes = type.IsGenericTypeDefinition + && !type.GetGenericArguments().GenericSequenceEqual(type.DeclaringType?.GetGenericArguments() ?? []) + ? type.GetGenericArguments().Select(t => t.Name).ToArray() + : []; + if (existingIndex >= 0 && (templateTypes.Length == 0 || _structs[existingIndex].TemplateTypes.Length > 0)) + return null; + if (overrideType?.IsInStructList(_structs) ?? false) return null; var vtable = type.GetField("VirtualTable", ExporterStatics.BindingFlags)?.FieldType; var vtableSize = 0; ProcessedVirtualFunction[]? virtualFunctions = null; @@ -517,7 +556,7 @@ .. memberFunctionParameters.Select(ProcessParameter).ToArray()] VirtualFunctions = virtualFunctions, MemberFunctions = memberFunctionsArray, StructTypeOverride = overrideType, - TemplateTypes = type.IsGenericTypeDefinition && !type.GetGenericArguments().GenericSequenceEqual(type.DeclaringType?.GetGenericArguments() ?? []) ? type.GetGenericArguments().Select(t => t.Name).ToArray() : [], + TemplateTypes = templateTypes, InlineStructs = nestedTypes.Where(t => t.IsStruct()).Select(t => (ProcessedStruct?)PreProcessType(t, quiet)).Where(t => t != null).ToArray()!, InlineEnums = nestedTypes.Where(t => t.IsEnum).Select(t => (ProcessedEnum?)PreProcessType(t, quiet)).Where(t => t != null).ToArray()!, IsSequential = type.StructLayoutAttribute?.Value == LayoutKind.Sequential, @@ -600,7 +639,7 @@ public static ProcessedField[] ProcessFields(FieldInfo[] fields) { } for (var i = 0; i < fieldsToProcess.Length; i++) { var field = fieldsToProcess[i]; - if (field.FieldType.ContainsGenericParameters) continue; + if (field.FieldType.ContainsGenericParameters && !field.FieldType.IsGenericTypeParameterOrPointer()) continue; if (field.GetCustomAttribute() != null) _processType.Add(field.FieldType); if (field.GetCustomAttribute() != null) { diff --git a/scripts/src_wrapper.py b/scripts/src_wrapper.py index a9d2bc0995..f52992af31 100644 --- a/scripts/src_wrapper.py +++ b/scripts/src_wrapper.py @@ -89,6 +89,35 @@ def _normalize_type(self, type_name: str) -> str: normalized = " ".join(base_type.split()) return normalized + pointer_suffix + def _qualify_template_arguments( + self, type_name: str, template_parameters: tuple[str, ...] = () + ) -> str: + """Make non-primitive template arguments independent of the current namespace.""" + type_name = type_name.strip() + pointer_suffix = "" + while type_name.endswith("*"): + pointer_suffix += "*" + type_name = type_name[:-1].strip() + + base_type, template_args = self._split_template_type(type_name) + if not template_args: + return type_name + pointer_suffix + + qualified_args = [] + for argument in template_args: + argument = self._qualify_template_arguments(argument, template_parameters) + argument_base = argument + while argument_base.endswith("*"): + argument_base = argument_base[:-1].strip() + if ( + argument_base not in template_parameters + and not self.is_base_type(argument_base) + and not argument_base.startswith("::") + ): + argument = f"::{argument}" + qualified_args.append(argument) + return f"{base_type}<{', '.join(qualified_args)}>{pointer_suffix}" + def _get_struct_size( self, type_name: str, @@ -362,10 +391,20 @@ def get_string_define_and_size( template_struct_lookup: dict[str, DefinedStruct] | None = None, template_parameters: list[str] = [], struct_definition_lookup: dict[str, DefinedStruct] | None = None, + field_name_override: str | None = None, + opaque_pointer_types: set[str] | None = None, ) -> tuple[str, int]: template_struct_lookup = template_struct_lookup or {} field_type = self._get_field_type(field, template_struct_lookup) - field_name = field.name + opaque_pointer_types = opaque_pointer_types or set() + pointer_count = 0 + pointer_base = field_type + while pointer_base.endswith("*"): + pointer_base = pointer_base[:-1].strip() + pointer_count += 1 + if pointer_count and self._normalize_type(pointer_base) in opaque_pointer_types: + field_type = "void" + "*" * pointer_count + field_name = field_name_override or field.name field_size = self.get_size_from_type( field_type, struct_lookup, @@ -374,7 +413,9 @@ def get_string_define_and_size( template_parameters, struct_definition_lookup, ) - display_type = field_type + display_type = self._qualify_template_arguments( + field_type, tuple(template_parameters) + ) if display_type.startswith(namespace): display_type = display_type.removeprefix(f"{namespace}::") elif not self.is_base_type(display_type) and not display_type.rstrip('*') in template_parameters: @@ -397,7 +438,9 @@ def get_short_name(self, full_type: str) -> str: index += 1 return full_type[last_namespace_separator + 1 :] - def build_vfunc_signature(self, vfunc: DefinedStructVFunc) -> str: + def build_vfunc_signature( + self, vfunc: DefinedStructVFunc, name_override: str | None = None + ) -> str: """Build a function pointer signature for a virtual function. Args: @@ -406,8 +449,9 @@ def build_vfunc_signature(self, vfunc: DefinedStructVFunc) -> str: Returns: Function pointer signature string """ + function_name = name_override or vfunc.name if not (vfunc.return_type and vfunc.parameters): - return f"void* {vfunc.name};" + return f"void* {function_name};" return_type = vfunc.return_type if vfunc.return_type else "void" @@ -426,7 +470,7 @@ def build_vfunc_signature(self, vfunc: DefinedStructVFunc) -> str: param_list = ", ".join(params) - return f"{return_type} (__fastcall *{vfunc.name})({param_list});" + return f"{return_type} (__fastcall *{function_name})({param_list});" def build_export_string( self, @@ -438,6 +482,7 @@ def build_export_string( all_structs: list[DefinedStruct] = [] inline_structs: list[DefinedStruct] = [] inline_enums: list[DefinedStructEnum] = [] + inline_type_owners: dict[str, str] = {} def collect_inline_types(struct: DefinedStruct): for inline_enum in struct.inline_enums: @@ -454,6 +499,19 @@ def collect_inline_types(struct: DefinedStruct): inline_struct_types = { self._normalize_type(struct.type) for struct in inline_structs } + + def collect_inline_type_owners(struct: DefinedStruct, owner_type: str): + for inline_struct in struct.inline_structs: + inline_type_owners.setdefault( + self._normalize_type(inline_struct.type), + self._normalize_type(owner_type), + ) + collect_inline_type_owners(inline_struct, owner_type) + + for struct in export.structs: + if self._normalize_type(struct.type) not in inline_struct_types: + collect_inline_type_owners(struct, struct.type) + inline_enum_types = { self._normalize_type(enum.type) for enum in inline_enums } @@ -478,6 +536,125 @@ def collect_inline_types(struct: DefinedStruct): if struct.template_types: template_struct_lookup[normalized_type] = struct + top_level_structs = [ + struct + for struct in export.structs + if self._normalize_type(struct.type) not in inline_struct_types + ] + top_level_types = { + self._normalize_type(struct.type) for struct in top_level_structs + } + struct_dependencies = {type_name: set() for type_name in top_level_types} + + def references_inline_type(reference: str, inline_type: str) -> bool: + start = reference.find(inline_type) + while start >= 0: + end = start + len(inline_type) + before_is_identifier = start > 0 and ( + reference[start - 1].isalnum() or reference[start - 1] == "_" + ) + after_is_identifier = end < len(reference) and ( + reference[end].isalnum() or reference[end] == "_" + ) + if not before_is_identifier and not after_is_identifier: + return True + start = reference.find(inline_type, start + 1) + return False + + def iter_struct_tree(struct: DefinedStruct): + yield struct + for inline_struct in struct.inline_structs: + yield from iter_struct_tree(inline_struct) + + for struct in top_level_structs: + struct_type = self._normalize_type(struct.type) + references = [] + for nested_struct in iter_struct_tree(struct): + for field in nested_struct.fields: + references.append(field.type) + references.extend(field.template_args or []) + field_type = self._normalize_type(field.type) + if ( + field_type in struct_dependencies + and field_type != struct_type + and not field_type.endswith("*") + ): + struct_dependencies[struct_type].add(field_type) + if field.base: + if field_type in struct_dependencies and field_type != struct_type: + struct_dependencies[struct_type].add(field_type) + if hasattr(field, "return_type"): + references.append(field.return_type or "") + if getattr(field, "parameters", None): + references.extend( + parameter.type for parameter in field.parameters + ) + for virtual_function in nested_struct.virtual_functions or []: + references.append(virtual_function.return_type or "") + references.extend( + parameter.type + for parameter in virtual_function.parameters or [] + ) + + for inline_type, owner_type in inline_type_owners.items(): + if owner_type == struct_type or owner_type not in top_level_types: + continue + if any( + references_inline_type( + self._normalize_type(reference), inline_type + ) + for reference in references + ): + struct_dependencies[struct_type].add(owner_type) + + ordered_structs = [] + remaining_structs = list(top_level_structs) + emitted_types = set() + while remaining_structs: + ready_index = next( + ( + index + for index, struct in enumerate(remaining_structs) + if struct_dependencies[self._normalize_type(struct.type)] + <= emitted_types + ), + None, + ) + if ready_index is None: + ready_index = 0 + struct = remaining_structs.pop(ready_index) + ordered_structs.append(struct) + emitted_types.add(self._normalize_type(struct.type)) + + ordered_type_indices = { + self._normalize_type(struct.type): index + for index, struct in enumerate(ordered_structs) + } + opaque_pointer_types_by_struct: dict[str, set[str]] = {} + for struct in ordered_structs: + struct_type = self._normalize_type(struct.type) + opaque_pointer_types = set() + for nested_struct in iter_struct_tree(struct): + for field in nested_struct.fields: + field_type = self._normalize_type(field.type) + pointer_base = field_type + has_pointer = pointer_base.endswith("*") + while pointer_base.endswith("*"): + pointer_base = pointer_base[:-1].strip() + if not has_pointer: + continue + owner_type = inline_type_owners.get(pointer_base) + if ( + owner_type is not None + and owner_type != struct_type + and owner_type in ordered_type_indices + and ordered_type_indices[owner_type] + > ordered_type_indices[struct_type] + ): + opaque_pointer_types.add(pointer_base) + if opaque_pointer_types: + opaque_pointer_types_by_struct[struct_type] = opaque_pointer_types + def append_chunk(chunk: IndentWriter | str): chunk_text = str(chunk) for line in chunk_text.split("\n"): @@ -564,9 +741,7 @@ def collect_vtable_names(struct: DefinedStruct): for inline_struct in struct.inline_structs: collect_vtable_names(inline_struct) - for struct in export.structs: - if self._normalize_type(struct.type) in inline_struct_types: - continue + for struct in ordered_structs: forward, define, vtable, vtable_full_name = self.build_struct_string( struct, struct_lookup, @@ -574,6 +749,9 @@ def collect_vtable_names(struct: DefinedStruct): template_struct_lookup, full_declspec_align, struct_definition_lookup, + opaque_pointer_types_by_struct.get( + self._normalize_type(struct.type), set() + ), ) if vtable_full_name and vtable_full_name not in seen_vtable_names: seen_vtable_names.add(vtable_full_name) @@ -638,6 +816,7 @@ def build_struct_string( template_struct_lookup: dict[str, DefinedStruct] | None = None, declspec_align: int = 0, struct_definition_lookup: dict[str, DefinedStruct] | None = None, + opaque_pointer_types: set[str] | None = None, ) -> tuple[IndentWriter, IndentWriter, IndentWriter, str | None]: """Builds the C++ struct definition with namespace wrapping. @@ -651,6 +830,7 @@ def build_struct_string( struct_string_define: IndentWriter = IndentWriter() struct_string_vtable: IndentWriter = IndentWriter() template_struct_lookup = template_struct_lookup or {} + opaque_pointer_types = opaque_pointer_types or set() template_declaration = "" template_suffix = "" if struct.template_types: @@ -728,6 +908,7 @@ def append_chunk(chunk: IndentWriter | str): template_struct_lookup, declspec_align, struct_definition_lookup, + opaque_pointer_types, ) for inline_struct in struct.inline_structs ] @@ -743,9 +924,29 @@ def append_chunk(chunk: IndentWriter | str): struct_string_define += f"{short_name}_vtbl{template_suffix}* __vtable;" prev_size = 8 + field_name_counts: dict[str, int] = {} + field_name_last_indices: dict[str, int] = {} + for field_index, field in enumerate(struct.fields): + field_name_counts[field.name] = field_name_counts.get(field.name, 0) + 1 + field_name_last_indices[field.name] = field_index + reserved_field_names = set(field_name_counts) + emitted_field_names = set() + field_names = [] + for field_index, field in enumerate(struct.fields): + field_name = field.name + if ( + field_name_counts[field.name] > 1 + and field_name_last_indices[field.name] != field_index + ): + field_name = f"{field.name}_{field.offset:X}" + while field_name in reserved_field_names or field_name in emitted_field_names: + field_name += "_" + field_names.append(field_name) + emitted_field_names.add(field_name) + offset = 0 - for field in struct.fields: + for field_index, field in enumerate(struct.fields): if field.type in base_fields_used: continue @@ -758,7 +959,7 @@ def append_chunk(chunk: IndentWriter | str): struct_string_define += line prev_size += size - field_name = field.name + field_name = field_names[field_index] field_type = field.type if isinstance(field, DefinedStructFixedField): field_define, field_define_size = self.get_string_define_and_size( @@ -769,6 +970,8 @@ def append_chunk(chunk: IndentWriter | str): template_struct_lookup, struct.template_types, struct_definition_lookup, + field_name, + opaque_pointer_types, ) struct_string_define += f"{field_define[:-1]}[{field.size}];" prev_size += field_define_size * field.size @@ -792,6 +995,8 @@ def append_chunk(chunk: IndentWriter | str): template_struct_lookup, struct.template_types, struct_definition_lookup, + field_name, + opaque_pointer_types, ) struct_string_define += field_define prev_size += field_define_size @@ -813,12 +1018,39 @@ def append_chunk(chunk: IndentWriter | str): struct_string_vtable += f"struct __cppobj {short_name}_vtbl {{" struct_string_vtable.indent() previous_offset = 0 - for vfunc in struct.virtual_functions: + vfunc_name_counts: dict[str, int] = {} + vfunc_preferred_indices: dict[str, int] = {} + for vfunc_index, vfunc in enumerate(struct.virtual_functions): + vfunc_name_counts[vfunc.name] = vfunc_name_counts.get(vfunc.name, 0) + 1 + preferred_index = vfunc_preferred_indices.get(vfunc.name) + if preferred_index is None or ( + vfunc.return_type + and vfunc.parameters + and not ( + struct.virtual_functions[preferred_index].return_type + and struct.virtual_functions[preferred_index].parameters + ) + ): + vfunc_preferred_indices[vfunc.name] = vfunc_index + + emitted_vfunc_names = set() + for vfunc_index, vfunc in enumerate(struct.virtual_functions): while previous_offset < vfunc.offset: struct_string_vtable += f"void* vf{int(previous_offset/8)};" previous_offset += 8 - struct_string_vtable += self.build_vfunc_signature(vfunc) + vfunc_name = vfunc.name + if ( + vfunc_name_counts[vfunc.name] > 1 + and vfunc_preferred_indices[vfunc.name] != vfunc_index + ): + vfunc_name = f"vf{vfunc.offset // 8}" + while vfunc_name in emitted_vfunc_names: + vfunc_name += "_" + emitted_vfunc_names.add(vfunc_name) + struct_string_vtable += self.build_vfunc_signature( + vfunc, vfunc_name + ) previous_offset = vfunc.offset + 8 struct_string_vtable.unindent() struct_string_vtable += "};" From 773f83e17c0f990e99709a181862d8a0719bdea2 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Sun, 11 Oct 2026 15:05:32 +0200 Subject: [PATCH 43/95] prepare for the large importer --- scripts/ida/{exdgetters.py => struct_import.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/ida/{exdgetters.py => struct_import.py} (100%) diff --git a/scripts/ida/exdgetters.py b/scripts/ida/struct_import.py similarity index 100% rename from scripts/ida/exdgetters.py rename to scripts/ida/struct_import.py From 99f447119f8fdae5a54bdaf73eb2f7e06aaeaa61 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Sun, 11 Oct 2026 15:21:41 +0200 Subject: [PATCH 44/95] fix another declaration issue --- .../FFXIV/Common/Component/Excel/ExcelSheet.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/FFXIVClientStructs/FFXIV/Common/Component/Excel/ExcelSheet.cs b/FFXIVClientStructs/FFXIV/Common/Component/Excel/ExcelSheet.cs index e4b097738e..37ef4b2591 100644 --- a/FFXIVClientStructs/FFXIV/Common/Component/Excel/ExcelSheet.cs +++ b/FFXIVClientStructs/FFXIV/Common/Component/Excel/ExcelSheet.cs @@ -74,12 +74,12 @@ public struct LinkedListContainer { public struct WaiterContainer { [FieldOffset(0x00), CExporterBaseType] public LinkList Base; [FieldOffset(0x18)] public int Count; + } - [StructLayout(LayoutKind.Explicit, Size = 0x18)] - public struct WaiterEntry { - [FieldOffset(0x00)] public ExcelSheetWaiter* Waiter; - [FieldOffset(0x08)] public int Index; // ? - [FieldOffset(0x10)] public WaiterEntry* Next; - } + [StructLayout(LayoutKind.Explicit, Size = 0x18)] + public struct WaiterEntry { + [FieldOffset(0x00)] public ExcelSheetWaiter* Waiter; + [FieldOffset(0x08)] public int Index; // ? + [FieldOffset(0x10)] public WaiterEntry* Next; } } From ee3ce271855095512278e5478fb284d0b1fee3aa Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Sun, 11 Oct 2026 15:33:29 +0200 Subject: [PATCH 45/95] fix some definitions --- scripts/ida/ida_wrapper.py | 2 +- scripts/src_wrapper.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/ida/ida_wrapper.py b/scripts/ida/ida_wrapper.py index d8115456d0..8c1b65b90c 100644 --- a/scripts/ida/ida_wrapper.py +++ b/scripts/ida/ida_wrapper.py @@ -180,7 +180,7 @@ def select_parser(self): path = Path(path) vc_path = path / "vc" / "tools" / "msvc" vc_path = first_subfolder(vc_path) / "include" - window_kit = first_subfolder(path / "Windows Kits" / "10" / "Include") + window_kit = first_subfolder(path / "kits" / "10" / "include") winsdk_ucrt_path = window_kit / "ucrt" winsdk_um_path = window_kit / "um" winsdk_shared_path = window_kit / "shared" diff --git a/scripts/src_wrapper.py b/scripts/src_wrapper.py index f52992af31..001bf4e768 100644 --- a/scripts/src_wrapper.py +++ b/scripts/src_wrapper.py @@ -1015,7 +1015,7 @@ def append_chunk(chunk: IndentWriter | str): if struct.virtual_functions: if template_declaration: struct_string_vtable += template_declaration - struct_string_vtable += f"struct __cppobj {short_name}_vtbl {{" + struct_string_vtable += f"struct {short_name}_vtbl {{" struct_string_vtable.indent() previous_offset = 0 vfunc_name_counts: dict[str, int] = {} @@ -1073,7 +1073,7 @@ def get_predef_header(self): #include #include #include -#include +#include #include #include #include From 8cb8719d38698110e0e1102ea9b2683fd69f941a Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Mon, 12 Oct 2026 08:06:52 +0200 Subject: [PATCH 46/95] some changes to cleanup code --- scripts/data_schema.py | 105 ++- scripts/ffxiv_exdgetters.py | 3 +- scripts/ffxiv_structimporter.py | 350 +------ scripts/ida/form_builder.py | 2 +- scripts/ida/ida_wrapper.py | 1508 ++++++------------------------- scripts/ida/struct_import.py | 115 ++- 6 files changed, 451 insertions(+), 1632 deletions(-) diff --git a/scripts/data_schema.py b/scripts/data_schema.py index b0ea1b96b8..4c58ce7d0f 100644 --- a/scripts/data_schema.py +++ b/scripts/data_schema.py @@ -1,34 +1,89 @@ +from dataclasses import dataclass, field, fields +from typing import TypeAlias +from yaml import load + +try: + from yaml import CSafeLoader as Loader +except ImportError: + from yaml import SafeLoader as Loader + + +@dataclass class DefinedDataClassInstance: - def __init__(self, ea, pointer=False, name=None): - # type: (int, bool, str | None) -> None - self.ea = ea - self.pointer = pointer - self.name = name + ea: int + pointer: bool = False + name: str | None = None + +@dataclass class DefinedDataClassVtable: - def __init__(self, ea, base=None, size=None): - # type: (int, None | str, None | int) -> None - self.ea = ea - self.base = base - self.size = size + ea: int + base: str | None = None + size: int | None = None + +@dataclass class DefinedDataClassFunction: - def __init__(self, num, name): - # type: (int, str) -> None - self.num = num - self.name = name + num: int + name: str + +@dataclass class DefinedDataClass: - def __init__(self, name, instances=[], vtbls=[], functions=[], vfuncs=[]): - # type: (str, list[DefinedDataClassInstance], list[DefinedDataClassVtable], list[DefinedDataClassFunction], list[DefinedDataClassFunction]) -> None - self.instances = instances - self.vtbls = vtbls - self.functions = functions - self.vfuncs = vfuncs - self.name = name + instances: list[DefinedDataClassInstance] = field(default_factory=list) + vtbls: list[DefinedDataClassVtable] = field(default_factory=list) + functions: list[DefinedDataClassFunction] = field( + default_factory=list, metadata={"yaml_name": "funcs"} + ) + vfuncs: list[DefinedDataClassFunction] = field(default_factory=list) + +@dataclass class DefinedData: - def __init__(self, classes=[]): - # type: (list[DefinedDataClass]) -> None - self.classes = classes - pass \ No newline at end of file + classes: dict[str, DefinedDataClass] = field(default_factory=dict) + version: str | None = None + globals: dict[int, str] = field(default_factory=dict) + + +YamlExport: TypeAlias = dict[str, object] + + +def get_yaml(stream, loader=Loader) -> DefinedData: + dic: YamlExport = load(stream, loader) + classes = {} + for name, class_values in dic.get("classes", {}).items(): + class_values = class_values or {} + values = dict(class_values) + + for key, value_type in ( + ("instances", DefinedDataClassInstance), + ("vtbls", DefinedDataClassVtable), + ): + if key in class_values and class_values[key] is not None: + values[key] = [ + _from_dict(value_type, value) for value in class_values[key] + ] + else: + values.pop(key, None) + + for yaml_key in ("funcs", "vfuncs"): + if yaml_key in class_values and class_values[yaml_key] is not None: + values[yaml_key] = [ + DefinedDataClassFunction(num, function_name) + for num, function_name in class_values[yaml_key].items() + ] + else: + values.pop(yaml_key, None) + + classes[name] = _from_dict(DefinedDataClass, values) + + return _from_dict(DefinedData, {**dic, "classes": classes}) + + +def _from_dict(data_class, values): + class_values = {} + for data_field in fields(data_class): + key = data_field.metadata.get("yaml_name", data_field.name) + if key in values: + class_values[data_field.name] = values[key] + return data_class(**class_values) diff --git a/scripts/ffxiv_exdgetters.py b/scripts/ffxiv_exdgetters.py index a3896a18a8..85e2c50c06 100644 --- a/scripts/ffxiv_exdgetters.py +++ b/scripts/ffxiv_exdgetters.py @@ -42,8 +42,7 @@ def process_pattern(self, pattern): except ImportError as e: print("Warning: Unable to load IDA failed with {0}".format(e)) else: - print("Script has moved to new location inside of ida folder.") - exit(1) + raise ValueError("Script moved to new location in ./ida/struct_import.py") if api is None: try: diff --git a/scripts/ffxiv_structimporter.py b/scripts/ffxiv_structimporter.py index 89edeca06f..59904a664e 100644 --- a/scripts/ffxiv_structimporter.py +++ b/scripts/ffxiv_structimporter.py @@ -160,358 +160,10 @@ def load_data_yaml(self): if api is None: try: import idaapi - import idc - import ida_bytes - import ida_search - import ida_typeinf - import ida_funcs - import ida_name - import ida_kernwin - import ida_srclang - import hashlib - import copy - from ida.ida_wrapper import IdaInterface except ImportError: print("Warning: Unable to load IDA") else: - # noinspection PyUnresolvedReferences - class IdaApi(BaseApi, IdaInterface): - def validate_name_cfg(self): - """Verifies that the user's IDA config allows template characters for type names""" - temporary_name = "struc_name_test" - template_name = "OuterStructTest*>" - - def delete_test_type(name): - sid = self.get_struct_id(name) - if sid == idaapi.BADADDR: - return - if idaapi.IDA_SDK_VERSION >= 900: - ida_typeinf.del_named_type(idaapi.get_idati(), name, ida_typeinf.NTF_TYPE) - else: - idc.del_struc(sid) - - created_name = None - try: - template_sid = self.get_struct_id(template_name) - temporary_sid = self.get_struct_id(temporary_name) - - if template_sid != idaapi.BADADDR: - if temporary_sid != idaapi.BADADDR: - return True - - self.rename_struct(template_sid, temporary_name) - if self.get_struct_id(template_name) != idaapi.BADADDR: - raise RuntimeError("could not rename existing template test type") - - temporary_sid = self.get_struct_id(temporary_name) - - if temporary_sid == idaapi.BADADDR: - temporary_sid = self.create_struct_type(temporary_name) - - if temporary_sid == idaapi.BADADDR: - raise RuntimeError("could not create temporary struct") - - created_name = temporary_name - - self.rename_struct(temporary_sid, template_name) - if self.get_struct_id(template_name) == idaapi.BADADDR: - raise RuntimeError("could not rename temporary struct to template type name") - - created_name = template_name - except Exception as exc: - return False # dropping exception, but will prompt the user to fix their cfg and exit - - finally: - if created_name is not None: - delete_test_type(created_name) - delete_test_type(temporary_name) - - return True - - def get_fallback_vfunc_name(self, class_name, index, visited=None): - # type: (str, int, set) -> str - if not hasattr(self, "data_yaml") or not self.data_yaml or "classes" not in self.data_yaml: - return None - - if visited is None: - visited = set() - - if class_name in visited: - return None - visited.add(class_name) - - if class_name not in self.data_yaml["classes"]: - return None - - class_data = self.data_yaml["classes"][class_name] - if not class_data: - return None - - if "vfuncs" in class_data and index in class_data["vfuncs"]: - return class_data["vfuncs"][index] - - if "vtbls" in class_data and isinstance(class_data["vtbls"], list) and len(class_data["vtbls"]) > 0: - vtbl = class_data["vtbls"][0] - if "base" in vtbl: - res = self.get_fallback_vfunc_name(vtbl["base"], index, visited) - if res: - return res - - return None - - def get_vft_from_data(self, class_name): - # type: (str) -> dict | None - if not self.data_yaml or "classes" not in self.data_yaml: - return None - - class_data = self.data_yaml["classes"].get(class_name) or {} - vtables = class_data.get("vtbls", []) - if not isinstance(vtables, list) or not vtables: - return None - - vtable = vtables[0] - return vtable if isinstance(vtable, dict) else None - - def get_vfunc_ea_from_data(self, class_name, offset): - # type: (str, int) -> int - vtable = self.get_vft_from_data(class_name) - if vtable is None or "ea" not in vtable: - return idc.BADADDR - - # assuming data.yml will continue using default windows base - vtable_ea = vtable["ea"] + idaapi.get_imagebase() - 0x140000000 - return idc.get_qword(vtable_ea + offset) - - def delete_struct_members(self, fullname): - # type: (str) -> None - pass - - @property - def get_file_path(self): - return os.path.join( - os.path.dirname(os.path.realpath(__file__)), "ffxiv_structs.yml" - ) - - def generate_hashed_type_name(self, name: str) -> str: - return "" - - def get_srclang_type_name(self, name: str) -> str: - return "" - - def can_run(self): - return self.enum_exists("Component::Exd::SheetsEnum") - - def create_enum_struct(self, enum): - # type: (DefinedStructEnum) -> None - return - - def delete_enum(self, enum): - # type: (DefinedStructEnum) -> None - return - - def delete_struct(self, struct): - # type: (DefinedStruct) -> None - return - - def create_struct(self, struct): - # type: (DefinedStruct) -> None - return - - def validate_srclang_struct(self, struct: DefinedStruct): - pass - - def get_srclang_fill_type(self, available_bytes: int, current_offset: int) -> tuple[str, int]: - return ("", 0) - - def append_srclang_padding(self, decl: list[str], current_size: int, target_size: int) -> int: - return 0 - - def create_srclang_decl(self, struct: DefinedStruct) -> str: - return "" - - def create_struct_member_fill(self, struct_name, offset): - # type: (str, int) -> None - pass - - def create_struct_members(self, struct): - # type: (DefinedStruct) -> None - pass - - def create_vtable(self, struct): - # type: (DefinedStruct) -> None - pass - - def finalise_struct(self, struct: DefinedStruct): - pass - - def create_union(self, struct): - # type: (DefinedStruct) -> None - pass - - def update_member_func(self, member_func, struct): - # type: (DefinedStructMemFunc, DefinedStruct) -> None - func_name = "{0}.{1}".format( - self.clean_name(struct.type), member_func.name - ) - ea = self.get_func_ea_by_name(func_name) - if ea == idc.BADADDR: - ea = self.get_func_ea_by_sig(member_func.signature) - if ea == idc.BADADDR: - print( - "Error: {0} not found bad sig? {1}".format( - func_name, member_func.signature - ) - ) - return - if ida_funcs.get_func_name(ea) == "sub_{0:X}".format(ea): - idc.set_name(ea, func_name) - tif = ida_typeinf.tinfo_t() - ida_typeinf.guess_tinfo(tif, ea) - func_data = ida_typeinf.func_type_data_t() - tif.get_func_details(func_data) - func_data.clear() - func_data.cc = ida_typeinf.CM_CC_FASTCALL - func_data.rettype = self.get_tinfo_from_type(member_func.return_type) - for param in member_func.parameters: - arg = ida_typeinf.funcarg_t() - try: - arg.type = self.get_tinfo_from_type(param.type) - except ValueError as exc: - print( - "Error: update_member_func: function={!r}, ea={:#x}, " - "parameter={!r}, type={!r}, error={}".format( - func_name, ea, - param.name, param.type, exc - ) - ) - raise - arg.name = param.name - func_data.push_back(arg) - tif.create_func(func_data) - ida_typeinf.apply_tinfo(ea, tif, ida_typeinf.TINFO_DEFINITE) - - def update_virt_func(self, virt_func, struct): - # type: (DefinedStructVFunc, DefinedStruct) -> None - func_name = "{0}.{1}".format( - self.clean_name(struct.type), virt_func.name - ) - - ea = self.get_func_ea_by_name(func_name) - if ea == idc.BADADDR: - ea = self.get_vfunc_ea_from_data(struct.type, virt_func.offset) - if ea in (0, idc.BADADDR): - print(f"Error: {func_name} not found and its VFT slot could not be resolved") - return - - actual_name = ida_funcs.get_func_name(ea) or "" - normalized_name = actual_name - for prefix in ("j_", "thunk_"): - if normalized_name.startswith(prefix): - normalized_name = normalized_name[len(prefix):] - - # pure virtual, nothing to do - if normalized_name == "_purecall": - return - - primary_vtable = self.get_vft_from_data(struct.type) - primary_base = primary_vtable.get("base") if primary_vtable else None - if primary_base: - base_ea = self.get_vfunc_ea_from_data(primary_base, virt_func.offset) - # if the derived vft slot points to the same as the baseclass, leave baseclass type - if base_ea == ea: - return - - # does it look inherited based on name? leave it alone - if ( - normalized_name.endswith("." + virt_func.name) - and not normalized_name.startswith(self.clean_name(struct.type) + ".") - ): - return - - tif = ida_typeinf.tinfo_t() - ida_typeinf.guess_tinfo(tif, ea) - - func_data = ida_typeinf.func_type_data_t() - tif.get_func_details(func_data) - - func_data.clear() - func_data.cc = ida_typeinf.CM_CC_FASTCALL - func_data.rettype = self.get_tinfo_from_type(virt_func.return_type) - - for param in virt_func.parameters: - arg = ida_typeinf.funcarg_t() - try: - arg.type = self.get_tinfo_from_type(param.type) - except ValueError as exc: - print( - "Error: update_virt_func: function={!r}, ea={:#x}, " - "parameter={!r}, type={!r}, error={}".format( - func_name, ea, param.name, param.type, exc - ) - ) - raise - - arg.name = param.name - func_data.push_back(arg) - - tif.create_func(func_data) - ida_typeinf.apply_tinfo(ea, tif, ida_typeinf.TINFO_DEFINITE) - - def update_static_member(self, static_member, struct): - # type: (DefinedStructStaticMember, DefinedStruct) -> None - ea = self.search_binary( - 0, static_member.signature, ida_search.SEARCH_DOWN - ) - if ea == idc.BADADDR: - print( - "Error: {0} not found something is wrong".format( - static_member.signature - ) - ) - return - for follows in static_member.relative_offsets: - ea = ea + follows - ea = ea + 4 + self.get_dword(ea) - tif = ida_typeinf.tinfo_t() - ida_typeinf.guess_tinfo(tif, ea) - return_type = static_member.return_type - if static_member.is_pointer: - return_type = return_type + "*" - ida_typeinf.apply_tinfo( - ea, - self.get_tinfo_from_type(return_type), - ida_typeinf.TINFO_DEFINITE, - ) - ida_name.set_name( - ea, - "g_{0}_{1}".format(self.clean_name(struct.type), static_member.name), - ) - - def should_update_member_func(self): - return ( - ida_kernwin.ask_yn( - ida_kernwin.ASKBTN_YES, "Update member function types?" - ) - == ida_kernwin.ASKBTN_YES - ) - - def should_update_virt_func(self): - return ( - ida_kernwin.ask_yn( - ida_kernwin.ASKBTN_YES, "Update virtual function types?" - ) - == ida_kernwin.ASKBTN_YES - ) - - api = IdaApi() - if not api.validate_name_cfg(): - ida_kernwin.warning( - "Type name validation failed.\n" - "\n" - "Your IDA.cfg is missing necessary NameChars and TypeNameChars.\n" - "\n" - "Please see https://github.com/aers/FFXIVClientStructs/blob/main/ida/idauser.cfg to update your config accordingly.") - exit() + raise ("Script moved to new location in ./ida/struct_import.py") if api is None: try: diff --git a/scripts/ida/form_builder.py b/scripts/ida/form_builder.py index 4b1a091758..cce031b70e 100644 --- a/scripts/ida/form_builder.py +++ b/scripts/ida/form_builder.py @@ -54,7 +54,7 @@ def build_spec(fields, change_cb): names.append(name) last = (i == len(items) - 1) if last: - end = f">{{{key}}}>" if t == "checks" else f">>{{{key}}}>" + end = f">{{{key}}}>" if t == "checks" else f">{{{key}}}>" else: end = ">" lines.append(f"{prepend_line}<{ilabel}:{{{name}}}{end}") diff --git a/scripts/ida/ida_wrapper.py b/scripts/ida/ida_wrapper.py index 8c1b65b90c..8342e2ff3e 100644 --- a/scripts/ida/ida_wrapper.py +++ b/scripts/ida/ida_wrapper.py @@ -8,13 +8,16 @@ import ida_name import ida_funcs import ida_srclang +import ida_kernwin from structs_schema import * from form_builder import DynamicForm +from data_schema import get_yaml, DefinedData from abc import abstractmethod from platform import system from pathlib import Path -from webbrowser import open +from webbrowser import open as webopen +import os def pick_folder(form_content: list[dict[str, any]]) -> str | None: f = DynamicForm(form_content) @@ -30,7 +33,7 @@ def pick_folder(form_content: list[dict[str, any]]) -> str | None: { "type": "label", "text": "Select msvc"}, { "type": "space" }, { "type": "label", "text": "It should be located where you used:", "prepend": True}, - { "type": "button", "key": "msvcdownload", "label": "mstorsjo/msvc-wine", "on_click": lambda: open("https://github.com/mstorsjo/msvc-wine")}, + { "type": "button", "key": "msvcdownload", "label": "mstorsjo/msvc-wine", "on_click": lambda: webopen("https://github.com/mstorsjo/msvc-wine")}, { "type": "space" }, { "type": "folder", "key": "folder", "label": "Folder", "value": "", "tip": "Pick the folder where ./vsdownload.sh was run" } ] @@ -41,65 +44,52 @@ def first_subfolder(base: Path) -> Path | None: return p return None -# For more information about TERR_ constants, see: -# https://docs.hex-rays.com/developer-guide/idapython/idapython-porting-guide-ida-9#type-information-error-codes - - def _get_ida_major() -> int: return idaapi.IDA_SDK_VERSION // 100 -class BaseIdaInterface(object): - @abstractmethod - def get_struct_id(self, name): - pass +if _get_ida_major() < 9: + raise "Unsupported IDA version used. Need to be on IDA 9 or newer." - @abstractmethod - def get_enum_id(self, name): - pass +class IdaInterface(object): + data_yaml: DefinedData - @abstractmethod - def delete_enum_members(self, eid: int): - """Remove all enum members + def get_base_path(self): + return os.path.join( + os.path.dirname(os.path.realpath(__file__)), ".." + ) - Args: - eid (int): The id of the enum - """ - pass + def get_data_path(self): + return os.path.join(self.get_base_path(), "data.yml") + + def load_data_yml(self): + with open(self.get_data_path()) as f: + self.data_yaml = get_yaml(f) - def enum_exists(self, name: str): - return self.get_enum_id(name) != idaapi.BADADDR + def get_struct_path(self): + return os.path.join(self.get_base_path(), "ffxiv_structs.yml") - def clean_name(self, name: str): - """Clean a name + def get_enum_id(self, name: str): + """Get the id of an enum by its name Args: - name (str): The name + name (str): The name of the enum Returns: - str: The cleaned name + int: The id of the enum """ - return name - def clean_struct_name(self, name: str): - """Clean a struct name + return idc.get_enum(name) + + def get_struct_id(self, name: str) -> int: + """Get the struct id Args: - name (str): The struct name + name (str): The name of the struct Returns: - str: The cleaned struct name + int: The struct id """ - - if name == "Tm": - return "tm" # tm is a keyword in IDA for the time struct but C# exports it as Tm - return ( - name.replace(" ", "") - .replace("unsigned", "u") - .replace("__int64", "long") - .replace("__int32", "int") - .replace("__int16", "short") - .replace("__int8", "char") - ) + return ida_typeinf.get_named_type_tid(name) def get_named_type(self, name: str): """Retrieve a tinfo_t from the named type. @@ -109,13 +99,13 @@ def get_named_type(self, name: str): """ tinfo = ida_typeinf.tinfo_t() - clean_name = self.clean_struct_name(name) + clean_name = name if ( self.get_struct_id(clean_name) != idaapi.BADADDR or self.get_enum_id(clean_name) != idaapi.BADADDR ): if not tinfo.get_named_type(idaapi.get_idati(), clean_name): - raise ValueError("{0} not found in IDA database".format(clean_name)) + raise ValueError(f"{clean_name} not found in IDA database") return tinfo if name == "void": @@ -154,7 +144,12 @@ def get_tinfo_from_type(self, raw_type: str, array_size=0): for i in range(ptr_count): ptr_tinfo = idaapi.tinfo_t() if not ptr_tinfo.create_ptr(type_tinfo): - print("! failed to create pointer") + print( + f"! failed to create pointer for {raw_type!r}: " + f"base type {type!r} resolved to {type_tinfo.dstr()!r} " + f"(correct={type_tinfo.is_correct()}, " + f"forward={type_tinfo.is_forward_decl()})" + ) return None type_tinfo = ptr_tinfo else: @@ -188,7 +183,7 @@ def select_parser(self): ida_srclang.set_parser_argv( "clang", - f"-x c++ -target x86_64-pc-windows-msvc -fms-compatibility -fms-extensions -fdelayed-template-parsing -U__int64 -std=c++17{includes}", + f"-x c++ -target x86_64-pc-windows-msvc -fms-compatibility -fms-extensions -U__int64 -std=c++17{includes}", ) def parse_string(self, decl: str, vtables: list[str]): @@ -196,1264 +191,279 @@ def parse_string(self, decl: str, vtables: list[str]): ida_srclang.parse_decls_with_parser("clang", None, decl, False) for vtable in vtables: self.mark_as_vtable(vtable) - - def mark_as_vtable(self, vtbl_name: str) -> str | None: - """ - Mark a struct as a vtable (TAUDT_VFTABLE). - Returns: - An error string on failure, None on success. + def op_type_as_name(self, n: int): """ - tif = ida_typeinf.tinfo_t() - - if not tif.get_named_type(None, vtbl_name): - return f"Type '{vtbl_name}' not found in type library" - - if not tif.is_struct(): - actual = ( - "union" if tif.is_union() else "enum" if tif.is_enum() else "unknown" - ) - return f"'{vtbl_name}' is not a struct (got {actual})" - - udt = ida_typeinf.udt_type_data_t() - if not tif.get_udt_details(udt): - return f"Failed to retrieve UDT details for '{vtbl_name}'" - - if udt.taudt_bits & ida_typeinf.TAUDT_VFTABLE: - return f"'{vtbl_name}' is already marked as a vtable" - - udt.taudt_bits |= ida_typeinf.TAUDT_VFTABLE - - if not tif.create_udt(udt, ida_typeinf.BTF_STRUCT): - return ( - f"Failed to reconstruct UDT for '{vtbl_name}' after setting vtable flag" - ) - - if _get_ida_major() >= 9: - result = tif.set_named_type(None, vtbl_name) - else: - result = tif.set_named_type( - ida_typeinf.get_idati(), vtbl_name, ida_typeinf.NTF_REPLACE - ) - - if not result: - return f"Failed to write back '{vtbl_name}' to type library" - - return None - - def get_idc_type_from_ida_type(self, type: str) -> int: - """Retrieve the idc type from the ida type. + Retrieve the name of the operand type constant. Args: - type (str): Name of the type. + n (int): The integer value of the operand type. + + Returns: + str: The name of the operand type constant if found; otherwise, None. """ - if ( - type == "unsigned byte" - or type == "unsigned char" - or type == "unsigned __int8" - or type == "ubyte" - or type == "byte" - or type == "bool" - or type == "char" - or type == "uchar" - or type == "__int8" - ): - return ida_bytes.byte_flag() - elif ( - type == "unsigned short" - or type == "ushort" - or type == "unsigned __int16" - or type == "short" - or type == "__int16" - or type == "_WORD" - or type == "wchar_t" - ): - return ida_bytes.word_flag() - elif ( - type == "unsigned int" - or type == "unsigned __int32" - or type == "uint" - or type == "int" - or type == "__int32" - or type == "_DWORD" - ): - return ida_bytes.dword_flag() - elif ( - type == "unsigned long" - or type == "unsigned __int64" - or type == "ulong" - or type == "long" - or type == "__int64" - or type == "__fastcall" - or type.endswith("*") - ): - return ida_bytes.qword_flag() - elif type == "float": - return ida_bytes.float_flag() - elif type == "double": - return ida_bytes.double_flag() - elif self.get_enum_id(type) != idaapi.BADADDR: - return ida_bytes.enum_flag() - elif self.get_struct_id(type) != idaapi.BADADDR: - return ida_bytes.stru_flag() - else: - return ida_bytes.stru_flag() - - def get_idc_type_from_size(self, size: int, offset=0) -> int: - if offset == 0: - offset = size - if offset % 8 == 0 and size >= 8: - return ida_bytes.qword_flag() - elif offset % 4 == 0 and size >= 4: - return ida_bytes.dword_flag() - elif offset % 2 == 0 and size >= 2: - return ida_bytes.word_flag() - else: - return ida_bytes.byte_flag() - - def get_size_from_idc_type(self, type: int) -> int: - if type == ida_bytes.byte_flag(): - return 1 - elif type == ida_bytes.word_flag(): - return 2 - elif type == ida_bytes.dword_flag(): - return 4 - elif type == ida_bytes.qword_flag(): - return 8 - elif type == ida_bytes.float_flag(): - return 4 - elif type == ida_bytes.double_flag(): - return 8 - else: - return 0 - def get_size_from_ida_type(self, type: str) -> int: - return self.get_size_from_idc_type(self.get_idc_type_from_ida_type(type)) + for item in [x for x in dir(idc) if x.startswith("o_")]: + if getattr(idc, item) == n: + return item + + def get_vft_from_data(self, class_name: str) -> dict | None: + class_data = self.data_yaml.classes.get(class_name) or {} + vtables = class_data.get("vtbls", []) + if not isinstance(vtables, list) or not vtables: + return None + + vtable = vtables[0] + return vtable if isinstance(vtable, dict) else None + + def get_vfunc_ea_from_data(self, class_name: str, offset: int) -> int: + vtable = self.get_vft_from_data(class_name) + if vtable is None or "ea" not in vtable: + return idc.BADADDR + + vtable_ea = vtable["ea"] + idaapi.get_imagebase() - 0x140000000 + return idc.get_qword(vtable_ea + offset) def set_func_name(self, ea, name, cmt): ida_name.set_name(ea, name) ida_bytes.set_cmt(ea, cmt, 0) + def search_binary(self, ea: int, pattern: str, flag: int): + """Search for a pattern in a binary - def build_struct_string(self, struct: DefinedStruct, struct_lookup: dict[str, int], enum_lookup: dict[str, str]) -> tuple[str, str, list[str]]: - """Builds the srclang definition required to parse the object. - Args: - struct (DefinedStruct): The struct data to build""" - struct_string_forward: str = "" - struct_string_define: str = "" - struct_string_vtables: list[str] = [] - struct_name = struct.type - - struct_string_forward += f"struct {struct_name};\n" - if (struct.virtual_functions): - struct_string_forward += f"struct {struct_name}_vtbl;\n" - struct_string_vtables.append(f"{struct_name}_vtbl") - - struct_string_define += f"struct {struct_name}" - - prev_size = 0 - contiguous_fields = True - for field in struct.fields: - offset = field.offset - - while offset > prev_size: - contiguous_fields = False - fill_size = offset - prev_size - if self.full_padding: - flag = self.get_idc_type_from_size(prev_size) - size = self.get_size_from_idc_type(flag) - if size > fill_size: - flag = self.get_idc_type_from_size( - fill_size, prev_size - ) - size = self.get_size_from_idc_type(flag) - - struct_string_define += f" {self.get_string_from_idc_type_and_sign(flag)} field_{prev_size:X};" - else: - struct_string_define += f" char field_{prev_size:X}[{fill_size}];" - - prev_size += size - - field_is_base = field.base and contiguous_fields - field_name = ( - field.name - if not field_is_base - else "baseclass_{0:X}".format(offset) - ) - field_type = field.type - if field_type == "__fastcall": - struct_string_define += f" {field.return_type} ({field_type} *{field_name})(" - for param in field.parameters: - struct_string_define += param.type + " " + param.name + ", " - struct_string_define = struct_string_define[:-2] + ");" - prev_size += 8 - elif (field_type in struct_lookup): - struct_string_define += f" {field_type} {field_name};" - prev_size += struct_lookup[field_type] - elif (field_type in enum_lookup): - struct_string_define += f" {field_type} {field_name};" - prev_size += self.get_size_from_ida_type(enum_lookup[field_type]) - else: - struct_string_define += f" {field_type} {field_name};" - - if struct.size is not None and struct.size != 0: - while struct.size > prev_size: - fill_size = struct.size - prev_size - if self.full_padding: - flag = self.get_idc_type_from_size(prev_size) - size = self.get_size_from_idc_type(flag) - if size > fill_size: - flag = self.get_idc_type_from_size( - fill_size, prev_size - ) - size = self.get_size_from_idc_type(flag) - - struct_string_define += f" {self.get_string_from_idc_type_and_sign(flag)} field_{prev_size:X};" - else: - struct_string_define += f" char field_{prev_size:X}[{fill_size}];" - - prev_size += size - - return (struct_string_forward, struct_string_define, struct_string_vtables) - - def rename_struct(self, sid: int, new_name: str): - """Rename a struct in IDA. + ea (int): The address to start searching from + pattern (str): The pattern to search for + flag (int): The search flags - Args: - sid (int): The struct ID. - new_name (str): The new struct name. + Returns: + int: The start address of the pattern """ - idc.set_struc_name(sid, new_name) - - -if _get_ida_major() < 9: - import ida_struct # pyright: ignore[reportMissingImports] - import ida_enum # pyright: ignore[reportMissingImports] -else: - print("Using IDA 9+ API") - - -class IdaInterface(BaseIdaInterface): - # This is only for IDA 7 and 8 due to a change in the API for IDA 9 - if 7 <= _get_ida_major() < 9: - - def get_tinfo_from_func_data(self, data: DefinedStructFuncField): - """Retrieve a tinfo_t from a raw function data. - - Args: - data (DefinedStructFuncField): Function data. - - Returns: - idaapi.tinfo_t: tinfo_t created from the function data. - """ - tinfo = ida_typeinf.tinfo_t() - func_data = ida_typeinf.func_type_data_t() - func_data.cc = ida_typeinf.CM_CC_FASTCALL - func_data.rettype = self.get_tinfo_from_type(data.return_type) - for param in data.parameters: - arg = ida_typeinf.funcarg_t() - arg.type = self.get_tinfo_from_type(param.type) - arg.name = param.name - func_data.push_back(arg) - - tinfo.create_func(func_data) - return tinfo - - def get_struct_opinfo_from_type(self, raw_type: str): - """Retrieve an opinfo_t from a raw structure type string. - - Args: - raw_type (str): Raw structure type string. - - Returns: - ida_nalt.opinfo_t: opinfo_t created from the structure type string. - """ - opinf = ida_nalt.opinfo_t() - opinf.tid = ida_struct.get_struc_id(raw_type) - return opinf - - def get_enum_opinfo_from_type(self, raw_type: str): - """Retrieve an opinfo_t from a raw enum type string. - - Args: - raw_type (str): Raw enum type string. - - Returns: - ida_nalt.opinfo_t: opinfo_t created from the enum type string. - """ - opinf = ida_nalt.opinfo_t() - opinf.ec.tid = ida_enum.get_enum(raw_type) - return opinf - - def search_binary(self, ea: int, pattern: str, flag: int): - """Search for a pattern in a binary - - Args: - ea (int): The address to start searching from - pattern (str): The pattern to search for - flag (int): The search flags - - Returns: - int: The start address of the pattern - """ - return ida_search.find_binary( - ea, - flag & 1 and ida_ida.cvar.inf.max_ea or ida_ida.cvar.inf.min_ea, - pattern, - 16, - flag, - ) - - def get_dword(self, ea: int): - """Retrieve a dword (32-bit value) from the specified address. - - Args: - ea (int): The effective address to read the dword from. - - Returns: - int: The dword value read from the given address. - """ - return ida_bytes.get_original_dword(ea) - - def get_func_ea_by_name(self, name: str): - """Retrieve the effective address of a function by its name. + return ida_bytes.find_bytes( + pattern, ea, None, ida_ida.inf_get_max_ea(), None, flag + ) - Args: - name (str): The name of the function. + def get_dword(self, ea: int): + """Retrieve a dword (32-bit value) from the specified address. - Returns: - int: The effective address of the function. - """ - return ida_name.get_name_ea(0, name) + Args: + ea (int): The effective address to read the dword from. - def get_func_ea_by_sig(self, pattern: str): - """Retrieve the effective address of a function by its signature. + Returns: + int: The dword value read from the given address. + """ + return ida_bytes.get_original_dword(ea) - Args: - pattern (str): The signature of the function. + def get_func_ea_by_name(self, name: str): + """Retrieve the effective address of a function by its name. - Returns: - int: The effective address of the function. - """ - ea = self.search_binary(0, pattern, ida_search.SEARCH_DOWN) + Args: + name (str): The name of the function. - if ida_funcs.get_func(ea) is None: - finf = ida_funcs.func_t() - finf.start_ea = ea - finf.end_ea = idc.BADADDR - ida_funcs.add_func_ex(finf) + Returns: + int: The effective address of the function. + """ + return ida_name.get_name_ea(0, name) - if ida_funcs.get_func(ea) is None: - return idc.BADADDR + def get_func_ea_by_sig(self, pattern: str): + """Retrieve the effective address of a function by its signature. - if ida_funcs.get_func(ea).start_ea == ea: - return ea - mnem = idc.print_insn_mnem(ea) - if not mnem: - return idc.BADADDR + Args: + pattern (str): The signature of the function. - opType0 = idc.get_operand_type(ea, 0) - if mnem == "jmp" or mnem == "call" or mnem[0] == "j": - if opType0 != idc.o_near and opType0 != idc.o_mem: - print( - "Error: Can't follow opType0 {0}".format( - self.opTypeAsName(opType0) - ) - ) - return idc.BADADDR - return idc.get_operand_value(ea, 0) + Returns: + int: The effective address of the function. + """ + ea = self.search_binary(0, pattern, ida_search.SEARCH_DOWN) - if idc.next_head(ea) == ea + idc.get_item_size(ea) and idc.is_flow( - idc.get_full_flags(idc.next_head(ea)) - ): - return idc.next_head(ea) + if ida_funcs.get_func(ea) is None: + finf = ida_funcs.func_t() + finf.start_ea = ea + finf.end_ea = idc.BADADDR + ida_funcs.add_func_ex(finf) + if ida_funcs.get_func(ea) is None: return idc.BADADDR - def opTypeAsName(self, n: int): - """ - Retrieve the name of the operand type constant. - - Args: - n (int): The integer value of the operand type. - - Returns: - str: The name of the operand type constant if found; otherwise, None. - """ - - for item in [x for x in dir(idc) if x.startswith("o_")]: - if getattr(idc, item) == n: - return item - - def create_struct_type(self, name: str, union: bool = False) -> int: - """Create a struct type - - Args: - name (str): The name of the struct - union (bool, optional): Whether the struct is a union. Defaults to False. - - Returns: - int: The struct id - """ - return ida_struct.add_struc(idc.BADADDR, name, union) - - def get_struct_id(self, name: str) -> int: - """Get the struct id - - Args: - name (str): The name of the struct - - Returns: - int: The struct id - """ - return ida_struct.get_struc_id(name) - - def get_struct(self, sid: int) -> ida_struct.struc_t: - """Get the struct - - Args: - sid (int): The struct id - - Returns: - ida_struct.struc_t: The struct - """ - return ida_struct.get_struc(sid) - - def get_struct_size(self, sid: ida_struct.struc_t) -> int: - """Get the struct size - - Args: - sid (ida_struct.struc_t): The struct - - Returns: - int: The struct size - """ - return ida_struct.get_struc_size(sid) + if ida_funcs.get_func(ea).start_ea == ea: + return ea + mnem = idc.print_insn_mnem(ea) + if not mnem: + return idc.BADADDR - def create_struct_member( - self, - sid: ida_struct.struc_t, - name: str, - offset: int = -1, - flag: int = idc.FF_DATA | idc.FF_QWORD, - typeid: int | None = None, - nbytes: int = 8, - ): - """Create a struct member - - Args: - sid (ida_struct.struc_t): The struct to add the member to - name (str): The name of the member - offset (int, optional): The offset of the member. Defaults to -1. - flag (int, optional): The type flag of the member. Defaults to idc.FF_DATA | idc.FF_QWORD. - typeid (int | None, optional): The type id of the member. Defaults to None. - nbytes (int, optional): The number of bytes of the member. Defaults to 8. - - Returns: - struc_error_t: Unknown IDA documentation for this - """ - return ida_struct.add_struc_member(sid, name, offset, flag, typeid, nbytes) - - def remove_struct_member(self, sid: int, name: str) -> bool: - """Remove a struct member - - Args: - sid (int): The struct id - name (str): The name of the member - - Returns: - bool: True if successful - """ - return ida_struct.del_struc_member(sid, name) - - def remove_struct_members(self, sid: int) -> int: - """Remove all struct members - - Args: - sid (int): The struct id - - Returns: - int: The number of members removed or -1 if failed - """ - return ida_struct.del_struc_members( - ida_struct.get_struc(sid), - 0, - ida_struct.get_struc_size(sid) + 1, - ) + opType0 = idc.get_operand_type(ea, 0) + if mnem == "jmp" or mnem == "call" or mnem[0] == "j": + if opType0 != idc.o_near and opType0 != idc.o_mem: + print(f"Error: Can't follow opType0 {self.op_type_as_name(opType0)}") + return idc.BADADDR + return idc.get_operand_value(ea, 0) - def get_struct_member( - self, sid: ida_struct.struc_t, offset: int - ) -> ida_struct.member_t: - """Get a struct member - - Args: - sid (ida_struct.struc_t): The struct - offset (int): The offset of the member - - Returns: - ida_struct.member_t: The member data - """ - return ida_struct.get_member(sid, offset) - - def get_struct_member_by_name( - self, sid: ida_struct.struc_t, name: str - ) -> ida_struct.member_t: - """Get a struct member - - Args: - sid (ida_struct.struc_t): The struct - name (str): The name of the member - - Returns: - ida_struct.member_t: The member data - """ - return ida_struct.get_member_by_name(sid, name) - - def set_struct_member_info( - self, - sid: ida_struct.struc_t, - member: ida_struct.member_t, - offset: int, - tif: ida_typeinf.tinfo_t, - flag: int = 0, - is_string: bool = False, + if idc.next_head(ea) == ea + idc.get_item_size(ea) and idc.is_flow( + idc.get_full_flags(idc.next_head(ea)) ): - """Set the info of a struct member - - Args: - sid (ida_struct.struc_t): The struct - member (ida_struct.member_t): The member - offset (int): The offset in the member - tif (ida_typeinf.tinfo_t): The type info - flag (int, optional): The flag of the member type. Defaults to 0. - - Returns: - smt_code_t: Unknown IDA documentation - """ - set_mem_ret = ida_struct.set_member_tinfo(sid, member, offset, tif, flag) - if is_string: - member.flag = member.flag | ida_bytes.FF_STRLIT - return set_mem_ret - - def get_struct_member_id(self, sid: ida_struct.struc_t, offset: int) -> int: - """Get the member id - - Args: - sid (ida_struct.struc_t): The struct - offset (int): The offset of the member in the struct - - Returns: - int: The member id inside of the struct - """ - return ida_struct.get_member_id(sid, offset) - - def get_base_class_flag(self): - """Get the base class flag - - Returns: - int: The flag for the base class type - """ - return ida_struct.MF_BASECLASS - - def get_enum_id(self, name: str): - """Get the id of an enum by its name - - Args: - name (str): The name of the enum - - Returns: - int: The id of the enum - """ - return ida_enum.get_enum(name) - - def remove_enum_member(self, eid: int, value: str, name: str): - """Remove an enum member by its value and name - - Args: - eid (int): The id of the enum - value (str): The value of the enum member - name (str): The name of the enum member - """ - mem = ida_enum.get_enum_member_by_name("{0}.{1}".format(name, value)) - ida_enum.del_enum_member( - eid, - ida_enum.get_enum_member_value(mem), - ida_enum.get_enum_member_serial(mem), - ida_enum.get_enum_member_bmask(mem), + return idc.next_head(ea) + + return idc.BADADDR + + def update_member_func(self, member_func: DefinedStructMemFunc, struct: DefinedStruct) -> None: + func_name = f"{struct.type}.{member_func.name}" + ea = self.get_func_ea_by_name(func_name) + if ea == idc.BADADDR: + ea = self.get_func_ea_by_sig(member_func.signature) + if ea == idc.BADADDR: + print( + f"Error: {func_name} not found bad sig? {member_func.signature}" ) - - def delete_enum_members(self, eid: int): - """Remove all enum members - - Args: - eid (int): The id of the enum - """ - - masks = [-1] - if idc.is_bf(eid): - mask = idc.get_first_bmask(eid) - while mask != idaapi.DEFMASK: - masks.append(mask) - mask = idc.get_next_bmask(eid, mask) - - members = [] - for mask in masks: - mem_val = idc.get_first_enum_member(eid, mask) - while mem_val != idaapi.BADNODE: - cid = ida_enum.get_first_serial_enum_member(eid, mem_val, mask) - members.append(cid[0]) - mem_val = idc.get_next_enum_member(eid, mem_val, mask) - - for cid in members: - ida_enum.del_enum_member( - eid, - ida_enum.get_enum_member_value(cid), - ida_enum.get_enum_member_serial(cid), - ida_enum.get_enum_member_bmask(cid), + return + if ida_funcs.get_func_name(ea) == f"sub_{ea:X}": + idc.set_name(ea, func_name) + tif = ida_typeinf.tinfo_t() + ida_typeinf.guess_tinfo(tif, ea) + func_data = ida_typeinf.func_type_data_t() + tif.get_func_details(func_data) + func_data.clear() + func_data.cc = ida_typeinf.CM_CC_FASTCALL + func_data.rettype = self.get_tinfo_from_type(member_func.return_type) + for param in member_func.parameters: + arg = ida_typeinf.funcarg_t() + try: + arg.type = self.get_tinfo_from_type(param.type) + except ValueError as exc: + print( + f"Error: update_member_func: function={func_name!r}, ea={ea:#x}, parameter={param.name!r}, type={param.type!r}, error={exc}" ) - - def create_enum(self, name: str) -> int: - """Create an enum by its name - - Args: - name (str): The name of the enum - - Returns: - int: The id of the added enum - """ - return ida_enum.add_enum(idc.BADADDR, name, 0) - - def set_enum_width(self, eid: int, width: int): - """Set the width of an enum by its id - - Args: - eid (int): The id of the enum - width (int): The width of the enum - """ - ida_enum.set_enum_width(eid, width) - - def set_enum_flag(self, eid: int, flag: int): - """Set a flag on an enum by its id - - Args: - eid (int): The id of the enum - flag (int): The flag to set - """ - ida_enum.set_enum_flag(eid, flag) - - def get_enum_default_mask(self, eid: int): - """Get the default bitmask for an enum - - Args: - eid (int): The id of the enum - - Returns: - int: The default bitmask for the enum - """ - width = idc.get_enum_width(eid) - mask = (1 << (width * 8)) - 1 - - if mask == ida_enum.DEFMASK: - mask >>= 1 - - return mask - - def get_enum_name(self, eid: int): - return idc.get_enum_name(eid) - - def get_enum_bf(self, eid: int): - return ida_enum.is_bf(eid) - - def set_enum_as_bf(self, eid: int): - ida_enum.set_enum_bf(eid, True) - - name = self.get_enum_name(eid) - bmask = self.get_enum_default_mask(eid) - - # this shouldn't happen under normal circumstances - if ida_enum.get_bmask_name(eid, bmask) == f"{name}_Mask": + raise + arg.name = param.name + func_data.push_back(arg) + tif.create_func(func_data) + ida_typeinf.apply_tinfo(ea, tif, ida_typeinf.TINFO_DEFINITE) + + def update_virt_func(self, virt_func: DefinedStructVFunc, struct: DefinedStruct): + func_name = f"{struct.type}.{virt_func.name}" + + ea = self.get_func_ea_by_name(func_name) + if ea == idc.BADADDR: + ea = self.get_vfunc_ea_from_data(struct.type, virt_func.offset) + if ea in (0, idc.BADADDR): + print(f"Error: {func_name} not found and its VFT slot could not be resolved") return - ida_enum.set_bmask_name(eid, bmask, f"{name}_Mask") - - def add_enum_member(self, eid: int, name: str, value: int, mask: int = -1): - """Add an enum member to an enum by its id - Args: - eid (int): The id of the enum - name (str): The name of the enum member - value (int): The value of the enum member - mask (int): The bitmask of the enum member, or -1 to use the default mask - """ - if mask == -1 and self.get_enum_bf(eid): - mask = self.get_enum_default_mask(eid) - - ec = ida_enum.add_enum_member(eid, name, value, mask) - - if ec == ida_enum.ENUM_MEMBER_ERROR_MASK: - ida_enum.add_enum_member(eid, name, value) - - def get_struct_flag(self): - """Get the flag for a struct data type - - Returns: - int: The flag for a struct data type - """ - return ida_bytes.stru_flag() - - def get_enum_flag(self): - """Get the flag for an enum data type - - Returns: - int: The flag for an enum data type - """ - return ida_bytes.enum_flag() - - elif _get_ida_major() >= 9: - - def get_tinfo_from_func_data(self, data: DefinedStructFuncField): - """Retrieve a tinfo_t from a raw function data. - - Args: - data (DefinedStructFuncField): Function data. - - Returns: - idaapi.tinfo_t: tinfo_t created from the function data. - """ - tinfo = ida_typeinf.tinfo_t() - func_data = ida_typeinf.func_type_data_t() - func_data.cc = ida_typeinf.CM_CC_FASTCALL - func_data.rettype = self.get_tinfo_from_type(data.return_type) - for param in data.parameters: - arg = ida_typeinf.funcarg_t() - arg.type = self.get_tinfo_from_type(param.type) - arg.name = param.name - func_data.push_back(arg) - - tinfo.create_func(func_data) - return tinfo - - def get_struct_opinfo_from_type(self, raw_type: str): - """Retrieve an opinfo_t from a raw structure type string. - - Args: - raw_type (str): Raw structure type string. - - Returns: - ida_nalt.opinfo_t: opinfo_t created from the structure type string. - """ - opinf = ida_nalt.opinfo_t() - opinf.tid = ida_typeinf.get_named_type_tid(raw_type) - - return opinf - - def get_enum_opinfo_from_type(self, raw_type: str): - """Retrieve an opinfo_t from a raw enum type string. - - Args: - raw_type (str): Raw enum type string. - - Returns: - ida_nalt.opinfo_t: opinfo_t created from the enum type string. - """ - opinf = ida_nalt.opinfo_t() - opinf.ec.tid = idc.get_enum(raw_type) - - return opinf - - def search_binary(self, ea: int, pattern: str, flag: int): - """Search for a pattern in a binary - - Args: - ea (int): The address to start searching from - pattern (str): The pattern to search for - flag (int): The search flags + actual_name = ida_funcs.get_func_name(ea) or "" + normalized_name = actual_name + for prefix in ("j_", "thunk_"): + if normalized_name.startswith(prefix): + normalized_name = normalized_name[len(prefix):] - Returns: - int: The start address of the pattern - """ - return ida_bytes.find_bytes( - pattern, ea, None, ida_ida.inf_get_max_ea(), None, flag - ) - - def get_dword(self, ea: int): - """Retrieve a dword (32-bit value) from the specified address. - - Args: - ea (int): The effective address to read the dword from. - - Returns: - int: The dword value read from the given address. - """ - return ida_bytes.get_original_dword(ea) - - def get_func_ea_by_name(self, name: str): - """Retrieve the effective address of a function by its name. - - Args: - name (str): The name of the function. - - Returns: - int: The effective address of the function. - """ - return ida_name.get_name_ea(0, name) - - def get_func_ea_by_sig(self, pattern: str): - """Retrieve the effective address of a function by its signature. - - Args: - pattern (str): The signature of the function. - - Returns: - int: The effective address of the function. - """ - ea = self.search_binary(0, pattern, ida_search.SEARCH_DOWN) - - if ida_funcs.get_func(ea) is None: - finf = ida_funcs.func_t() - finf.start_ea = ea - finf.end_ea = idc.BADADDR - ida_funcs.add_func_ex(finf) - - if ida_funcs.get_func(ea) is None: - return idc.BADADDR - - if ida_funcs.get_func(ea).start_ea == ea: - return ea - mnem = idc.print_insn_mnem(ea) - if not mnem: - return idc.BADADDR + if normalized_name == "_purecall": + return - opType0 = idc.get_operand_type(ea, 0) - if mnem == "jmp" or mnem == "call" or mnem[0] == "j": - if opType0 != idc.o_near and opType0 != idc.o_mem: - print( - "Error: Can't follow opType0 {0}".format( - self.opTypeAsName(opType0) - ) - ) - return idc.BADADDR - return idc.get_operand_value(ea, 0) + primary_vtable = self.get_vft_from_data(struct.type) + primary_base = primary_vtable.get("base") if primary_vtable else None + if primary_base: + base_ea = self.get_vfunc_ea_from_data(primary_base, virt_func.offset) + if base_ea == ea: + return - if idc.next_head(ea) == ea + idc.get_item_size(ea) and idc.is_flow( - idc.get_full_flags(idc.next_head(ea)) + if ( + normalized_name.endswith("." + virt_func.name) + and not normalized_name.startswith(struct.type + ".") ): - return idc.next_head(ea) - - return idc.BADADDR - - def opTypeAsName(self, n: int): - """ - Retrieve the name of the operand type constant. - - Args: - n (int): The integer value of the operand type. - - Returns: - str: The name of the operand type constant if found; otherwise, None. - """ - - for item in [x for x in dir(idc) if x.startswith("o_")]: - if getattr(idc, item) == n: - return item - - def create_struct_type(self, name: str, union: bool = False) -> int: - """Create a struct type - - Args: - name (str): The name of the struct - union (bool, optional): Whether the struct is a union. Defaults to False. - - Returns: - int: The struct id - """ - - udt = ida_typeinf.udt_type_data_t() - udt.is_union = union - - tinfo = ida_typeinf.tinfo_t() - if not tinfo.create_udt(udt): - return idc.BADADDR - - if tinfo.set_named_type(None, name) != ida_typeinf.TERR_OK: - return idc.BADADDR - - return tinfo.get_tid() - - def get_struct_id(self, name: str) -> int: - """Get the struct id - - Args: - name (str): The name of the struct - - Returns: - int: The struct id - """ - return ida_typeinf.get_named_type_tid(name) - - def get_struct(self, sid: int) -> ida_typeinf.tinfo_t: - """Get the struct - - Args: - sid (int): The struct id - - Returns: - ida_typeinf.tinfo_t: The struct's parent tinfo_t - """ - return ida_typeinf.tinfo_t(tid=sid) - - def get_struct_size(self, sid: ida_typeinf.tinfo_t | int) -> int: - """Get the struct size - - Args: - sid (ida_typeinf.tinfo_t): The struct - - Returns: - int: The struct size - """ - if isinstance(sid, int): - sid = self.get_struct(sid) - return sid.get_size() - - def create_struct_member( - self, - sid: ida_typeinf.tinfo_t, - name: str, - offset: int = -1, - flag: int = idc.FF_DATA | idc.FF_QWORD, - typeid: int | idaapi.opinfo_t | None = None, - nbytes: int = 8, - ): - """Create a struct member - - Args: - sid (ida_typeinf.tinfo_t): The struct to add the member to - name (str): The name of the member - offset (int, optional): The offset of the member. Defaults to -1. - flag (int, optional): The type flag of the member. Defaults to idc.FF_DATA | idc.FF_QWORD. - typeid (int | None, optional): The type id of the member. Defaults to None. - nbytes (int, optional): The number of bytes of the member. Defaults to 8. - - Returns: - bool: True if successful - """ - - if typeid is None: - typeid = -1 - - if isinstance(typeid, idaapi.opinfo_t): - tinfo = ida_typeinf.tinfo_t() - if tinfo.get_type_by_tid(typeid.tid): - nbytes = tinfo.get_size() - else: - raise ValueError("Cannot find type with tid {0}".format(typeid.tid)) - - typeid = typeid.tid - - ec = idc.add_struc_member(sid.get_tid(), name, offset, flag, typeid, nbytes) - if ec != ida_typeinf.TERR_OK: - return False - - return True - - def remove_struct_member(self, sid: int, name: str) -> bool: - """Remove a struct member - - Args: - sid (int): The struct id - name (str): The name of the member - - Returns: - tinfo_code_t: See ida_typeinfo.TERR_ constants - """ - try: - tinfo = ida_typeinf.tinfo_t(tid=sid) - idx = tinfo.find_udm(name=name) - if idx == -1: - return True + return + if self.get_tinfo_from_type(virt_func.return_type) is None: + raise ValueError(f"Got invalid return type on virtual function: {virt_func}") - return tinfo.del_udm(idx) - except: - return ida_typeinf.TERR_BAD_ARG + tif = ida_typeinf.tinfo_t() + ida_typeinf.guess_tinfo(tif, ea) - def remove_struct_members(self, sid: int) -> int: - """Remove all struct members + func_data = ida_typeinf.func_type_data_t() + tif.get_func_details(func_data) - Args: - sid (int): The struct id + func_data.clear() + func_data.cc = ida_typeinf.CM_CC_FASTCALL + func_data.rettype = self.get_tinfo_from_type(virt_func.return_type) - Returns: - int: The number of members removed or -1 if failed - """ + for param in virt_func.parameters: + arg = ida_typeinf.funcarg_t() try: - tinfo = ida_typeinf.tinfo_t() - if not tinfo.get_type_by_tid(sid): - return -1 - - size = tinfo.get_udt_nmembers() - tinfo.del_udms(0, size) - return size - except: - return -1 - - def get_struct_member( - self, sid: ida_typeinf.tinfo_t, offset: int - ) -> ida_typeinf.udm_t: - """Get a struct member - - Args: - sid (ida_typeinf.tinfo_t): The struct - offset (int): The offset of the member - - Returns: - ida_typeinf.udm_t: The member data - """ - _, udm = sid.get_udm_by_offset(offset) - return udm - - def get_struct_member_by_name( - self, sid: ida_typeinf.tinfo_t, name: str - ) -> ida_typeinf.udm_t: - """Get a struct member - - Args: - sid (ida_typeinf.tinfo_t): The struct - name (str): The name of the member - - Returns: - ida_typeinf.udm_t: The member data - """ - _, udm = sid.get_udm(name) - return udm - - def set_struct_member_info( - self, - sid: ida_typeinf.tinfo_t, - member: ida_typeinf.udm_t, - offset: int, - tif: ida_typeinf.tinfo_t, - flag: int = 0, - is_string: bool = False, - ): - """Set the info of a struct member - - Args: - sid (ida_typeinf.tinfo_t): The struct - member (ida_typeinf.udm_t): The member - offset (int): The offset in the member - tif (ida_typeinf.tinfo_t): The type info - flag (int, optional): The flag of the member type. Defaults to 0. - - Returns: - tinfo_code_t: See ida_typeinfo.TERR_ constants - """ - if offset != 0: - raise ValueError( - "IDA 9+ does not support offset != 0 in set_struct_member_info" - ) - - memberIdx, _ = sid.get_udm_by_offset(member.offset) - return sid.set_udm_type(index=memberIdx, tif=tif) - - def get_struct_member_id(self, sid: ida_typeinf.tinfo_t, offset: int) -> int: - """Get the member id - - Args: - sid (ida_typeinf.tinfo_t): The struct - offset (int): The offset of the member in the struct - - Returns: - int: The member id inside of the struct or idc.BADADDR if not found - """ - idx, _ = sid.get_udm_by_offset(offset * 8) - return idc.BADADDR if idx == -1 else idx - - def get_enum_id(self, name: str): - """Get the id of an enum by its name - - Args: - name (str): The name of the enum - - Returns: - int: The id of the enum - """ - - return idc.get_enum(name) - - def remove_enum_member(self, eid: int, value: str, name: str): - """Remove an enum member by its value and name - - Args: - eid (int): The id of the enum - value (str): The value of the enum member - name (str): The name of the enum member - """ - - mem = idc.get_enum_member_by_name("{0}.{1}".format(name, value)) - if mem != idc.BADADDR and idc.get_enum_member_value(mem) != None: - idc.del_enum_member( - eid, - idc.get_enum_member_value(mem), - 0, - idc.get_enum_member_bmask(mem) or -1, + arg.type = self.get_tinfo_from_type(param.type) + except ValueError as exc: + print( + f"Error: update_virt_func: function={func_name!r}, ea={ea:#x}, " + f"parameter={param.name!r}, type={param.type!r}, error={exc}" ) + raise - def delete_enum_members(self, eid: int): - """Remove all enum members - - Args: - eid (int): The id of the enum - """ - tif = ida_typeinf.tinfo_t() - if not tif.get_type_by_tid(eid): - raise RuntimeError(f"Failed to get tinfo_t for enum {eid}") + arg.name = param.name + func_data.push_back(arg) - eti = ida_typeinf.enum_type_data_t() - if not tif.get_enum_details(eti): - raise RuntimeError(f"Failed to get enum details for enum id {eid}") + tif.create_func(func_data) + ida_typeinf.apply_tinfo(ea, tif, ida_typeinf.TINFO_DEFINITE) - # must clear values before groups - - values = [] - for idx, _, _ in eti.all_constants(): - values.append(eti[idx].name) - - for name in values: - tif.del_edm(name) - - groups = [] - for idx, _ in eti.all_groups(): - groups.append(eti[idx].name) - - for name in groups: - tif.del_edm(name) - - def create_enum(self, name: str) -> int: - """Create an enum by its name - - Args: - name (str): The name of the enum - - Returns: - int: The id of the added enum - """ - return idc.add_enum(0, name, 0) - - def set_enum_width(self, eid: int, width: int): - """Set the width of an enum by its id - - Args: - eid (int): The id of the enum - width (int): The width of the enum - """ - # TODO(caitlyn): handle changing the width of a bitfield enum - - idc.set_enum_width(eid, width) - - def get_enum_default_mask(self, eid: int): - """Get the default bitmask for an enum - - Args: - eid (int): The id of the enum - - Returns: - int: The default bitmask for the enum - """ - width = idc.get_enum_width(eid) - mask = (1 << (width * 8)) - 1 - - # IDA 9+ has an issue wherein the default mask for 64-bit - # enums is equal to DEFMASK64. This causes issues when - # creating and assigning members to the default group. - # - # To work around this, we drop the highest order bit - # from the default group for 64-bit enums. - if mask == ida_typeinf.DEFMASK64: - mask >>= 1 - - return mask - - def set_enum_flag(self, eid: int, flag: int): - """Set a flag on an enum by its id + def update_static_member(self, static_member: DefinedStructStaticMember, struct: DefinedStruct): + ea = self.search_binary( + 0, static_member.signature, ida_search.SEARCH_DOWN + ) + if ea == idc.BADADDR: + print(f"Error: {static_member.signature} not found something is wrong") + return + for follows in static_member.relative_offsets: + ea = ea + follows + ea = ea + 4 + self.get_dword(ea) + tif = ida_typeinf.tinfo_t() + ida_typeinf.guess_tinfo(tif, ea) + return_type = static_member.return_type + if static_member.is_pointer: + return_type = return_type + "*" + ida_typeinf.apply_tinfo( + ea, + self.get_tinfo_from_type(return_type), + ida_typeinf.TINFO_DEFINITE, + ) + if static_member.is_pointer: + ida_name.set_name( + ea, + f"g_{struct.type}_PtrInstance", + ) + else: + ida_name.set_name( + ea, f"g_{struct.type}_Instance" + ) + + def mark_as_vtable(self, vtbl_name: str) -> str | None: + """ + Mark a struct as a vtable (TAUDT_VFTABLE). - Args: - eid (int): The id of the enum - flag (int): The flag to set - """ - idc.set_enum_flag(eid, flag) + Returns: + An error string on failure, None on success. + """ + tif = ida_typeinf.tinfo_t() - def get_enum_bf(self, eid: int): - return idc.is_bf(eid) + if not tif.get_named_type(None, vtbl_name): + return f"Type '{vtbl_name}' not found in type library" - def get_enum_name(self, eid: int): - return idc.get_enum_name(eid) + if not tif.is_struct(): + actual = ( + "union" if tif.is_union() else "enum" if tif.is_enum() else "unknown" + ) + return f"'{vtbl_name}' is not a struct (got {actual})" - def get_enum_bitmask_field(self, eid: int): - name = self.get_enum_name(eid) - bmask = self.get_enum_default_mask(eid) + udt = ida_typeinf.udt_type_data_t() + if not tif.get_udt_details(udt): + return f"Failed to retrieve UDT details for '{vtbl_name}'" - return (f"{name}_Mask", bmask) + if udt.taudt_bits & ida_typeinf.TAUDT_VFTABLE: + return f"'{vtbl_name}' is already marked as a vtable" - def set_enum_as_bf(self, eid: int): - if idc.is_bf(eid): - return + udt.taudt_bits |= ida_typeinf.TAUDT_VFTABLE - idc.set_enum_bf(eid, True) + if not tif.create_udt(udt, ida_typeinf.BTF_STRUCT): + return ( + f"Failed to reconstruct UDT for '{vtbl_name}' after setting vtable flag" + ) - name, bmask = self.get_enum_bitmask_field(eid) + result = tif.set_named_type(None, vtbl_name) - # this shouldn't happen under normal circumstances - if idc.get_enum_member_by_name(f"{name}_Mask") != idaapi.BADADDR: - return + if not result: + return f"Failed to write back '{vtbl_name}' to type library" - idc.add_enum_member(eid, f"{name}_Mask", bmask, bmask) - - def add_enum_member(self, eid: int, name: str, value: int, mask: int = -1): - """Add an enum member to an enum by its id - Args: - eid (int): The id of the enum - name (str): The name of the enum member - value (int): The value of the enum member - mask (int): The bitmask of the enum member, or -1 to use the default mask - """ - - if mask == -1 and idc.is_bf(eid): - mask = self.get_enum_default_mask(eid) - - # IDA 9+ requires all enum member names be unique across the DB. - # We attempt to avoid collisions by appending up to 3 underscores to the name. - retries = 0 - processedName = name - while idc.get_enum_member_by_name(processedName) != idaapi.BADADDR: - if retries >= 3: - raise RuntimeError( - f"Error: too many duplicate enum member names for '{name}'" - ) - retries += 1 - processedName += "_" - - # IDA 9.0 - 9.2 vary in how they handle errors. - # IDA 9.0 will return a TERR_ code, while IDA 9.2 will raise an exception. - # This will effectively convert the TERR_ to an equivalent ValueError for 9.0. - ec = idc.add_enum_member(eid, processedName, value, mask) - if ec != ida_typeinf.TERR_OK: - raise ValueError(ida_typeinf.tinfo_errstr(ec)) - - def get_struct_flag(self): - """Get the flag for a struct data type - - Returns: - int: The flag for a struct data type - """ - return ida_bytes.stru_flag() - - def get_enum_flag(self): - """Get the flag for an enum data type - - Returns: - int: The flag for an enum data type - """ - return ida_bytes.enum_flag() - - else: - raise RuntimeError("Unsupported IDA version") + return None \ No newline at end of file diff --git a/scripts/ida/struct_import.py b/scripts/ida/struct_import.py index 289af07afc..db8eabad8e 100644 --- a/scripts/ida/struct_import.py +++ b/scripts/ida/struct_import.py @@ -12,30 +12,102 @@ ) from src_wrapper import SrcInterface from ida_wrapper import IdaInterface +from form_builder import DynamicForm +from structs_schema import get_yaml import ida_bytes import ida_search import ida_typeinf import ida_hexrays import idc +import idaapi src = SrcInterface() +ida = IdaInterface() +print("Loading data.yml") +ida.load_data_yml() + +check_fields = [ + {"type": "label", "text": "Check version match."}, + {"type": "space"}, + {"type": "label", "text": f"Loaded with ver: {ida.data_yaml.version}"}, + {"type": "label", "text": "Check if this matches with what is reported in the version files of the game."} +] + +form = DynamicForm(check_fields) +form.Compile() +try: + if form.Execute() != 1: + raise "Version is not correct please update repo" +finally: + form.Free() + +ida.select_parser() +fields = [ + {"type": "label", "text": "Select function types to update"}, + {"type": "space"}, + {"type": "label", "text": "Types to update"}, + { + "type": "checks", + "key": "function_types", + "items": [ + ("virtual", "Virtual functions"), + ("member", "Member functions"), + ], + }, + {"type": "space"}, + {"type": "label", "text": "Base name?"}, + { + "type": "radios", + "key": "base_name", + "items": [ + ("ffxiv_dx11.exe", "x64dbg offset (ffxiv_dx11.exe)"), + ("__ImageBase", "IDA defualt (__ImageBase)"), + ], + } +] +form = DynamicForm(fields) +form.Compile() +try: + if form.Execute() == 1: + _function_update_options = form.results()["function_types"] + idc.set_name(idaapi.get_imagebase(), form.results()["base_name"]) + else: + _function_update_options = { + "virtual": False, + "member": False, + } +finally: + form.Free() + +print("Grabbing ExdSchema and Lumina") excel_header_files = get_excel_header_files() exdschema = get_exdschema_data("latest") -struct_export, excel_map = create_struct_from_header_and_schema( +excel_structs, excel_map = create_struct_from_header_and_schema( excel_header_files, exdschema ) exd_map = {excel_map[k]: k for k in excel_map} -header, vtables = src.build_export_string(struct_export, 4) +excel_header, excel_vtables = src.build_export_string(excel_structs, 4) -ida = IdaInterface() -ida.select_parser() -ida.parse_string(header, vtables) +print("Parsing ffxiv_structs.yml") + +with open(ida.get_struct_path()) as f: + import_structs = get_yaml(f) + +struct_header, struct_vtables = src.build_export_string(import_structs, 1) +print("Parsing combined Excel and FFXIV declarations") +ida.parse_string( + f"""{src.get_predef_header()} +{excel_header} +{struct_header} +""", + [*excel_vtables, *struct_vtables], +) def do_pattern(pattern, suffix, sheetSearchPattern, sheetSearchOffset): ea = 0 @@ -128,6 +200,7 @@ def get_all_eas(pattern: str): ea = ea + 1 ea = ida.search_binary(ea, pattern, ida_search.SEARCH_DOWN) +print("Fixing up Excel functions") for pattern in exd_func_patterns: suffix, search = exd_func_patterns[pattern] @@ -144,4 +217,34 @@ def get_all_eas(pattern: str): sheetName = exd_map[sheetIdx] ida_bytes.set_cmt(ea, "Sheet: {0} ({1})".format(sheetName, sheetIdx), 0) except Exception as e: - print(f"An unexpected error occurred at {hex(ea)}: {e}") \ No newline at end of file + print(f"An unexpected error occurred at {hex(ea)}: {e}") + +if _function_update_options["virtual"]: + for struct in import_structs.structs: + if struct.virtual_functions: + print(f"Updating virtual functions for {struct.type}") + for virt_func in struct.virtual_functions: + if virt_func is None: + continue + + if virt_func.return_type is not None and virt_func.parameters is not None: + ida.update_virt_func(virt_func, struct) + +print("Fixing up CS defined functions") + +if _function_update_options["member"]: + for struct in import_structs.structs: + if struct.member_functions != []: + print(f"Updating member functions for {struct.type}") + for member_func in struct.member_functions: + ida.update_member_func(member_func, struct) + + if struct.static_member_functions: + print(f"Updating static member functions for {struct.type}") + for member_func in struct.static_member_functions: + ida.update_member_func(member_func, struct) + + if struct.static_members: + print(f"Updating static members for {struct.type}") + for member in struct.static_members: + ida.update_static_member(member, struct) \ No newline at end of file From d29f966c999cbb50d53e73630ab6067e6a26cad1 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Mon, 12 Oct 2026 08:07:29 +0200 Subject: [PATCH 47/95] send up error log for cross referencing --- scripts/ida/ida_error.txt | 63 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 scripts/ida/ida_error.txt diff --git a/scripts/ida/ida_error.txt b/scripts/ida/ida_error.txt new file mode 100644 index 0000000000..4fbd6799a2 --- /dev/null +++ b/scripts/ida/ida_error.txt @@ -0,0 +1,63 @@ +Loading data.yml +Grabbing ExdSchema and Lumina +Requesting nugpkg Lumina with latest version +Requesting nugpkg BCnEncoder.Net with version 2.2.1 +Requesting nugpkg Microsoft.Extensions.ObjectPool with version 10.0.0 +Requesting nugpkg CommunityToolkit.HighPerformance with version 8.4.0 +Requesting nugpkg Microsoft.Bcl.HashCode with version 6.0.0 +Requesting nugpkg System.Memory with version 4.6.0 +Requesting nugpkg System.Runtime.CompilerServices.Unsafe with version 6.1.0 +Requesting nugpkg System.Threading.Tasks.Extensions with version 4.6.0 +Requesting nugpkg System.Buffers with version 4.6.0 +Requesting nugpkg System.Numerics.Vectors with version 4.6.0 +Downloading https://github.com/xivdev/EXDSchema/archive/refs/heads/latest.zip +Parsing ffxiv_structs.yml +Parsing combined Excel and FFXIV declarations +C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Tools\MSVC\14.51.36231\include\map:46:11 error Failed to prepare forward declaration for : name 'std::_Tmap_traits, std::allocator>, false>::value_compare' is not acceptable + +C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Tools\MSVC\14.51.36231\include\map:46:11 error Failed to prepare forward declaration for : name 'std::_Tmap_traits, std::allocator>, false>::value_compare' is not acceptable + +input.cc:99049:31 error Failed to prepare forward declaration for : name 'StdLinkedList::Node' is not acceptable + +C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Tools\MSVC\14.51.36231\include\map:46:11 error Failed to prepare forward declaration for : name 'std::_Tmap_traits, std::allocator>, false>::value_compare' is not acceptable + +C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Tools\MSVC\14.51.36231\include\map:46:11 error Failed to prepare forward declaration for : name 'std::_Tmap_traits, std::allocator>, false>::value_compare' is not acceptable + +C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Tools\MSVC\14.51.36231\include\map:46:11 error Failed to prepare forward declaration for : name 'std::_Tmap_traits, std::allocator>, false>::value_compare' is not acceptable + +C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Tools\MSVC\14.51.36231\include\map:46:11 error Failed to prepare forward declaration for : name 'std::_Tmap_traits, std::allocator>, false>::value_compare' is not acceptable + +:-1:-1 warning ignored: __cdecl is not supported by the current architecture +C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Tools\MSVC\14.51.36231\include\map:46:11 error Failed to prepare forward declaration for : name 'std::_Tmap_traits, std::allocator>, false>::value_compare' is not acceptable + +Fixing up Excel functions +Finding exd funcs of Row... please wait. +Finding exd funcs of Row... please wait. +Finding exd funcs of Row... please wait. +Finding exd funcs of Row... please wait. +Finding exd funcs of Row... please wait. +Finding exd funcs of Row... please wait. +Finding exd funcs of Row... please wait. +Finding exd funcs of RowAndSubRowId... please wait. +Func @ 0x140844120 references unknown sheet 1401! sheet dword offset = 0x14084413B +Func @ 0x140844170 references unknown sheet 1400! sheet dword offset = 0x14084418B +Func @ 0x1408441C0 references unknown sheet 1399! sheet dword offset = 0x1408441DB +Finding exd funcs of RowAndSubRowId... please wait. +Finding exd funcs of RowIndex... please wait. +Finding exd funcs of RowIndex... please wait. +Finding exd funcs of SheetIndex... please wait. +Finding exd funcs of RowCount... please wait. +An unexpected error occurred at 0x140844124: 1401 +An unexpected error occurred at 0x140844174: 1400 +An unexpected error occurred at 0x1408441c4: 1399 +Updating virtual functions for Animation::Playback::Control::Default::hkaDefaultAnimationControl::hkaDefaultAnimationControlListener +Error: Animation::Playback::Control::Default::hkaDefaultAnimationControl::hkaDefaultAnimationControlListener.LoopOverflowCallback not found and its VFT slot could not be resolved +Error: Animation::Playback::Control::Default::hkaDefaultAnimationControl::hkaDefaultAnimationControlListener.LoopUnderflowCallback not found and its VFT slot could not be resolved +Error: Animation::Playback::Control::Default::hkaDefaultAnimationControl::hkaDefaultAnimationControlListener.EasedInCallback not found and its VFT slot could not be resolved +Error: Animation::Playback::Control::Default::hkaDefaultAnimationControl::hkaDefaultAnimationControlListener.EasedOutCallback not found and its VFT slot could not be resolved +Error: Animation::Playback::Control::Default::hkaDefaultAnimationControl::hkaDefaultAnimationControlListener.dtor not found and its VFT slot could not be resolved +Updating virtual functions for Animation::Playback::Control::hkaAnimationControlListener +Error: Animation::Playback::Control::hkaAnimationControlListener.ControlDeletedCallback not found and its VFT slot could not be resolved +Error: Animation::Playback::Control::hkaAnimationControlListener.dtor not found and its VFT slot could not be resolved +Updating virtual functions for Application::Network::LobbyClient::LobbyRequestCallback +! failed to create pointer for 'Application::Network::LobbyClient::LobbyRequestCallback*': base type 'Application::Network::LobbyClient::LobbyRequestCallback' resolved to '?' (correct=True, forward=False) \ No newline at end of file From 91a7ef1660821de738cc4252e0cb4ebd044c9621 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Mon, 12 Oct 2026 11:22:13 +0200 Subject: [PATCH 48/95] correct defined size --- scripts/src_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/src_wrapper.py b/scripts/src_wrapper.py index 001bf4e768..5a1f297a03 100644 --- a/scripts/src_wrapper.py +++ b/scripts/src_wrapper.py @@ -218,7 +218,7 @@ def get_size_from_type( "std::wstring": 32, "std::basic_string": 32, "std::list": 16, - "std::deque": 32, + "std::deque": 40, } if base_type in stl_type_sizes: return stl_type_sizes[base_type] From 856672235cfdca7dac9d8d73af50683b90620fee Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Tue, 13 Oct 2026 01:54:46 +0200 Subject: [PATCH 49/95] add back removed lines due to rebase --- CExporter/Exporter.cs | 5 +++++ CExporter/ProcessedData.cs | 1 + CExporter/YamlConverters.cs | 2 ++ scripts/structs_schema.py | 2 ++ 4 files changed, 10 insertions(+) diff --git a/CExporter/Exporter.cs b/CExporter/Exporter.cs index d60413eac8..af33d0f873 100644 --- a/CExporter/Exporter.cs +++ b/CExporter/Exporter.cs @@ -109,9 +109,14 @@ public static (ProcessedStruct s, int c) ProcessStaticFunction(ProcessedStruct c foreach (var methodInfo in methods) { var staticAddress = methodInfo.GetCustomAttribute(); if (staticAddress != null) { + var name = methodInfo.Name; + if (name.StartsWith("Get")) { + name = name[3..]; + } currentStruct.StaticMembers ??= []; currentStruct.StaticMembers = [.. currentStruct.StaticMembers, new ProcessedStaticMembers { + Name = name, Signature = staticAddress.Signature, RelativeFollowOffsets = staticAddress.RelativeFollowOffsets, IsPointer = staticAddress.IsPointer, diff --git a/CExporter/ProcessedData.cs b/CExporter/ProcessedData.cs index bbd6729ff3..02c0a55325 100644 --- a/CExporter/ProcessedData.cs +++ b/CExporter/ProcessedData.cs @@ -50,6 +50,7 @@ public class ProcessedFunctionField : ProcessedField { } public class ProcessedStaticMembers { + public required string Name; public required string Signature; public required ushort[] RelativeFollowOffsets; public bool IsPointer; diff --git a/CExporter/YamlConverters.cs b/CExporter/YamlConverters.cs index 272ac13e22..fe5c3c920b 100644 --- a/CExporter/YamlConverters.cs +++ b/CExporter/YamlConverters.cs @@ -290,6 +290,8 @@ public class ProcessedStaticMembersConverter : IYamlTypeConverter { public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializer serializer) { if (value is not ProcessedStaticMembers s) return; emitter.Emit(new MappingStart()); + emitter.Emit(new Scalar("name")); + emitter.Emit(new Scalar(s.Name)); emitter.Emit(new Scalar("signature")); emitter.Emit(new Scalar(s.Signature)); emitter.Emit(new Scalar("relative_follow_offsets")); diff --git a/scripts/structs_schema.py b/scripts/structs_schema.py index c4ac535836..1705c7fcc8 100644 --- a/scripts/structs_schema.py +++ b/scripts/structs_schema.py @@ -65,6 +65,7 @@ class DefinedStructFuncField(DefinedStructField): @dataclass class DefinedStructStaticMember: + name: str signature: str relative_offsets: list[int] return_type: str @@ -229,6 +230,7 @@ def parse_struct(struct: YamlStructExport) -> DefinedStruct: for sm in struct["static_members"]: static_members.append( DefinedStructStaticMember( + sm["name"], sm["signature"], sm["relative_follow_offsets"], sm["return_type"], From b4900c1c56e05e153d31495a67201f0cf2efd5b5 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Tue, 13 Oct 2026 01:55:44 +0200 Subject: [PATCH 50/95] hopefully fix the last remaining bugs from parsing --- CExporter/Exporter.TypeProcessing.cs | 26 ++- CExporter/Exporter.cs | 2 + scripts/ida/ida_wrapper.py | 274 ++++++++++++++------------- scripts/ida/struct_import.py | 12 +- scripts/src_wrapper.py | 70 ++++++- scripts/structs_schema.py | 11 ++ 6 files changed, 253 insertions(+), 142 deletions(-) diff --git a/CExporter/Exporter.TypeProcessing.cs b/CExporter/Exporter.TypeProcessing.cs index bbb8ef6c48..2cf3b17466 100644 --- a/CExporter/Exporter.TypeProcessing.cs +++ b/CExporter/Exporter.TypeProcessing.cs @@ -94,6 +94,7 @@ private static (string typeOverride, Type[] typeOverrideTemplates) ProcessStdTyp } while (stdNameSpan[^1] == '\0') stdNameSpan = stdNameSpan[..^1]; if (stdNameSpan is "std::string") return ($"{stdNameSpan}", []); + if (stdNameSpan is "std::unorderedmap") stdNameSpan = "std::unordered_map".ToCharArray(); var genericsDefine = generics.Select(t => { var pointerCount = 0; while (t.IsPointer()) { @@ -355,6 +356,8 @@ private static ProcessedField ProcessField(FieldInfo field, int offset) { FieldOffset = field.GetFieldOffset() - offset, FieldName = field.Name, IsBase = field.IsDirectBase(), + // template_args only covers generic structs held by value + FieldTypeOverride = field.FieldType.IsPointer() ? GetGenericTypeOverride(field.FieldType) : null, Bits = [], IsUnion = false, UnionValues = [] @@ -379,6 +382,24 @@ private static ProcessedField ProcessVirtualParameter(Type parameter, int i, Par parameterInfo?.GetCustomAttribute()); } + /// + /// Builds the type name with its generic arguments for constructed non std generic structs (ex. hkArray<hkQsTransformf>*). + /// drops the arguments as fields export them separately through template_args. + /// + private static string? GetGenericTypeOverride(Type type) { + if (type.IsFunctionPointer || type.IsUnmanagedFunctionPointer) return null; + var pointerCount = 0; + while (type.IsPointer()) { + if (type.IsPointer) type = type.GetElementType()!; + else type = type.GenericTypeArguments[0]; + if (type.IsFunctionPointer || type.IsUnmanagedFunctionPointer) return null; + pointerCount++; + } + if (!type.IsConstructedGenericType || type.IsStdDefiniton() || type.IsFixedBuffer()) return null; + var arguments = type.GenericTypeArguments.Select(t => GetGenericTypeOverride(t) ?? t.FixTypeName()); + return $"{type.FixTypeName()}<{string.Join(',', arguments)}>{new string('*', pointerCount)}"; + } + private static ProcessedField ProcessParameter( Type parameterType, string parameterName, @@ -396,6 +417,7 @@ private static ProcessedField ProcessParameter( } else if (typeForceAttribute != null) { typeOverride = typeForceAttribute.TypeName; } else { + typeOverride = GetGenericTypeOverride(parameterType); _processType.Add(parameterType); } @@ -509,8 +531,10 @@ private static ProcessedField ProcessParameter( _processType.Add(requiredType); } memberFunctionOverrideType += new string('*', pointerCount); + } else { + memberFunctionOverrideType ??= GetGenericTypeOverride(memberFunction.ReturnType); } - var memberFunctionReturnType = new ProcessedMemberFunctionReturn(returnType, memberFunctionOverrideType); + var memberFunctionReturnType = new ProcessedMemberFunctionReturn(memberFunction.ReturnType, memberFunctionOverrideType); if (memberFunction.GetCustomAttribute() is not { } excelAttribute) _processType.Add(memberFunctionReturnType.Type); else diff --git a/CExporter/Exporter.cs b/CExporter/Exporter.cs index af33d0f873..3d281e160e 100644 --- a/CExporter/Exporter.cs +++ b/CExporter/Exporter.cs @@ -140,6 +140,8 @@ public static (ProcessedStruct s, int c) ProcessStaticFunction(ProcessedStruct c _processType.Add(requiredType); } overrideType += new string('*', pointerCount); + } else { + overrideType = GetGenericTypeOverride(methodInfo.ReturnType); } var memberFunctionReturnType = new ProcessedMemberFunctionReturn(methodInfo.ReturnType, overrideType); if (methodInfo.GetCustomAttribute() is not { } excelAttribute) diff --git a/scripts/ida/ida_wrapper.py b/scripts/ida/ida_wrapper.py index 8342e2ff3e..a63cce7c21 100644 --- a/scripts/ida/ida_wrapper.py +++ b/scripts/ida/ida_wrapper.py @@ -4,6 +4,7 @@ import ida_nalt import ida_search import ida_ida +import ida_idp import ida_typeinf import ida_name import ida_funcs @@ -11,7 +12,7 @@ import ida_kernwin from structs_schema import * from form_builder import DynamicForm -from data_schema import get_yaml, DefinedData +from data_schema import get_yaml, DefinedData, DefinedDataClassVtable from abc import abstractmethod from platform import system @@ -95,34 +96,35 @@ def get_named_type(self, name: str): """Retrieve a tinfo_t from the named type. Args: - name (str): Name of the type. + name (str): Name of the type, or any declaration IDA can parse. + + Returns: + tinfo_t | None: The type, None if IDA does not know it. """ tinfo = ida_typeinf.tinfo_t() - clean_name = name - if ( - self.get_struct_id(clean_name) != idaapi.BADADDR - or self.get_enum_id(clean_name) != idaapi.BADADDR - ): - if not tinfo.get_named_type(idaapi.get_idati(), clean_name): - raise ValueError(f"{clean_name} not found in IDA database") + if tinfo.get_named_type(idaapi.get_idati(), name): return tinfo if name == "void": - idaapi.parse_decl( - tinfo, idaapi.get_idati(), "void (__fastcall)();", idaapi.PT_SIL - ) - return tinfo.get_rettype() + return ida_typeinf.tinfo_t(ida_typeinf.BT_VOID) - terminated = name + ";" - idaapi.parse_decl(tinfo, idaapi.get_idati(), terminated, idaapi.PT_SIL) + if name == "__fastcall": + # function pointer parameters are exported without their signature + name = "void (__fastcall *)()" - tinfo_str = tinfo.dstr() - if tinfo_str == name or tinfo_str == clean_name: + if "<" in name and tinfo.get_named_type( + idaapi.get_idati(), get_template_alias(name) + ): + # use the type behind the typedef so IDA shows the real name + target = ida_typeinf.tinfo_t() + if target.get_named_type(idaapi.get_idati(), tinfo.get_final_type_name()): + return target return tinfo - terminated = clean_name + ";" - idaapi.parse_decl(tinfo, idaapi.get_idati(), terminated, idaapi.PT_SIL) + terminated = name.rstrip(";") + ";" + if idaapi.parse_decl(tinfo, idaapi.get_idati(), terminated, idaapi.PT_SIL) is None: + return None return tinfo @@ -132,41 +134,86 @@ def get_tinfo_from_type(self, raw_type: str, array_size=0): Args: raw_type (str): Raw type string. array_size (int, optional): Size of the array. Defaults to 0. + + Returns: + tinfo_t | None: The type, None if it could not be built. """ type = raw_type.rstrip("*") ptr_count = len(raw_type) - len(type) - type_tinfo = self.get_named_type(type) - - ptr_tinfo = None - if ptr_count > 0: - for i in range(ptr_count): - ptr_tinfo = idaapi.tinfo_t() - if not ptr_tinfo.create_ptr(type_tinfo): - print( - f"! failed to create pointer for {raw_type!r}: " - f"base type {type!r} resolved to {type_tinfo.dstr()!r} " - f"(correct={type_tinfo.is_correct()}, " - f"forward={type_tinfo.is_forward_decl()})" - ) - return None - type_tinfo = ptr_tinfo - else: - ptr_tinfo = type_tinfo + tinfo = self.get_named_type(type) + if tinfo is None: + return None + + for _ in range(ptr_count): + ptr_tinfo = idaapi.tinfo_t() + if not ptr_tinfo.create_ptr(tinfo): + return None + tinfo = ptr_tinfo if array_size > 0: array_tinfo = idaapi.tinfo_t() - if not array_tinfo.create_array(ptr_tinfo, array_size): - print("! failed to create array") + if not array_tinfo.create_array(tinfo, array_size): return None + tinfo = array_tinfo + + return tinfo + + def apply_func_type(self, ea: int, func_name: str, return_type: str, parameters: list[DefinedStructFuncParam]) -> bool: + """Apply a __fastcall prototype built from the yml types to a function. + + Returns: + bool: False if one of the types is not known to IDA. + """ + func_data = ida_typeinf.func_type_data_t() + func_data.cc = ida_typeinf.CM_CC_FASTCALL - ptr_tinfo = array_tinfo + rettype = self.get_tinfo_from_type(return_type) + if rettype is None: + print(f"Error: {func_name} @ {ea:#x} has unknown return type {return_type!r}") + return False + func_data.rettype = rettype + + for param in parameters: + arg = ida_typeinf.funcarg_t() + arg_type = self.get_tinfo_from_type(param.type) + if arg_type is None: + print(f"Error: {func_name} @ {ea:#x} has unknown type {param.type!r} on parameter {param.name!r}") + return False + arg.type = arg_type + arg.name = param.name + func_data.push_back(arg) - return ptr_tinfo + tif = ida_typeinf.tinfo_t() + if not tif.create_func(func_data): + print(f"Error: {func_name} @ {ea:#x} failed to create its function type") + return False + if not ida_typeinf.apply_tinfo(ea, tif, ida_typeinf.TINFO_DEFINITE): + print(f"Error: {func_name} @ {ea:#x} failed to apply {tif.dstr()!r}") + return False + return True + + def allow_template_type_names(self): + """Allow the characters clang puts in template instantiation names. + + Names such as 'std::vector>::_Reallocation_guard2' + contain spaces, which TypeNameChars rejects by default (and which the + idauser.cfg from this repository does not add). IDA then fails with + "Failed to prepare forward declaration ... is not acceptable", and a + single such error makes the clang parser drop every top level declaration. + """ + ida_idp.process_config_directive( + 'TypeNameChars = "~.<>?!/,*&[]-+ "' + '"_:$()`\'{}"' + '"0123456789"' + '"ABCDEFGHIJKLMNOPQRSTUVWXYZ"' + '"abcdefghijklmnopqrstuvwxyz";' + ) def select_parser(self): """Sets the parser used for building struct data.""" + self.allow_template_type_names() includes = "" if system() != "Windows": path = pick_folder(fields) @@ -186,11 +233,32 @@ def select_parser(self): f"-x c++ -target x86_64-pc-windows-msvc -fms-compatibility -fms-extensions -U__int64 -std=c++17{includes}", ) - def parse_string(self, decl: str, vtables: list[str]): - """""" - ida_srclang.parse_decls_with_parser("clang", None, decl, False) + def parse_string(self, decl: str, vtables: list[str]) -> int: + """Parse declarations with the clang parser and mark the vtable types. + + Returns: + int: The number of parser errors reported by IDA. + """ + error_count = ida_srclang.parse_decls_with_parser("clang", None, decl, False) + if error_count < 0: + raise RuntimeError( + "parse_decls_with_parser failed: the clang parser is not available" + ) + if error_count > 0: + print(f"Error: clang parser reported {error_count} error(s) while importing declarations") + else: + print("Clang parser imported declarations without errors") + + vtable_errors = [] for vtable in vtables: - self.mark_as_vtable(vtable) + error = self.mark_as_vtable(vtable) + if error is not None: + vtable_errors.append(error) + for error in vtable_errors: + print(f"Error: mark_as_vtable: {error}") + print(f"Marked {len(vtables) - len(vtable_errors)}/{len(vtables)} vtables ({len(vtable_errors)} failed)") + + return error_count def op_type_as_name(self, n: int): """ @@ -207,21 +275,19 @@ def op_type_as_name(self, n: int): if getattr(idc, item) == n: return item - def get_vft_from_data(self, class_name: str) -> dict | None: - class_data = self.data_yaml.classes.get(class_name) or {} - vtables = class_data.get("vtbls", []) - if not isinstance(vtables, list) or not vtables: + def get_vft_from_data(self, class_name: str) -> DefinedDataClassVtable | None: + class_data = self.data_yaml.classes.get(class_name) + if class_data is None or not class_data.vtbls: return None - vtable = vtables[0] - return vtable if isinstance(vtable, dict) else None - + return class_data.vtbls[0] + def get_vfunc_ea_from_data(self, class_name: str, offset: int) -> int: vtable = self.get_vft_from_data(class_name) - if vtable is None or "ea" not in vtable: + if vtable is None: return idc.BADADDR - vtable_ea = vtable["ea"] + idaapi.get_imagebase() - 0x140000000 + vtable_ea = vtable.ea + idaapi.get_imagebase() - 0x140000000 return idc.get_qword(vtable_ea + offset) def set_func_name(self, ea, name, cmt): @@ -276,16 +342,14 @@ def get_func_ea_by_sig(self, pattern: str): """ ea = self.search_binary(0, pattern, ida_search.SEARCH_DOWN) - if ida_funcs.get_func(ea) is None: - finf = ida_funcs.func_t() - finf.start_ea = ea - finf.end_ea = idc.BADADDR - ida_funcs.add_func_ex(finf) + if ida_funcs.get_func_start(ea) == idc.BADADDR: + ida_funcs.add_func(ea) - if ida_funcs.get_func(ea) is None: + func_start = ida_funcs.get_func_start(ea) + if func_start == idc.BADADDR: return idc.BADADDR - if ida_funcs.get_func(ea).start_ea == ea: + if func_start == ea: return ea mnem = idc.print_insn_mnem(ea) if not mnem: @@ -317,32 +381,16 @@ def update_member_func(self, member_func: DefinedStructMemFunc, struct: DefinedS return if ida_funcs.get_func_name(ea) == f"sub_{ea:X}": idc.set_name(ea, func_name) - tif = ida_typeinf.tinfo_t() - ida_typeinf.guess_tinfo(tif, ea) - func_data = ida_typeinf.func_type_data_t() - tif.get_func_details(func_data) - func_data.clear() - func_data.cc = ida_typeinf.CM_CC_FASTCALL - func_data.rettype = self.get_tinfo_from_type(member_func.return_type) - for param in member_func.parameters: - arg = ida_typeinf.funcarg_t() - try: - arg.type = self.get_tinfo_from_type(param.type) - except ValueError as exc: - print( - f"Error: update_member_func: function={func_name!r}, ea={ea:#x}, parameter={param.name!r}, type={param.type!r}, error={exc}" - ) - raise - arg.name = param.name - func_data.push_back(arg) - tif.create_func(func_data) - ida_typeinf.apply_tinfo(ea, tif, ida_typeinf.TINFO_DEFINITE) + self.apply_func_type(ea, func_name, member_func.return_type, member_func.parameters) def update_virt_func(self, virt_func: DefinedStructVFunc, struct: DefinedStruct): func_name = f"{struct.type}.{virt_func.name}" ea = self.get_func_ea_by_name(func_name) if ea == idc.BADADDR: + if self.get_vft_from_data(struct.type) is None: + # no vtable address in data.yml, so there is no slot to look up + return ea = self.get_vfunc_ea_from_data(struct.type, virt_func.offset) if ea in (0, idc.BADADDR): print(f"Error: {func_name} not found and its VFT slot could not be resolved") @@ -358,7 +406,7 @@ def update_virt_func(self, virt_func: DefinedStructVFunc, struct: DefinedStruct) return primary_vtable = self.get_vft_from_data(struct.type) - primary_base = primary_vtable.get("base") if primary_vtable else None + primary_base = primary_vtable.base if primary_vtable else None if primary_base: base_ea = self.get_vfunc_ea_from_data(primary_base, virt_func.offset) if base_ea == ea: @@ -369,35 +417,7 @@ def update_virt_func(self, virt_func: DefinedStructVFunc, struct: DefinedStruct) and not normalized_name.startswith(struct.type + ".") ): return - if self.get_tinfo_from_type(virt_func.return_type) is None: - raise ValueError(f"Got invalid return type on virtual function: {virt_func}") - - tif = ida_typeinf.tinfo_t() - ida_typeinf.guess_tinfo(tif, ea) - - func_data = ida_typeinf.func_type_data_t() - tif.get_func_details(func_data) - - func_data.clear() - func_data.cc = ida_typeinf.CM_CC_FASTCALL - func_data.rettype = self.get_tinfo_from_type(virt_func.return_type) - - for param in virt_func.parameters: - arg = ida_typeinf.funcarg_t() - try: - arg.type = self.get_tinfo_from_type(param.type) - except ValueError as exc: - print( - f"Error: update_virt_func: function={func_name!r}, ea={ea:#x}, " - f"parameter={param.name!r}, type={param.type!r}, error={exc}" - ) - raise - - arg.name = param.name - func_data.push_back(arg) - - tif.create_func(func_data) - ida_typeinf.apply_tinfo(ea, tif, ida_typeinf.TINFO_DEFINITE) + self.apply_func_type(ea, func_name, virt_func.return_type, virt_func.parameters) def update_static_member(self, static_member: DefinedStructStaticMember, struct: DefinedStruct): ea = self.search_binary( @@ -409,25 +429,18 @@ def update_static_member(self, static_member: DefinedStructStaticMember, struct: for follows in static_member.relative_offsets: ea = ea + follows ea = ea + 4 + self.get_dword(ea) - tif = ida_typeinf.tinfo_t() - ida_typeinf.guess_tinfo(tif, ea) + if ida_bytes.get_item_head(ea) != ea and static_member.return_type != struct.type: + # points into an already typed item (ex. a field of the struct instance) + return return_type = static_member.return_type if static_member.is_pointer: return_type = return_type + "*" - ida_typeinf.apply_tinfo( - ea, - self.get_tinfo_from_type(return_type), - ida_typeinf.TINFO_DEFINITE, - ) - if static_member.is_pointer: - ida_name.set_name( - ea, - f"g_{struct.type}_PtrInstance", - ) + tif = self.get_tinfo_from_type(return_type) + if tif is None: + print(f"Error: static member of {struct.type} @ {ea:#x} has unknown type {return_type!r}") else: - ida_name.set_name( - ea, f"g_{struct.type}_Instance" - ) + ida_typeinf.apply_tinfo(ea, tif, ida_typeinf.TINFO_DEFINITE) + ida_name.set_name(ea, f"g_{struct.type}_{static_member.name}") def mark_as_vtable(self, vtbl_name: str) -> str | None: """ @@ -452,7 +465,7 @@ def mark_as_vtable(self, vtbl_name: str) -> str | None: return f"Failed to retrieve UDT details for '{vtbl_name}'" if udt.taudt_bits & ida_typeinf.TAUDT_VFTABLE: - return f"'{vtbl_name}' is already marked as a vtable" + return None udt.taudt_bits |= ida_typeinf.TAUDT_VFTABLE @@ -461,9 +474,12 @@ def mark_as_vtable(self, vtbl_name: str) -> str | None: f"Failed to reconstruct UDT for '{vtbl_name}' after setting vtable flag" ) - result = tif.set_named_type(None, vtbl_name) + result = tif.set_named_type(None, vtbl_name, ida_typeinf.NTF_REPLACE) - if not result: - return f"Failed to write back '{vtbl_name}' to type library" + if result != ida_typeinf.TERR_OK: + return ( + f"Failed to write back '{vtbl_name}' to type library: " + f"{ida_typeinf.tinfo_errstr(result)}" + ) return None \ No newline at end of file diff --git a/scripts/ida/struct_import.py b/scripts/ida/struct_import.py index db8eabad8e..f72565a07a 100644 --- a/scripts/ida/struct_import.py +++ b/scripts/ida/struct_import.py @@ -212,12 +212,12 @@ def get_all_eas(pattern: str): for pattern_key in exd_comment_patterns: for ea in list(get_all_eas(pattern_key)): - try: - sheetIdx = ida.get_dword(ea + exd_comment_patterns[pattern_key]) - sheetName = exd_map[sheetIdx] - ida_bytes.set_cmt(ea, "Sheet: {0} ({1})".format(sheetName, sheetIdx), 0) - except Exception as e: - print(f"An unexpected error occurred at {hex(ea)}: {e}") + sheetIdx = ida.get_dword(ea + exd_comment_patterns[pattern_key]) + sheetName = exd_map.get(sheetIdx) + if sheetName is None: + print(f"Sheet getter @ 0x{ea:X} references unknown sheet {sheetIdx}!") + continue + ida_bytes.set_cmt(ea, "Sheet: {0} ({1})".format(sheetName, sheetIdx), 0) if _function_update_options["virtual"]: for struct in import_structs.structs: diff --git a/scripts/src_wrapper.py b/scripts/src_wrapper.py index 5a1f297a03..df30e488c8 100644 --- a/scripts/src_wrapper.py +++ b/scripts/src_wrapper.py @@ -5,6 +5,7 @@ DefinedStructExport, DefinedStructEnum, DefinedStructVFunc, + get_template_alias, ) from indent_writer import IndentWriter @@ -214,6 +215,7 @@ def get_size_from_type( "std::vector": 24, "std::set": 16, "std::map": 16, + "std::unordered_map": 64, "std::string": 32, "std::wstring": 32, "std::basic_string": 32, @@ -294,7 +296,7 @@ def _get_type_alignment( size = self.get_size_from_string(type_name) if size == 0: base_type, template_args = self._split_template_type(type_name) - if base_type in ("std::vector", "std::set", "std::map", "std::string", "std::wstring", "std::basic_string", "std::list", "std::deque", "std::linkedlist"): + if base_type in ("std::vector", "std::set", "std::map", "std::unordered_map", "std::string", "std::wstring", "std::basic_string", "std::list", "std::deque", "std::linkedlist"): return 8 if base_type == "std::pair": if len(template_args) != 2: @@ -463,7 +465,10 @@ def build_vfunc_signature( if vfunc.parameters: params = [] for param in vfunc.parameters: - if not self.is_base_type(param.type): + if param.type == "__fastcall": + # function pointer parameters are exported without their signature + params.append(f"void* {param.name}") + elif not self.is_base_type(param.type): params.append(f"::{param.type} {param.name}") else: params.append(f"{param.type} {param.name}") @@ -655,6 +660,22 @@ def iter_struct_tree(struct: DefinedStruct): if opaque_pointer_types: opaque_pointer_types_by_struct[struct_type] = opaque_pointer_types + # Windows headers define macros such as DrawState (-> DrawStateA) that + # would silently rename the types and members sharing their name + identifiers: set[str] = set() + for enum in [*export.enums, *inline_enums]: + identifiers.add(self.get_short_name(enum.name)) + for struct in all_structs: + identifiers.add(self.get_short_name(struct.type)) + identifiers.update(field.name for field in struct.fields) + identifiers.update( + virtual_function.name + for virtual_function in struct.virtual_functions or [] + ) + for identifier in sorted(identifiers): + if identifier.isidentifier(): + output.append(f"#undef {identifier}") + def append_chunk(chunk: IndentWriter | str): chunk_text = str(chunk) for line in chunk_text.split("\n"): @@ -733,7 +754,8 @@ def collect_inline_enum_definitions(struct: DefinedStruct): collect_inline_enum_definitions(inline_struct) def collect_vtable_names(struct: DefinedStruct): - if struct.virtual_functions: + # a template vtable only exists in IDA as its instantiations + if struct.virtual_functions and not struct.template_types: vtable_name = f"{struct.type}_vtbl" if vtable_name not in seen_vtable_names: seen_vtable_names.add(vtable_name) @@ -753,9 +775,6 @@ def collect_vtable_names(struct: DefinedStruct): self._normalize_type(struct.type), set() ), ) - if vtable_full_name and vtable_full_name not in seen_vtable_names: - seen_vtable_names.add(vtable_full_name) - vtable_names.append(vtable_full_name) collect_vtable_names(struct) collect_inline_enum_definitions(struct) forward_chunks.append((struct.namespace, forward)) @@ -773,8 +792,46 @@ def forward_chunk_key(key: tuple[str, IndentWriter]) -> str: append_ordered_chunks(definition_chunks) append_ordered_chunks(inline_enum_definition_chunks) + for alias, type_name in self._collect_template_aliases(export.structs).items(): + output.append(f"typedef ::{type_name} {alias};") + return (str(output), vtable_names) + def _collect_template_aliases(self, structs: list[DefinedStruct]) -> dict[str, str]: + """Collect the templated types used by function signatures. + + Returns: + dict: typedef name (see get_template_alias) to the type it aliases + """ + aliases: dict[str, str] = {} + + def collect(type_name: str | None): + type_name = (type_name or "").strip().rstrip("*").strip() + if "<" in type_name: + aliases.setdefault(get_template_alias(type_name), type_name) + + def collect_struct(struct: DefinedStruct): + # signatures of a template struct use its template parameters + if struct.template_types: + return + functions = [ + *(struct.virtual_functions or []), + *(struct.member_functions or []), + *(struct.static_member_functions or []), + ] + for function in functions: + collect(function.return_type) + for parameter in function.parameters or []: + collect(parameter.type) + for static_member in struct.static_members or []: + collect(static_member.return_type) + for inline_struct in struct.inline_structs: + collect_struct(inline_struct) + + for struct in structs: + collect_struct(struct) + return aliases + def build_enum_string( self, enum: DefinedStructEnum, @@ -1069,6 +1126,7 @@ def get_predef_header(self): #include #include #include +#include #include #include #include diff --git a/scripts/structs_schema.py b/scripts/structs_schema.py index 1705c7fcc8..a0858f6039 100644 --- a/scripts/structs_schema.py +++ b/scripts/structs_schema.py @@ -103,6 +103,17 @@ class DefinedStructExport: structs: list[DefinedStruct] +def get_template_alias(type_name: str) -> str: + """Name of the typedef the exported header declares for a templated type. + + The yml spells templated types differently than IDA names them + (std::vector is 'std::vector>' + in IDA), so they are found through a typedef with a predictable name instead. + """ + alias = "".join(char if char.isalnum() else "_" for char in type_name.replace(" ", "").replace("*", "_ptr")) + return f"__cs_{alias}" + + def get_yaml(stream, loader=Loader) -> DefinedStructExport: dic: YamlExport = load(stream, loader) enums = [] From 848e106ec6d78c9ec0efbe40a492eee9f5b0f5dd Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Tue, 13 Oct 2026 03:12:13 +0200 Subject: [PATCH 51/95] clean up some comment lines --- scripts/exdschema_wrapper.py | 3 - scripts/ida/ida_wrapper.py | 111 ++--------------------------------- scripts/ida/struct_import.py | 1 - scripts/src_wrapper.py | 34 +---------- scripts/structs_schema.py | 7 +-- 5 files changed, 9 insertions(+), 147 deletions(-) diff --git a/scripts/exdschema_wrapper.py b/scripts/exdschema_wrapper.py index b8970bf641..f93113aa70 100644 --- a/scripts/exdschema_wrapper.py +++ b/scripts/exdschema_wrapper.py @@ -97,9 +97,6 @@ def get_exdschema_data(version: str): def get_size_from_string(type: str) -> int: - """ - Gets the int of a base type string. - """ if type == "int8_t" or type == "uint8_t" or type == "bool": return 1 if type == "int16_t" or type == "uint16_t": diff --git a/scripts/ida/ida_wrapper.py b/scripts/ida/ida_wrapper.py index a63cce7c21..9815f241b0 100644 --- a/scripts/ida/ida_wrapper.py +++ b/scripts/ida/ida_wrapper.py @@ -70,38 +70,13 @@ def get_struct_path(self): return os.path.join(self.get_base_path(), "ffxiv_structs.yml") def get_enum_id(self, name: str): - """Get the id of an enum by its name - - Args: - name (str): The name of the enum - - Returns: - int: The id of the enum - """ - return idc.get_enum(name) def get_struct_id(self, name: str) -> int: - """Get the struct id - - Args: - name (str): The name of the struct - - Returns: - int: The struct id - """ return ida_typeinf.get_named_type_tid(name) def get_named_type(self, name: str): - """Retrieve a tinfo_t from the named type. - - Args: - name (str): Name of the type, or any declaration IDA can parse. - - Returns: - tinfo_t | None: The type, None if IDA does not know it. - """ - + """Returns None if IDA does not know the type.""" tinfo = ida_typeinf.tinfo_t() if tinfo.get_named_type(idaapi.get_idati(), name): return tinfo @@ -129,16 +104,7 @@ def get_named_type(self, name: str): return tinfo def get_tinfo_from_type(self, raw_type: str, array_size=0): - """Retrieve a tinfo_t from a raw type string. - - Args: - raw_type (str): Raw type string. - array_size (int, optional): Size of the array. Defaults to 0. - - Returns: - tinfo_t | None: The type, None if it could not be built. - """ - + """Returns None if the type could not be built.""" type = raw_type.rstrip("*") ptr_count = len(raw_type) - len(type) @@ -161,11 +127,7 @@ def get_tinfo_from_type(self, raw_type: str, array_size=0): return tinfo def apply_func_type(self, ea: int, func_name: str, return_type: str, parameters: list[DefinedStructFuncParam]) -> bool: - """Apply a __fastcall prototype built from the yml types to a function. - - Returns: - bool: False if one of the types is not known to IDA. - """ + """Returns False if one of the types is not known to IDA.""" func_data = ida_typeinf.func_type_data_t() func_data.cc = ida_typeinf.CM_CC_FASTCALL @@ -195,14 +157,7 @@ def apply_func_type(self, ea: int, func_name: str, return_type: str, parameters: return True def allow_template_type_names(self): - """Allow the characters clang puts in template instantiation names. - - Names such as 'std::vector>::_Reallocation_guard2' - contain spaces, which TypeNameChars rejects by default (and which the - idauser.cfg from this repository does not add). IDA then fails with - "Failed to prepare forward declaration ... is not acceptable", and a - single such error makes the clang parser drop every top level declaration. - """ + """clang puts spaces in template instantiation names, a single rejected name makes it drop every top level declaration.""" ida_idp.process_config_directive( 'TypeNameChars = "~.<>?!/,*&[]-+ "' '"_:$()`\'{}"' @@ -212,7 +167,6 @@ def allow_template_type_names(self): ) def select_parser(self): - """Sets the parser used for building struct data.""" self.allow_template_type_names() includes = "" if system() != "Windows": @@ -234,11 +188,7 @@ def select_parser(self): ) def parse_string(self, decl: str, vtables: list[str]) -> int: - """Parse declarations with the clang parser and mark the vtable types. - - Returns: - int: The number of parser errors reported by IDA. - """ + """Returns the number of parser errors reported by IDA.""" error_count = ida_srclang.parse_decls_with_parser("clang", None, decl, False) if error_count < 0: raise RuntimeError( @@ -261,16 +211,6 @@ def parse_string(self, decl: str, vtables: list[str]) -> int: return error_count def op_type_as_name(self, n: int): - """ - Retrieve the name of the operand type constant. - - Args: - n (int): The integer value of the operand type. - - Returns: - str: The name of the operand type constant if found; otherwise, None. - """ - for item in [x for x in dir(idc) if x.startswith("o_")]: if getattr(idc, item) == n: return item @@ -295,51 +235,17 @@ def set_func_name(self, ea, name, cmt): ida_bytes.set_cmt(ea, cmt, 0) def search_binary(self, ea: int, pattern: str, flag: int): - """Search for a pattern in a binary - - Args: - ea (int): The address to start searching from - pattern (str): The pattern to search for - flag (int): The search flags - - Returns: - int: The start address of the pattern - """ return ida_bytes.find_bytes( pattern, ea, None, ida_ida.inf_get_max_ea(), None, flag ) def get_dword(self, ea: int): - """Retrieve a dword (32-bit value) from the specified address. - - Args: - ea (int): The effective address to read the dword from. - - Returns: - int: The dword value read from the given address. - """ return ida_bytes.get_original_dword(ea) def get_func_ea_by_name(self, name: str): - """Retrieve the effective address of a function by its name. - - Args: - name (str): The name of the function. - - Returns: - int: The effective address of the function. - """ return ida_name.get_name_ea(0, name) def get_func_ea_by_sig(self, pattern: str): - """Retrieve the effective address of a function by its signature. - - Args: - pattern (str): The signature of the function. - - Returns: - int: The effective address of the function. - """ ea = self.search_binary(0, pattern, ida_search.SEARCH_DOWN) if ida_funcs.get_func_start(ea) == idc.BADADDR: @@ -443,12 +349,7 @@ def update_static_member(self, static_member: DefinedStructStaticMember, struct: ida_name.set_name(ea, f"g_{struct.type}_{static_member.name}") def mark_as_vtable(self, vtbl_name: str) -> str | None: - """ - Mark a struct as a vtable (TAUDT_VFTABLE). - - Returns: - An error string on failure, None on success. - """ + """Returns an error string on failure, None on success.""" tif = ida_typeinf.tinfo_t() if not tif.get_named_type(None, vtbl_name): diff --git a/scripts/ida/struct_import.py b/scripts/ida/struct_import.py index f72565a07a..30652da4e1 100644 --- a/scripts/ida/struct_import.py +++ b/scripts/ida/struct_import.py @@ -150,7 +150,6 @@ def do_pattern(pattern, suffix, sheetSearchPattern, sheetSearchOffset): while True: uniqueName = fnName + (f"_{uniquifier}" if uniquifier > 0 else "") - # check if this name is unique now if idc.get_name_ea_simple(uniqueName) == idc.BADADDR and uniquifier > 0: fnName = uniqueName break diff --git a/scripts/src_wrapper.py b/scripts/src_wrapper.py index df30e488c8..961315480c 100644 --- a/scripts/src_wrapper.py +++ b/scripts/src_wrapper.py @@ -25,9 +25,6 @@ def get_type_and_type_size_from_gap(self, gap: int, offset: int = 0) -> tuple[st return ("uint8_t", 1) def get_size_from_string(self, type: str) -> int: - """ - Gets the int of a base type string. - """ if type in ("int8_t", "uint8_t", "bool", "char", "unsigned char", "signed char"): return 1 if type in ("int16_t", "uint16_t", "wchar_t"): @@ -48,7 +45,6 @@ def is_base_type(self, type: str) -> bool: return type in ("byte", "wchar_t", "char", "float", "double", "bool", "int8_t", "uint8_t", "int16_t", "uint16_t", "int32_t", "uint32_t", "int64_t", "uint64_t", "void") def _split_template_type(self, type_name: str) -> tuple[str, list[str]]: - """Split a type's base name and top-level template arguments.""" type_name = type_name.strip() template_start = type_name.find("<") if template_start < 0: @@ -425,7 +421,6 @@ def get_string_define_and_size( return (f"{display_type} {field_name};", field_size) def get_short_name(self, full_type: str) -> str: - """Extract the short name from a fully qualified type name.""" depth = 0 last_namespace_separator = -1 index = 0 @@ -443,14 +438,6 @@ def get_short_name(self, full_type: str) -> str: def build_vfunc_signature( self, vfunc: DefinedStructVFunc, name_override: str | None = None ) -> str: - """Build a function pointer signature for a virtual function. - - Args: - vfunc: Virtual function definition - - Returns: - Function pointer signature string - """ function_name = name_override or vfunc.name if not (vfunc.return_type and vfunc.parameters): return f"void* {function_name};" @@ -460,7 +447,6 @@ def build_vfunc_signature( if not self.is_base_type(return_type): return_type = f"::{return_type}" - # Build parameter list param_list = "" if vfunc.parameters: params = [] @@ -681,7 +667,6 @@ def append_chunk(chunk: IndentWriter | str): for line in chunk_text.split("\n"): if not line.strip(): continue - # preserve the chunk's indentation by measuring leading spaces leading = len(line) - len(line.lstrip(" ")) chunk_indent = leading // 2 prev_indent = output.indent_level @@ -798,11 +783,7 @@ def forward_chunk_key(key: tuple[str, IndentWriter]) -> str: return (str(output), vtable_names) def _collect_template_aliases(self, structs: list[DefinedStruct]) -> dict[str, str]: - """Collect the templated types used by function signatures. - - Returns: - dict: typedef name (see get_template_alias) to the type it aliases - """ + """Map the typedef name (see get_template_alias) to the templated type for every function signature.""" aliases: dict[str, str] = {} def collect(type_name: str | None): @@ -875,14 +856,7 @@ def build_struct_string( struct_definition_lookup: dict[str, DefinedStruct] | None = None, opaque_pointer_types: set[str] | None = None, ) -> tuple[IndentWriter, IndentWriter, IndentWriter, str | None]: - """Builds the C++ struct definition with namespace wrapping. - - Args: - struct (DefinedStruct): The struct data to build - - Returns: - tuple: (forward_declarations, struct_definition, vtable_definition, vtable_full_name or None) - """ + """Returns (forward_declarations, struct_definition, vtable_definition, vtable_full_name or None).""" struct_string_forward: IndentWriter = IndentWriter() struct_string_define: IndentWriter = IndentWriter() struct_string_vtable: IndentWriter = IndentWriter() @@ -904,7 +878,6 @@ def build_struct_string( else: declspec = "" - # Forward declarations if template_declaration: struct_string_forward += template_declaration struct_string_forward += f"struct {struct.name};" @@ -914,7 +887,6 @@ def build_struct_string( struct_string_forward += template_declaration struct_string_forward += f"struct {short_name}_vtbl;" - # Struct definition if template_declaration: struct_string_define += template_declaration struct_define_line = f"struct {declspec}{short_name}" @@ -976,7 +948,6 @@ def append_chunk(chunk: IndentWriter | str): append_chunk(inline_define) append_chunk(inline_vtable) - # Add __vtable member if this struct has virtual functions if struct.virtual_functions and prev_size == 0: struct_string_define += f"{short_name}_vtbl{template_suffix}* __vtable;" prev_size = 8 @@ -1068,7 +1039,6 @@ def append_chunk(chunk: IndentWriter | str): struct_string_define.unindent() struct_string_define += "};" - # Virtual table struct definition if struct.virtual_functions: if template_declaration: struct_string_vtable += template_declaration diff --git a/scripts/structs_schema.py b/scripts/structs_schema.py index a0858f6039..43fb47dd05 100644 --- a/scripts/structs_schema.py +++ b/scripts/structs_schema.py @@ -104,12 +104,7 @@ class DefinedStructExport: def get_template_alias(type_name: str) -> str: - """Name of the typedef the exported header declares for a templated type. - - The yml spells templated types differently than IDA names them - (std::vector is 'std::vector>' - in IDA), so they are found through a typedef with a predictable name instead. - """ + """Typedef name the exported header declares for a templated type, as IDA spells template names differently than the yml.""" alias = "".join(char if char.isalnum() else "_" for char in type_name.replace(" ", "").replace("*", "_ptr")) return f"__cs_{alias}" From 0cc47130589b5e76b9acc005e7a9a26b67f373f6 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Tue, 13 Oct 2026 03:12:47 +0200 Subject: [PATCH 52/95] try to fix file lock when loading into IDA --- scripts/lumina_wrapper.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/scripts/lumina_wrapper.py b/scripts/lumina_wrapper.py index f58ce9dd62..a99272488d 100644 --- a/scripts/lumina_wrapper.py +++ b/scripts/lumina_wrapper.py @@ -6,19 +6,15 @@ from sys import platform, path from shutil import rmtree from xml.dom import pulldom -from pythonnet import load as dotnetload, unload +from pythonnet import load as dotnetload from json import load as jsonload from enum import IntEnum from dataclasses import dataclass import gc tempdirname = join(getcwd(), "luminatmp") -try: - rmtree(tempdirname) -except: - pass # ignore error -finally: - makedirs(tempdirname) +rmtree(tempdirname, ignore_errors=True) +makedirs(tempdirname, exist_ok=True) if platform.startswith("linux"): launcher_ini = join(getenv("HOME"), ".xlcore", "launcher.ini") @@ -162,6 +158,7 @@ def _isolated_clr(): from System.Runtime.Loader import AssemblyLoadContext # type: ignore from System import Activator, GC # type: ignore + from System.IO import File, MemoryStream # type: ignore ctx = AssemblyLoadContext("temp", isCollectible=True) @@ -170,8 +167,9 @@ def _isolated_extract(context): for file in listdir(tempdirname): if file.endswith(".dll"): - assemblies[file] = context.LoadFromAssemblyPath( - abspath(join(tempdirname, file)) + # LoadFromAssemblyPath would keep the file locked until the process exits + assemblies[file] = context.LoadFromStream( + MemoryStream(File.ReadAllBytes(abspath(join(tempdirname, file)))) ) assembly = assemblies["Lumina.dll"] @@ -216,8 +214,5 @@ def _isolated_extract(context): GC.WaitForPendingFinalizers() return excel_header_files - excel_header_files = _isolated_clr() - - unload() - - return excel_header_files + # pythonnet.unload() would break every later use of clr in the same process + return _isolated_clr() From 04dcab650638056cc808433256c287b3aa488ea9 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Tue, 13 Oct 2026 13:39:02 +0200 Subject: [PATCH 53/95] speedup import a bit --- CExporter/Exporter.cs | 75 +++++++++++++++++++++++++++++++++++++++ scripts/src_wrapper.py | 80 ++---------------------------------------- 2 files changed, 77 insertions(+), 78 deletions(-) diff --git a/CExporter/Exporter.cs b/CExporter/Exporter.cs index 3d281e160e..395be45941 100644 --- a/CExporter/Exporter.cs +++ b/CExporter/Exporter.cs @@ -5,6 +5,7 @@ using System.IO; using System.Linq; using System.Reflection; +using System.Text.RegularExpressions; using FFXIVClientStructs.Attributes; using InteropGenerator.Runtime.Attributes; using Pastel; @@ -348,6 +349,78 @@ HashSet resolve(ProcessedStruct s) { } } + [GeneratedRegex(@"[A-Za-z_]\w*(::[A-Za-z_]\w*)*")] + private static partial Regex TypeNameRegex(); + + // A nested type can't be forward declared in C++, so any reference to one (even through a pointer or a + // function signature) needs the struct that declares it to be emitted first. + private static void OrderByNestedTypeReferences(Dictionary> structToFullDeps) { + var inlineOwners = new Dictionary(); + var inlineTypes = _structs.SelectMany(t => t.ChildrenAndSelf.Skip(1)).Select(t => t.StructTypeName).ToHashSet(); + var topLevel = _structs.Where(t => !inlineTypes.Contains(t.StructTypeName)).ToArray(); + foreach (var owner in topLevel) + foreach (var inline in owner.ChildrenAndSelf.Skip(1)) + inlineOwners.TryAdd(inline.StructTypeName, owner.StructTypeName); + + var topLevelNames = topLevel.Select(t => t.StructTypeName).ToHashSet(); + var dependencies = new Dictionary>(); + foreach (var s in topLevel) { + var deps = new HashSet(); + foreach (var dependency in structToFullDeps[s.StructTypeName]) + addDependency(dependency); + foreach (var reference in s.ChildrenAndSelf.SelectMany(typeReferences)) + foreach (Match match in TypeNameRegex().Matches(reference)) + if (inlineOwners.ContainsKey(match.Value)) addDependency(match.Value); + dependencies[s.StructTypeName] = deps; + + void addDependency(string name) { + if (inlineOwners.TryGetValue(name, out var owner)) name = owner; + if (name != s.StructTypeName && topLevelNames.Contains(name)) deps.Add(name); + } + } + + // keeps the existing order wherever it already satisfies the dependencies + var remaining = topLevel.ToList(); + var ordered = new List(_structs.Count); + var emitted = new HashSet(); + while (remaining.Count > 0) { + var index = remaining.FindIndex(t => dependencies[t.StructTypeName].IsSubsetOf(emitted)); + if (index < 0) index = 0; // cyclic, the importer falls back to an opaque pointer for these + ordered.Add(remaining[index]); + emitted.Add(remaining[index].StructTypeName); + remaining.RemoveAt(index); + } + ordered.AddRange(_structs.Where(t => inlineTypes.Contains(t.StructTypeName))); + _structs = ordered; + + static IEnumerable typeReferences(ProcessedStruct s) { + foreach (var field in s.Fields) { + foreach (var reference in fieldReferences(field)) + yield return reference; + if (field is not ProcessedFunctionField functionField) continue; + yield return functionField.FunctionReturnType.FixTypeName(); + foreach (var reference in functionField.FunctionParameters.SelectMany(fieldReferences)) + yield return reference; + } + foreach (var virtualFunction in s.VirtualFunctions ?? []) { + if (virtualFunction.VirtualFunctionReturnType != null) + yield return virtualFunction.VirtualFunctionReturnTypeOverride ?? virtualFunction.VirtualFunctionReturnType.FixTypeName(); + foreach (var reference in (virtualFunction.VirtualFunctionParameters ?? []).SelectMany(fieldReferences)) + yield return reference; + } + } + + static IEnumerable fieldReferences(ProcessedField field) { + if (field.FieldType.IsFunctionPointer || field.FieldType.IsUnmanagedFunctionPointer) yield break; + yield return field.FieldTypeOverride ?? field.FieldType.FixTypeName(); + var templateArguments = field.FieldType.IsConstructedGenericType && !field.FieldType.IsStdDefiniton() + ? field.FieldType.GenericTypeArguments + : field.FieldTypeOverrideTemplates ?? []; + foreach (var templateArgument in templateArguments) + yield return templateArgument.FixTypeName(); + } + } + public static void Write(DirectoryInfo dir) { // make sure we have all the dependencies for each struct before we write them var structToFullDeps = ResolveFullDependencies(); @@ -356,6 +429,8 @@ public static void Write(DirectoryInfo dir) { return cmp != 0 ? cmp : string.Compare(a.StructTypeName, b.StructTypeName, StringComparison.Ordinal); }); + OrderByNestedTypeReferences(structToFullDeps); + void orderInlineStructs(ProcessedStruct parent) { if (parent.InlineStructs == null) return; parent.InlineStructs = [.. parent.InlineStructs diff --git a/scripts/src_wrapper.py b/scripts/src_wrapper.py index 961315480c..6f31b412b0 100644 --- a/scripts/src_wrapper.py +++ b/scripts/src_wrapper.py @@ -532,90 +532,14 @@ def collect_inline_type_owners(struct: DefinedStruct, owner_type: str): for struct in export.structs if self._normalize_type(struct.type) not in inline_struct_types ] - top_level_types = { - self._normalize_type(struct.type) for struct in top_level_structs - } - struct_dependencies = {type_name: set() for type_name in top_level_types} - - def references_inline_type(reference: str, inline_type: str) -> bool: - start = reference.find(inline_type) - while start >= 0: - end = start + len(inline_type) - before_is_identifier = start > 0 and ( - reference[start - 1].isalnum() or reference[start - 1] == "_" - ) - after_is_identifier = end < len(reference) and ( - reference[end].isalnum() or reference[end] == "_" - ) - if not before_is_identifier and not after_is_identifier: - return True - start = reference.find(inline_type, start + 1) - return False def iter_struct_tree(struct: DefinedStruct): yield struct for inline_struct in struct.inline_structs: yield from iter_struct_tree(inline_struct) - for struct in top_level_structs: - struct_type = self._normalize_type(struct.type) - references = [] - for nested_struct in iter_struct_tree(struct): - for field in nested_struct.fields: - references.append(field.type) - references.extend(field.template_args or []) - field_type = self._normalize_type(field.type) - if ( - field_type in struct_dependencies - and field_type != struct_type - and not field_type.endswith("*") - ): - struct_dependencies[struct_type].add(field_type) - if field.base: - if field_type in struct_dependencies and field_type != struct_type: - struct_dependencies[struct_type].add(field_type) - if hasattr(field, "return_type"): - references.append(field.return_type or "") - if getattr(field, "parameters", None): - references.extend( - parameter.type for parameter in field.parameters - ) - for virtual_function in nested_struct.virtual_functions or []: - references.append(virtual_function.return_type or "") - references.extend( - parameter.type - for parameter in virtual_function.parameters or [] - ) - - for inline_type, owner_type in inline_type_owners.items(): - if owner_type == struct_type or owner_type not in top_level_types: - continue - if any( - references_inline_type( - self._normalize_type(reference), inline_type - ) - for reference in references - ): - struct_dependencies[struct_type].add(owner_type) - - ordered_structs = [] - remaining_structs = list(top_level_structs) - emitted_types = set() - while remaining_structs: - ready_index = next( - ( - index - for index, struct in enumerate(remaining_structs) - if struct_dependencies[self._normalize_type(struct.type)] - <= emitted_types - ), - None, - ) - if ready_index is None: - ready_index = 0 - struct = remaining_structs.pop(ready_index) - ordered_structs.append(struct) - emitted_types.add(self._normalize_type(struct.type)) + # CExporter writes the structs in dependency order + ordered_structs = top_level_structs ordered_type_indices = { self._normalize_type(struct.type): index From 487c9262c8515261ed1425f9df7b8c0df65552a2 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Tue, 13 Oct 2026 13:41:50 +0200 Subject: [PATCH 54/95] remove testing script --- scripts/test_execute.py | 48 ----------------------------------------- 1 file changed, 48 deletions(-) delete mode 100644 scripts/test_execute.py diff --git a/scripts/test_execute.py b/scripts/test_execute.py deleted file mode 100644 index 34a7da9815..0000000000 --- a/scripts/test_execute.py +++ /dev/null @@ -1,48 +0,0 @@ -from time import time - -current_time = time() - -from structs_schema import get_yaml -from src_wrapper import SrcInterface -from exdschema_wrapper import get_exdschema_data, create_struct_from_header_and_schema -from lumina_wrapper import get_excel_header_files -from io import open - -src = SrcInterface() - -with open("ffxiv_structs.yml") as f: - struct_export = get_yaml(f) - -# for struct in struct_export.structs: -# for virtual_function in struct.virtual_functions: -# not_gui = not struct.namespace.endswith("GUI") -# not_agent = not struct.namespace.endswith("Agent") -# not_gui_or_agent = not_gui or not_agent -# not_receive_event = virtual_function.name != "ReceiveEvent" -# not_gui_or_agent_and_receive_event = not_gui_or_agent and not_receive_event -# not_dtor = virtual_function.name != "dtor" -# not_dtor_and_gui_or_agent_and_receive_event = not_dtor and not_gui_or_agent_and_receive_event -# not_offset = not virtual_function.offset -# if not_offset and not_dtor_and_gui_or_agent_and_receive_event: -# print(f"{struct.namespace}.{struct.name}_vtbl.{virtual_function.name}", not_offset, not_dtor_and_gui_or_agent_and_receive_event, not_dtor, not_gui_or_agent_and_receive_event, not_receive_event, not_gui_or_agent, not_agent, not_gui) - -header, vtables = src.build_export_string(struct_export) - -excel_header_files = get_excel_header_files() -exdschema = get_exdschema_data("latest") - -struct_export_excel, excel_map = create_struct_from_header_and_schema( - excel_header_files, exdschema -) - -header_excel, vtables_excel = src.build_export_string(struct_export_excel, 4) - -with open("test.h", "w") as f: - f.write(src.get_predef_header()) - f.write("\n") - f.write(header_excel) - f.write("\n") - f.write(header) - -running_time = time() - current_time -print(f"execution took {running_time}s") \ No newline at end of file From 5378f192a266631e4f925404e5528bd1483bf7f4 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:31:38 +0200 Subject: [PATCH 55/95] remove scripts no longer being maintained --- scripts/ffxiv_fullrun_ida.py | 32 ---------- scripts/ffxiv_luafuncs_ida.py | 109 ---------------------------------- 2 files changed, 141 deletions(-) delete mode 100644 scripts/ffxiv_fullrun_ida.py delete mode 100644 scripts/ffxiv_luafuncs_ida.py diff --git a/scripts/ffxiv_fullrun_ida.py b/scripts/ffxiv_fullrun_ida.py deleted file mode 100644 index ece68d0026..0000000000 --- a/scripts/ffxiv_fullrun_ida.py +++ /dev/null @@ -1,32 +0,0 @@ -from os import path -import idautils -import ida_auto -import idc -import ida_kernwin -import idaapi -import sys - -print("Loading ffxiv_idarename.py") -exec(open(path.join(path.dirname(path.realpath(__file__)), "ffxiv_idarename.py")).read()) - -print("Loading ffxiv_exdgetters.py") -exec(open(path.join(path.dirname(path.realpath(__file__)), "ffxiv_exdgetters.py")).read()) - -ida_base = ida_kernwin.ask_buttons( - "ffxiv_dx11.exe", - "__ImageBase", - "", - ida_kernwin.ASKBTN_YES, - "HIDECANCEL\nWhich name would you prefer from base offset?\n\nffxiv_dx11.exe : CheatEngine and a few other utilities will use the offset of ffxiv_dx11.exe+ as the base.\n__ImageBase : This is the default IDA behavior.", -) == ida_kernwin.ASKBTN_YES - -print("Loading ffxiv_structimporter.py") -exec(open(path.join(path.dirname(path.realpath(__file__)), "ffxiv_structimporter.py")).read()) - -if (ida_base): - idc.set_name(idaapi.get_imagebase(), "ffxiv_dx11.exe") - -for s in idautils.Segments(): - start = idc.get_segm_start(s) - end = idc.get_segm_end(s) - ida_auto.plan_range(start, end) \ No newline at end of file diff --git a/scripts/ffxiv_luafuncs_ida.py b/scripts/ffxiv_luafuncs_ida.py deleted file mode 100644 index eb7b3eac90..0000000000 --- a/scripts/ffxiv_luafuncs_ida.py +++ /dev/null @@ -1,109 +0,0 @@ -import idc -import ida_name -import ida_funcs - -namespace = "Client::Game::Event::Lua" - -def get_string_from_ptr(ea): - val = idc.get_strlit_contents(ea) - if val == None: - val = idc.get_strlit_contents(idc.get_qword(ea)) - return val.decode('utf-8') - -def get_asm_string(ea): - ins = idc.print_insn_mnem(ea) - - op = idc.print_operand(ea, 0) - if ins == 'push' or ins == 'call' or ins == 'jz': - val = idc.get_operand_value(ea, 0) - return {'ins': ins, 'op': op, 'val': val} - - val_text = idc.print_operand(ea, 1) - val = idc.get_operand_value(ea, 1) - if val_text.startswith('cs:'): - val_text = get_string_from_ptr(val) - return {'ins': ins, 'reg': op, 'val_text': val_text, 'val': val} - -def get_registers(ea): - regs = {} - loop = True - while loop: - asm = get_asm_string(ea) - if asm['ins'] == 'call' and asm['op'] == 'Common::Lua::LuaState.SetStringField': - loop = False - break - if asm['ins'] == 'mov': - regs[asm['reg']] = {'val_text': asm['val_text'], 'val': asm['val']} - ea = idc.next_head(ea) - return regs - -def get_func_registers(ea, reg_count): - regs = {} - while reg_count > 0: - ea = idc.prev_head(ea) - asm = get_asm_string(ea) - regs[asm['reg']] = {'val_text': asm['val_text'], 'val': asm['val']} - reg_count = reg_count - 1 - return regs - -def get_func_xrefs(ea): - eas = [] - xref = idc.get_first_cref_to(ea) - while xref != idc.BADADDR: - if idc.print_insn_mnem(xref).lower() == "call": - eas.append(xref) - xref = idc.get_next_cref_to(ea, xref) - return eas - -def set_func_name(ea, name, count: None | int = None): - act_name = name - if count != None: - act_name = f"{name}{count}" - - cur_name = idc.get_func_name(ea) - if cur_name.startswith('sub_'): - print(act_name) - found_ea = ida_name.get_name_ea(0, act_name) - if found_ea == idc.BADADDR or found_ea == ea: - idc.set_name(ea, act_name) - else: - set_func_name(ea, name, 2 if count is None else count + 1) - -def run(): - string_eas = get_func_xrefs(ida_name.get_name_ea(0, 'Common::Lua::LuaState.SetStringField')) - func_eas = get_func_xrefs(ida_name.get_name_ea(0, 'Common::Lua::LuaState.SetFunctionField')) - - string_eas = [ida_funcs.get_func(x).start_ea for x in string_eas] - - eas = {} - for x in func_eas: - regs = get_func_registers(x, 4) - regs['r8'] = get_string_from_ptr(regs['r8']['val']) - eas.setdefault(ida_funcs.get_func(x).start_ea, {'calls': []})['calls'].append({'off': x, 'regs': regs}) - - eas = {k: v for k,v in eas.items() if k in string_eas} - - for func_ea in eas: - eas[func_ea]['regs'] = get_registers(func_ea) - - for func_ea in eas: - eas[func_ea]['name'] = eas[func_ea]['regs'][eas[func_ea]['regs']['r9']['val_text']]['val_text'] - eas[func_ea]['name_val'] = eas[func_ea]['regs'][eas[func_ea]['regs']['r9']['val_text']]['val'] - - for func_ea in eas: - if eas[func_ea]['name'] == 'r8': - eas[func_ea]['name'] = idc.get_func_name(func_ea).split('.')[0].split(':')[-1] - eas[func_ea]['name_val'] = func_ea - if eas[func_ea]['name'] == 'rax': - eas[func_ea]['name_val'] = get_func_xrefs(func_ea)[0] - eas[func_ea]['name'] = idc.get_func_name(eas[func_ea]['name_val']).split('.')[0].split(':')[-1] - - for func_ea in eas: - name = f"{namespace}::{eas[func_ea]['name']}" - idc.set_name(func_ea, f"{name}.SetupLuaClass") - for call in eas[func_ea]['calls']: - regs = call['regs'] - func_name = regs['r8'] - set_func_name(regs['r9']['val'], f"{name}.{func_name}") - -run() \ No newline at end of file From c94dbc565dcf049563316435d469b6e2ed0258b3 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:32:12 +0200 Subject: [PATCH 56/95] add some more wrapper functions used in the rewrite --- scripts/ida/ida_wrapper.py | 216 ++++++++++++++++++++++++++++++++++--- 1 file changed, 202 insertions(+), 14 deletions(-) diff --git a/scripts/ida/ida_wrapper.py b/scripts/ida/ida_wrapper.py index 9815f241b0..4ea42dbc70 100644 --- a/scripts/ida/ida_wrapper.py +++ b/scripts/ida/ida_wrapper.py @@ -1,7 +1,7 @@ import idaapi import idc +import idautils import ida_bytes -import ida_nalt import ida_search import ida_ida import ida_idp @@ -9,12 +9,11 @@ import ida_name import ida_funcs import ida_srclang -import ida_kernwin +import ida_hexrays from structs_schema import * from form_builder import DynamicForm from data_schema import get_yaml, DefinedData, DefinedDataClassVtable -from abc import abstractmethod from platform import system from pathlib import Path from webbrowser import open as webopen @@ -52,6 +51,8 @@ def _get_ida_major() -> int: raise "Unsupported IDA version used. Need to be on IDA 9 or newer." class IdaInterface(object): + BADADDR = idc.BADADDR + data_yaml: DefinedData def get_base_path(self): @@ -151,11 +152,26 @@ def apply_func_type(self, ea: int, func_name: str, return_type: str, parameters: if not tif.create_func(func_data): print(f"Error: {func_name} @ {ea:#x} failed to create its function type") return False - if not ida_typeinf.apply_tinfo(ea, tif, ida_typeinf.TINFO_DEFINITE): + if not self.apply_tinfo(ea, tif): print(f"Error: {func_name} @ {ea:#x} failed to apply {tif.dstr()!r}") return False return True + def apply_tinfo(self, ea: int, tif) -> bool: + return ida_typeinf.apply_tinfo(ea, tif, ida_typeinf.TINFO_DEFINITE) + + def can_guess_func_type(self, ea: int) -> bool: + """Returns False if IDA can not work out the type of the function, even after decompiling it.""" + tif, funcdata = ida_typeinf.tinfo_t(), ida_typeinf.func_type_data_t() + + ida_typeinf.guess_tinfo(tif, ea) + if tif.get_func_details(funcdata): + return True + + ida_hexrays.decompile(ea) + ida_typeinf.guess_tinfo(tif, ea) + return tif.get_func_details(funcdata) + def allow_template_type_names(self): """clang puts spaces in template instantiation names, a single rejected name makes it drop every top level declaration.""" ida_idp.process_config_directive( @@ -227,18 +243,190 @@ def get_vfunc_ea_from_data(self, class_name: str, offset: int) -> int: if vtable is None: return idc.BADADDR - vtable_ea = vtable.ea + idaapi.get_imagebase() - 0x140000000 - return idc.get_qword(vtable_ea + offset) + vtable_ea = vtable.ea + self.get_image_base() - 0x140000000 + return self.get_qword(vtable_ea + offset) + + def set_func_name(self, ea, name, cmt=None, unique=False): + """ + :param unique: Only name functions IDA has not named yet, a number is appended if the name is taken by another address + """ + if unique: + if not idc.get_func_name(ea).startswith("sub_"): + return + base_name, count = name, 1 + while self.get_func_ea_by_name(name) not in (idc.BADADDR, ea): + count += 1 + name = f"{base_name}{count}" + print(name) - def set_func_name(self, ea, name, cmt): ida_name.set_name(ea, name) - ida_bytes.set_cmt(ea, cmt, 0) + if cmt is not None: + ida_bytes.set_cmt(ea, cmt, 0) + + def get_image_base(self) -> int: + return idaapi.get_imagebase() + + def is_offset(self, ea: int) -> bool: + return idc.is_off0(idc.get_full_flags(ea)) + + def xrefs_to(self, ea: int) -> list[int]: + return [xref.to for xref in idautils.XrefsTo(ea)] + + def get_qword(self, ea: int) -> int: + return idc.get_qword(ea) + + def get_addr_name(self, ea: int) -> str: + name = idc.get_name(ea) + # if name is None or '' and segment name is .text then create the function and get the name of it. + # this is done for parts of the code that has no xrefs to it due to how IDA detects function boundaries + if (name is None or name == '') and idc.get_segm_name(ea) == '.text': + ida_funcs.add_func(ea) + name = idc.get_name(ea) + return name + + def set_addr_name(self, ea: int, name: str) -> bool: + if ida_bytes.get_item_head(ea) != ea: + return True + result = idc.set_name(ea, name) + return bool(result) + + def get_comment(self, ea: int): + idc.get_cmt(ea, False) + + def set_comment(self, ea: int, comment: str): + idc.set_cmt(ea, comment, False) + + def get_func_name(self, ea: int) -> str: + """Name of the function containing ea.""" + return idc.get_func_name(ea) + + def get_func_start(self, ea: int) -> int: + """Start of the function containing ea.""" + return ida_funcs.get_func(ea).start_ea + + def get_string_from_ptr(self, ea: int) -> str: + val = idc.get_strlit_contents(ea) + if val == None: + val = idc.get_strlit_contents(idc.get_qword(ea)) + return val.decode('utf-8') + + def heads_before(self, ea: int, count: int): + """The instructions in front of ea, closest first, ex. the argument setup of a call.""" + for _ in range(count): + ea = idc.prev_head(ea) + yield ea + + def heads_until_call(self, ea: int, func_name: str): + """The instructions from ea up to the call of the named function.""" + while not (idc.print_insn_mnem(ea) == 'call' and idc.print_operand(ea, 0) == func_name): + yield ea + ea = idc.next_head(ea) + + def get_registers(self, eas, mnemonics=None) -> dict[str, dict[str, object]]: + """ + Collect what the instructions write to each register, keyed by register name. + :param eas: Instruction addresses to look at, later ones win + :param mnemonics: Only look at these instructions, all of them if omitted + """ + regs = {} + for ea in eas: + if mnemonics is not None and idc.print_insn_mnem(ea) not in mnemonics: + continue + val_text = idc.print_operand(ea, 1) + val = idc.get_operand_value(ea, 1) + if val_text.startswith('cs:'): + val_text = self.get_string_from_ptr(val) + regs[idc.print_operand(ea, 0)] = {'val_text': val_text, 'val': val} + return regs + + def get_func_xrefs(self, ea: int) -> list[int]: + """The addresses calling ea.""" + return [xref for xref in idautils.CodeRefsTo(ea, True) if idc.print_insn_mnem(xref).lower() == "call"] + + def format_class_name_for_vtbl(self, class_name: str) -> str: + return f"vtbl_{class_name}" + + def format_class_name_for_secondary_vtbl(self, class_name: str, secondary_name: str) -> str: + return f"vtbl_{class_name}___{secondary_name}" + + def format_class_name(self, class_name: str) -> str: + return class_name + + def format_vfunc_name(self, ea: int, current_func_name: str, proposed_func_name: str, class_name: str, base_class_names: list[str]) -> str | None: + if current_func_name.startswith("j_"): # jump + current_func_name = current_func_name.lstrip("j_") + + if current_func_name.startswith("qword_"): + idc.auto_mark_range(ea, ea + 1, idc.AU_CODE) + idc.create_insn(ea) + idc.add_func(ea) + current_func_name = self.get_addr_name(ea) + print(f"Info: qword in vtbl of {class_name} at 0x{ea:X}, it may be an offset to undefined code") + + # Previously renamed as a vfunc + if current_func_name.startswith(class_name): + # return the proposed func name in case it was updated since last run + current_class_name = current_func_name.rsplit(".", 1)[0] + return f"{current_class_name}.{proposed_func_name}" + + # This should have been handled in the base class + if any(current_func_name.startswith(name) for name in base_class_names): + return "" + + if current_func_name.startswith("sub_"): + return f"{class_name}.{proposed_func_name}" + + if current_func_name.startswith("unknown_libname_"): + return f"{class_name}.{proposed_func_name}" + + if current_func_name.startswith("nullsub_"): + return f"{class_name}.{proposed_func_name}_nullsub" + + if current_func_name.startswith("loc_"): + return f"{class_name}.{proposed_func_name}_loc" + + if current_func_name.startswith("locret_"): + return f"{class_name}.{proposed_func_name}_locret" + + # Name it later in a child class when it gets overridden + if current_func_name == "_purecall": + return "" + + # Mangled func names, thanks IDA + if current_func_name.startswith("?") or current_func_name.startswith("_"): + return f"{class_name}.{proposed_func_name}" - def search_binary(self, ea: int, pattern: str, flag: int): + return None + + def format_func_name(self, ea: int, current_func_name: str, proposed_func_name: str, class_name: str) -> str | None: + if current_func_name.startswith("j_"): # jump + current_func_name = current_func_name.lstrip("j_") + + # Mangled func names, thanks IDA + if proposed_func_name.startswith("?") or proposed_func_name.startswith("_"): + return proposed_func_name + + proposed_qualified_func_name = f"{class_name}.{proposed_func_name}" + if current_func_name == proposed_qualified_func_name: + return "" + + if any(current_func_name.startswith(prefix) for prefix in ("sub_", "nullsub_", "loc_", "qword_", "unknown_libname_", "?", "_")): + return proposed_qualified_func_name + + return None + + def search_binary(self, ea: int, pattern: str, flag: int = ida_search.SEARCH_DOWN): return ida_bytes.find_bytes( pattern, ea, None, ida_ida.inf_get_max_ea(), None, flag ) + def search_binary_all(self, pattern: str, ea: int = 0): + """Every address matching the pattern, starting from ea.""" + ea = self.search_binary(ea, pattern) + while ea != idc.BADADDR: + yield ea + ea = self.search_binary(ea + 1, pattern) + def get_dword(self, ea: int): return ida_bytes.get_original_dword(ea) @@ -345,7 +533,7 @@ def update_static_member(self, static_member: DefinedStructStaticMember, struct: if tif is None: print(f"Error: static member of {struct.type} @ {ea:#x} has unknown type {return_type!r}") else: - ida_typeinf.apply_tinfo(ea, tif, ida_typeinf.TINFO_DEFINITE) + self.apply_tinfo(ea, tif) ida_name.set_name(ea, f"g_{struct.type}_{static_member.name}") def mark_as_vtable(self, vtbl_name: str) -> str | None: @@ -361,14 +549,14 @@ def mark_as_vtable(self, vtbl_name: str) -> str | None: ) return f"'{vtbl_name}' is not a struct (got {actual})" + if tif.is_vftable(): + return None + udt = ida_typeinf.udt_type_data_t() if not tif.get_udt_details(udt): return f"Failed to retrieve UDT details for '{vtbl_name}'" - if udt.taudt_bits & ida_typeinf.TAUDT_VFTABLE: - return None - - udt.taudt_bits |= ida_typeinf.TAUDT_VFTABLE + udt.set_vftable() if not tif.create_udt(udt, ida_typeinf.BTF_STRUCT): return ( From 4af837a2ae63dc04828b831aa69b50d5da37225c Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:32:48 +0200 Subject: [PATCH 57/95] split ida parts from rename to seperate folder --- scripts/class_factory.py | 386 +++++++++++++++++++++++++ scripts/data_schema.py | 7 +- scripts/ffxiv_idarename.py | 559 +------------------------------------ scripts/ida/rename.py | 60 ++++ 4 files changed, 455 insertions(+), 557 deletions(-) create mode 100644 scripts/class_factory.py create mode 100644 scripts/ida/rename.py diff --git a/scripts/class_factory.py b/scripts/class_factory.py new file mode 100644 index 0000000000..9edb193763 --- /dev/null +++ b/scripts/class_factory.py @@ -0,0 +1,386 @@ +from __future__ import annotations + +import itertools +from collections import deque +from dataclasses import dataclass, field +from typing import Any + +from anytree import Node, RenderTree, PreOrderIter + +from data_schema import DefinedData, DefinedDataClass, DefinedDataClassInstance, DefinedDataClassVtable + +STANDARD_IMAGE_BASE = 0x140000000 + + +@dataclass(eq=False) +class Vtbl: + """ + Object representing a class's vtbl + """ + # Address of vtbl + ea: int + # Name of the base class this vtbl inherits, if it exists + base_name: str | None = None + # Explicit size of vtbl used to override cases where size detection fails + size: int | None = None + resolved_base: FfxivClass | None = None + + @classmethod + def from_data(cls, vtbl_data: DefinedDataClassVtable, rebase_offset: int = 0) -> Vtbl: + """ + Create a vtbl from its data.yml definition + :param vtbl_data: Vtbl definition as contained in data.yml + :param rebase_offset: Offset to apply to the address if the program has been rebased + """ + return cls(ea=vtbl_data.ea + rebase_offset, base_name=vtbl_data.base, size=vtbl_data.size) + + +@dataclass(eq=False, repr=False) +class FfxivClass: + """ + Object representing a class + """ + # Disassembler api, see BaseApi in ffxiv_idarename.py + api: Any + name: str + vtbls: list[Vtbl] = field(default_factory=list) + # Mapping of vtbl index to func names + vfuncs: dict[int, str] = field(default_factory=dict) + # Mapping of effective addresses to func names + funcs: dict[int, str] = field(default_factory=dict) + # List of global instances of the object + instances: list[DefinedDataClassInstance] = field(default_factory=list) + first_base: FfxivClass | None = None + + _main_vtbl_size: int = 0 + _inheritance_tree: Node | None = None + _finalized: bool = False + + @classmethod + def from_data(cls, api: Any, class_name: str, class_data: DefinedDataClass) -> FfxivClass: + """ + Create a class from its data.yml definition + :param api: Disassembler api + :param class_name: Class name + :param class_data: Class definition as contained in data.yml + """ + # Offset the vtbl and funcs if the program has been rebased + rebase_offset = api.get_image_base() - STANDARD_IMAGE_BASE + + return cls( + api=api, + name=class_name, + vtbls=[Vtbl.from_data(vtbl, rebase_offset) for vtbl in class_data.vtbls], + vfuncs={vfunc.num: vfunc.name for vfunc in class_data.vfuncs}, + funcs={func.num + rebase_offset: func.name for func in class_data.functions}, + instances=class_data.instances, + ) + + # region vtbl_size + + @property + def main_vtbl_size(self) -> int: + """ + Iterate from the vtbl start until a non-offset or xref is encountered. + This strategy implies that the only xref in a vtbl is the first vfunc. + Can be overriden by explicit size in the vtbl definition. + :return: VTable func count + """ + if not self.vtbls: + return self._main_vtbl_size + + if self.vtbls[0].size: + self._main_vtbl_size = self.vtbls[0].size + return self._main_vtbl_size + + if self._main_vtbl_size == 0: + self._main_vtbl_size = 1 # Set to 1, skip the first entry + for ea in itertools.count(self.vtbls[0].ea + 8, 8): + if self.api.get_addr_name(ea) != '': + break + + if self.api.is_offset(ea) and self.api.xrefs_to(ea) == []: + self._main_vtbl_size += 1 + else: + break + + if self.vtbls[0].resolved_base and self._main_vtbl_size < self.vtbls[0].resolved_base._main_vtbl_size: + print( + "Error: The sum of \"{0}\"'s base vtbl sizes ({1}) is greater than the actual class itself ({2})" + .format(self.name, self.vtbls[0].resolved_base._main_vtbl_size, self._main_vtbl_size)) + + return self._main_vtbl_size + + # endregion + + @property + def inheritance_tree(self) -> Node: + def recurse_tree(current_node: Node, current_class: FfxivClass, all_bases: set[str]): + for vtbl in current_class.vtbls: + if vtbl.resolved_base and vtbl.resolved_base.name not in all_bases: + new_node = Node(vtbl.resolved_base.name, parent=current_node) + all_bases.add(vtbl.resolved_base.name) + recurse_tree(new_node, vtbl.resolved_base, all_bases) + + if not self._inheritance_tree: + self._inheritance_tree = Node(self.name) + base_set = set() + recurse_tree(self._inheritance_tree, self, base_set) + + return self._inheritance_tree + + # region finalized + + @property + def finalized(self) -> bool: + """ + Has this class and its hierarchy been written out or not + :return: bool yes/no + """ + if self.vtbls: + return self._finalized and all( + vtbl.resolved_base is not None and vtbl.resolved_base._finalized for vtbl in self.vtbls) + else: + return self._finalized + + @finalized.setter + def finalized(self, value: bool): + """ + Set if finalized or not + :param value: Finalized state + """ + self._finalized = value + + # endregion + + # region finalize + + def finalize(self): + """ + Write out this class + """ + self._inherit_func_names_from_main_base() + self._comment_vtbls_with_inheritance_tree() + + self._write_vtbl_functions() + self._write_funcs() + + self._write_instances() + + self.finalized = True + + def _inherit_func_names_from_main_base(self): + """ + A base is guaranteed to be finalized before the child, + so a base has all the vfunc names of its base already. + """ + if self.vtbls and self.vtbls[0].resolved_base: + for idx, base_vfunc_name in self.vtbls[0].resolved_base.vfuncs.items(): + if idx in self.vfuncs: + print("Warning: 0x{0:X} \"{1}\" overwrites the name of inherited function \"{2}\"".format( + self.vtbls[0].ea, self.name, base_vfunc_name)) + else: + self.vfuncs[idx] = base_vfunc_name + + def _comment_vtbls_with_inheritance_tree(self): + """ + Adds the inheritance tree as a comment to the start of the vtbl. + grandbase_name + base_name + self_name + """ + for vtbl in self.vtbls: + comment = self.api.get_comment(vtbl.ea) or "" + + for pre, _, node in RenderTree(self.inheritance_tree): + if comment: + comment += "\n" + comment += pre + self.api.format_class_name(node.name) + self.api.set_comment(vtbl.ea, comment) + + def write_vtbl_names(self): + """ + Write out the vtbl name. + """ + self.api.set_addr_name(self.vtbls[0].ea, self.api.format_class_name_for_vtbl(self.name)) + for vtbl in self.vtbls[1:]: + self.api.set_addr_name(vtbl.ea, self.api.format_class_name_for_secondary_vtbl(self.name, + vtbl.base_name)) + + def _write_vtbl_functions(self): + """ + Write out the vtbl function names + """ + + def collect_vtbl_functions(ea: int, size: int, class_name: str, vfunc_names: dict[int, str], + base_class_names: list[str]) -> list[tuple[int, str]]: + vtbl_builder = [] + # Iterate through each offset + for func_idx in range(0, size): + vtbl_vfunc_ea = ea + func_idx * 8 + vfunc_ea = self.api.get_qword(vtbl_vfunc_ea) # type: int + + current_func_name = self.api.get_addr_name(vfunc_ea) # type: str + proposed_func_name = vfunc_names.get(func_idx, "vf{0}".format(func_idx)) + formatted_class_name = self.api.format_class_name(class_name) + + formatted_func_name = self.api.format_vfunc_name(vfunc_ea, current_func_name, proposed_func_name, + formatted_class_name, + base_class_names) + + if formatted_func_name == "": + pass + elif formatted_func_name is None: + print( + "Error: Function at 0x{0:X} had unexpected name \"{1}\" during naming of {2}.{3} (vtbl[{4}])" + .format(vfunc_ea, current_func_name, self.name, proposed_func_name, func_idx)) + else: + vtbl_builder.append((vfunc_ea, formatted_func_name)) + + return vtbl_builder + + if self.vtbls: + formatted_base_class_names = [self.api.format_class_name(node.name) for node in + PreOrderIter(self.inheritance_tree)] + formatted_base_class_names.remove(self.name) + for idx, vtbl in enumerate(self.vtbls): + if idx == 0: + funcs = collect_vtbl_functions(vtbl.ea, self.main_vtbl_size, self.name, self.vfuncs, + formatted_base_class_names) + else: + funcs = collect_vtbl_functions(vtbl.ea, vtbl.resolved_base.main_vtbl_size, + self.name + "___" + vtbl.resolved_base.name, + vtbl.resolved_base.vfuncs, formatted_base_class_names) + for (func_ea, func_name) in funcs: + self.api.set_addr_name(func_ea, func_name) + + def _write_funcs(self): + """ + Write the names of all non-vtbl funcs + """ + for func_ea, proposed_func_name in self.funcs.items(): + current_func_name = self.api.get_addr_name(func_ea) # type: str + + func_name = self.api.format_func_name(func_ea, current_func_name, proposed_func_name, self.name) + if func_name == "": + pass + elif func_name is None: + print("Error: Function at 0x{0:X} had unexpected name \"{1}\" during naming of {2}.{3}" + .format(func_ea, current_func_name, self.name, proposed_func_name)) + else: + self.api.set_addr_name(func_ea, func_name) + + def _write_instances(self): + """ + Write the names of all instances + """ + for instance in self.instances: + name = "g_{}_{}".format(self.name, instance.name or "Instance") + self.api.set_addr_name(instance.ea, name) + + # endregion + + def __repr__(self): + return "<{0}(\"{1}\")>".format(self.__class__.__name__, self.name) + + +@dataclass +class FfxivClassFactory: + # Disassembler api, see BaseApi in ffxiv_idarename.py + api: Any + _vtbl_addresses: list[int] = field(default_factory=list) + _classes: dict[str, FfxivClass] = field(default_factory=dict) + _finalize_stack: deque[FfxivClass] = field(default_factory=deque) + + def register(self, class_name: str, class_data: DefinedDataClass | None = None): + """ + Register a class + :param class_name: Class name + :param class_data: Class definition as contained in data.yml, omit for a placeholder entry + """ + if class_data is None: + class_data = DefinedDataClass() + + for vtbl in class_data.vtbls: + if vtbl.ea != 0x0 and vtbl.ea in self._vtbl_addresses: + print("Error: Multiple vtables are defined at 0x{0:X}".format(vtbl.ea)) + return + + if class_name in self._classes: + print("Error: Multiple classes are registered with the name \"{0}\"".format(class_name)) + return + for vtbl in class_data.vtbls: + self._vtbl_addresses.append(vtbl.ea) + + self._classes[class_name] = FfxivClass.from_data(self.api, class_name, class_data) + + def finalize(self): + """ + Perform the class naming + """ + self._resolve_base_classes() + + # We write the vtbl names first for an added check so that + # the size finder will not advance past a named offset. + for cls in self._classes.values(): + if cls.vtbls: + cls.write_vtbl_names() + + for cls in self._classes.values(): + self._finalize_class(cls) + + def _resolve_base_classes(self): + """ + Resolve base classes for a class + If missing, warn the user and add a stub entry. + """ + for cls in list(self._classes.values()): + for idx, vtbl in enumerate(cls.vtbls): + if vtbl.resolved_base is None and vtbl.base_name: + if vtbl.base_name not in self._classes: + print("Warning: Inherited class \"{0}\" is not documented, add a placeholder entry".format( + vtbl.base_name)) + self.register(class_name=vtbl.base_name) + vtbl.resolved_base = self._classes[vtbl.base_name] + if idx == 0: + cls.first_base = self._classes[vtbl.base_name] + + def _finalize_class(self, cls: FfxivClass): + """ + Perform a single class naming + :param cls: Class object + """ + if cls in self._finalize_stack: + names = [c.name for c in self._finalize_stack] + [cls.name] + names = "\n".join([" - {0}".format(name) for name in names]) + raise ValueError("Inheritance cycle detected: \n{0}".format(names)) + + if not cls.finalized: + self._finalize_stack.append(cls) + for vtbl in cls.vtbls: + if vtbl.resolved_base and not vtbl.resolved_base.finalized: + self._finalize_class(vtbl.resolved_base) + cls.finalize() + + self._finalize_stack.pop() + + +def apply_data(api: Any, data: DefinedData): + """ + Name everything contained in data.yml + :param api: Disassembler api + :param data: Parsed data.yml + """ + for section in (data.globals, data.functions): + for ea, name in (section or {}).items(): + if not isinstance(ea, int): + print('Warning: {0} has an invalid address {1}'.format(name, ea)) + continue + api.set_addr_name(ea, name) + + if data.classes: + factory = FfxivClassFactory(api) + for class_name, class_data in data.classes.items(): + factory.register(class_name, class_data) + + factory.finalize() diff --git a/scripts/data_schema.py b/scripts/data_schema.py index 4c58ce7d0f..56fde2c441 100644 --- a/scripts/data_schema.py +++ b/scripts/data_schema.py @@ -43,6 +43,7 @@ class DefinedData: classes: dict[str, DefinedDataClass] = field(default_factory=dict) version: str | None = None globals: dict[int, str] = field(default_factory=dict) + functions: dict[int, str] = field(default_factory=dict) YamlExport: TypeAlias = dict[str, object] @@ -51,10 +52,14 @@ class DefinedData: def get_yaml(stream, loader=Loader) -> DefinedData: dic: YamlExport = load(stream, loader) classes = {} - for name, class_values in dic.get("classes", {}).items(): + class_keys = {f.metadata.get("yaml_name", f.name) for f in fields(DefinedDataClass)} + for name, class_values in (dic.get("classes") or {}).items(): class_values = class_values or {} values = dict(class_values) + for leftover in class_values.keys() - class_keys: + print('Warning: Extra key "{0}" present in {1}'.format(leftover, name)) + for key, value_type in ( ("instances", DefinedDataClassInstance), ("vtbls", DefinedDataClassVtable), diff --git a/scripts/ffxiv_idarename.py b/scripts/ffxiv_idarename.py index 166a61c938..ef59e953ba 100644 --- a/scripts/ffxiv_idarename.py +++ b/scripts/ffxiv_idarename.py @@ -5,8 +5,6 @@ from __future__ import print_function import os -import yaml -from anytree import Node, RenderTree, PreOrderIter try: from typing import Any, Dict, List, Optional, Union # noqa @@ -14,14 +12,11 @@ pass import sys -import itertools from abc import abstractmethod -from collections import deque if sys.version_info[0] >= 3: long = int - # region Api class BaseApi(object): @@ -198,123 +193,10 @@ def format_func_name(self, ea, current_func_name, proposed_func_name, class_name if api is None: try: import idaapi # noqa - import idc # noqa - import idautils # noqa - import ida_funcs # noqa - import ida_bytes # noqa except ImportError: print("Warning: Unable to load IDA") else: - # noinspection PyUnresolvedReferences - class IdaApi(BaseApi): - - @property - def data_file_path(self): - return os.path.join(os.path.dirname(os.path.realpath(__file__)), "data.yml") - - def get_image_base(self): - return idaapi.get_imagebase() - - def is_offset(self, ea): - return idc.is_off0(idc.get_full_flags(ea)) - - def xrefs_to(self, ea): - return [xref.to for xref in idautils.XrefsTo(ea)] - - def get_qword(self, ea): - return idc.get_qword(ea) - - def get_addr_name(self, ea): - name = idc.get_name(ea) - # if name is None or '' and segment name is .text then create the function and get the name of it. - # this is done for parts of the code that has no xrefs to it due to how IDA detects function boundaries - if (name is None or name == '') and idc.get_segm_name(ea) == '.text': - finf = ida_funcs.func_t() - finf.start_ea = ea - finf.end_ea = idc.BADADDR - ida_funcs.add_func_ex(finf) - name = idc.get_name(ea) - return name - - def set_addr_name(self, ea, name): - # print("{0} {1}".format(ea, name)) - if ida_bytes.get_item_head(ea) != ea: - return True - result = idc.set_name(ea, name) - return bool(result) - - def get_comment(self, ea): - idc.get_cmt(ea, False) - - def set_comment(self, ea, comment): - idc.set_cmt(ea, comment, False) - - def format_vfunc_name(self, ea, current_func_name, proposed_func_name, class_name, base_class_names): - if current_func_name.startswith("j_"): # jump - current_func_name = current_func_name.lstrip("j_") - - if current_func_name.startswith("qword_"): - idc.auto_mark_range(ea, ea + 1, idc.AU_CODE) - idc.create_insn(ea) - idc.add_func(ea) - current_func_name = api.get_addr_name(ea) - print("Info: qword in vtbl of {1} at 0x{0:X}, it may be an offset to undefined code".format(ea, - class_name)) - - # Previously renamed as a vfunc - if current_func_name.startswith(class_name): - # return the proposed func name in case it was updated since last run - current_class_name = current_func_name.rsplit(".", 1)[0] - return "{0}.{1}".format(current_class_name, proposed_func_name) - - # This should have been handled in the base class - if any(current_func_name.startswith(name) for name in base_class_names): - return "" - - if current_func_name.startswith("sub_"): - return "{0}.{1}".format(class_name, proposed_func_name) - - if current_func_name.startswith("unknown_libname_"): - return "{0}.{1}".format(class_name, proposed_func_name) - - if current_func_name.startswith("nullsub_"): - return "{0}.{1}_nullsub".format(class_name, proposed_func_name) - - if current_func_name.startswith("loc_"): - return "{0}.{1}_loc".format(class_name, proposed_func_name) - - if current_func_name.startswith("locret_"): - return "{0}.{1}_locret".format(class_name, proposed_func_name) - - # Name it later in a child class when it gets overridden - if current_func_name == "_purecall": - return "" - - # Mangled func names, thanks IDA - if current_func_name.startswith("?") or current_func_name.startswith("_"): - return "{0}.{1}".format(class_name, proposed_func_name) - - return None - - def format_func_name(self, ea, current_func_name, proposed_func_name, class_name): - if current_func_name.startswith("j_"): # jump - current_func_name = current_func_name.lstrip("j_") - - # Mangled func names, thanks IDA - if proposed_func_name.startswith("?") or proposed_func_name.startswith("_"): - return proposed_func_name - - proposed_qualified_func_name = "{0}.{1}".format(class_name, proposed_func_name) - if current_func_name == proposed_qualified_func_name: - return "" - - if any(current_func_name.startswith(prefix) for prefix in ("sub_", "nullsub_", "loc_", "qword_", "unknown_libname_", "?", "_")): - return proposed_qualified_func_name - - return None - - - api = IdaApi() + raise ValueError("Script moved to new location in ./ida/rename.py") # endregion # region Ghidra Api @@ -426,445 +308,10 @@ def format_func_name(self, ea, current_func_name, proposed_func_name, class_name def load_data(): with open(api.data_file_path, "r") as fd: - data = yaml.safe_load(fd) - - if data.get("globals"): - for ea, name in data["globals"].items(): - if not isinstance(ea, (int, long)): - print('Warning: {0} has an invalid address {1}'.format(name, ea)) - continue - api.set_addr_name(ea, name) - - if data.get("functions"): - for ea, name in data["functions"].items(): - if not isinstance(ea, (int, long)): - print('Warning: {0} has an invalid address {1}'.format(name, ea)) - continue - api.set_addr_name(ea, name) - - - if data.get("classes"): - factory = FfxivClassFactory() - for class_name, class_data in data["classes"].items(): - if not class_data: - class_data = {} - - vtbls_raw = class_data.pop("vtbls", []) - vtbls = [(vtbl["ea"], vtbl["base"] if "base" in vtbl else None, vtbl["size"] if "size" in vtbl else None) for vtbl in vtbls_raw] - vfuncs = class_data.pop("vfuncs", {}) - funcs = class_data.pop("funcs", {}) - instances_raw = class_data.pop("instances", []) - instances = [(instance["ea"], instance["name"] if "name" in instance else "Instance") for instance in instances_raw] if instances_raw is not None else [] - for leftover in class_data: - print("Warning: Extra key \"{0}\" present in {1}".format(leftover, class_name)) - - factory.register( - class_name=class_name, vtbls=vtbls, vfuncs=vfuncs, funcs=funcs, instances=instances) - - factory.finalize() - - -class FfxivClassFactory: - _vtbl_addresses = [] # type: List[int] - _classes = {} # type: Dict[str, FfxivClass] - - def register(self, class_name, vtbls=None, vfuncs=None, funcs=None, instances=None): - """ - Register a class - :param class_name: Class name - :type class_name: str - :param vtbls: List of (vtbl_ea, base_name) pairs - :type vtbls: list[(int, str)] - :param funcs: Mapping of effective addresses to func names - :type funcs: Dict[int, str] - :param vfuncs: Mapping of vtbl index to func names - :type funcs: Dict[int, str] - :param instances: List of global instances of the object - :type instances: List[(int, str)] - :return: None - :rtype: None - """ - if vtbls is None: - vtbls = [] - - if not vfuncs: - vfuncs = {} - - if not funcs: - funcs = {} - - for (vtbl_ea, _, _) in vtbls: - if vtbl_ea != 0x0 and vtbl_ea in self._vtbl_addresses: - print("Error: Multiple vtables are defined at 0x{0:X}".format(vtbl_ea)) - return - - if class_name in self._classes: - print("Error: Multiple classes are registered with the name \"{0}\"".format(class_name)) - return - for (vtbl_ea, _, _) in vtbls: - self._vtbl_addresses.append(vtbl_ea) - - if not instances: - instances = [] - - self._classes[class_name] = FfxivClass( - class_name=class_name, vtbls=vtbls, vfuncs=vfuncs, funcs=funcs, instances=instances) - - def finalize(self): - """ - Perform the class naming - :return: None - :rtype: None - """ - self._resolve_base_classes() + data = get_yaml(fd) - # We write the vtbl names first for an added check so that - # the size finder will not advance past a named offset. - for cls in self._classes.values(): - if cls.vtbls: - cls.write_vtbl_names() + apply_data(api, data) - for cls in self._classes.values(): - self._finalize_class(cls) - - def _resolve_base_classes(self): - """ - Resolve base classes for a class - If missing, warn the user and add a stub entry. - :return: None - """ - for class_name, cls in list(self._classes.items()): - if cls.vtbls: - for idx, vtbl in enumerate(cls.vtbls): - if vtbl.resolved_base is None and vtbl.base_name: - if vtbl.base_name not in self._classes: - print("Warning: Inherited class \"{0}\" is not documented, add a placeholder entry".format( - vtbl.base_name)) - self.register(class_name=vtbl.base_name) - vtbl.resolved_base = self._classes[vtbl.base_name] - if idx == 0: - cls.first_base = self._classes[vtbl.base_name] - - _finalize_stack = deque() - - def _finalize_class(self, cls): - """ - Perform a single class naming - :param cls: Class object - :type cls: FfxivClass - :return: None - :rtype: None - """ - if cls in self._finalize_stack: - names = [c.name for c in self._finalize_stack] + [cls.name] - names = "\n".join([" - {0}".format(name) for name in names]) - raise ValueError("Inheritance cycle detected: \n{0}".format(names)) - - if not cls.finalized: - self._finalize_stack.append(cls) - if cls.vtbls: - for vtbl in cls.vtbls: - if vtbl.resolved_base and not vtbl.resolved_base.finalized: - self._finalize_class(vtbl.resolved_base) - cls.finalize() - - self._finalize_stack.pop() - - -class Vtbl: - resolved_base = None # type: FfxivClass - - def __init__(self, ea, base_name=None, size=None): - """ - Object representing a class's vtbl - :param ea: Address of vtbl - :type ea: int - :param base_name: Name of the base class this vtbl inherits, if it exists - :type base_name: str - :param size: Explicit size of vtbl used to override cases where size detection fails - :type size: int - """ - self.ea = ea - self.base_name = base_name - self.size = size - - -class FfxivClass: - STANDARD_IMAGE_BASE = 0x140000000 - - vtbls = None # type: list[Vtbl] - - def __init__(self, class_name, vtbls, vfuncs, funcs, instances): - """ - Object representing a class - :param class_name: Class name - :type class_name: str - :param vtbls: List of (vtbl_ea, base_name) pairs - :type vtbls: list[(int, str)] - :param vfuncs: Mapping of vtbl index to func names - :type vfuncs: Dict[int, str] - :param funcs: Mapping of effective addresses to func names - :type funcs: Dict[int, str] - :param instances: List of global instances of the object - :type instances: List[(int, str)] - """ - self.name = class_name - if vtbls: - self.vtbls = [Vtbl(ea, base_name, size) for (ea, base_name, size) in vtbls] - self.vfuncs = vfuncs - self.funcs = funcs - self.instances = instances - - # Offset the vtbl and funcs if the program has been rebased - current_image_base = api.get_image_base() - if self.STANDARD_IMAGE_BASE != current_image_base: - rebase_offset = current_image_base - self.STANDARD_IMAGE_BASE - if self.vtbls: - for vtbl in self.vtbls: - vtbl.ea += rebase_offset - for ea in list(funcs.keys()): - funcs[ea + rebase_offset] = funcs.pop(ea) - - # region vtbl_size - - _main_vtbl_size = 0 - - @property - def main_vtbl_size(self): - """ - Iterate from the vtbl start until a non-offset or xref is encountered. - This strategy implies that the only xref in a vtbl is the first vfunc. - Can be overriden by explicit size in the vtbl definition. - :return: VTable func count - :rtype: int - """ - if not self.vtbls: - return self._main_vtbl_size - - if self.vtbls[0].size: - self._main_vtbl_size = self.vtbls[0].size - return self._main_vtbl_size - - if self._main_vtbl_size == 0: - self._main_vtbl_size = 1 # Set to 1, skip the first entry - for ea in itertools.count(self.vtbls[0].ea + 8, 8): - if api.get_addr_name(ea) != '': - break - - if api.is_offset(ea) and api.xrefs_to(ea) == []: - self._main_vtbl_size += 1 - else: - break - - if self.vtbls[0].resolved_base and self._main_vtbl_size < self.vtbls[0].resolved_base._main_vtbl_size: - print( - "Error: The sum of \"{0}\"'s base vtbl sizes ({1}) is greater than the actual class itself ({2})" - .format(self.name, self.vtbls[0].resolved_base._main_vtbl_size, self._main_vtbl_size)) - - return self._main_vtbl_size - - # endregion - - _inheritance_tree = None - - @property - def inheritance_tree(self): - def recurse_tree(current_node, current_class, all_bases): - if current_class.vtbls: - for vtbl in current_class.vtbls: - if vtbl.resolved_base and vtbl.resolved_base.name not in all_bases: - new_node = Node(vtbl.resolved_base.name, parent=current_node) - all_bases.add(vtbl.resolved_base.name) - recurse_tree(new_node, vtbl.resolved_base, all_bases) - - if not self._inheritance_tree: - self._inheritance_tree = Node(self.name) - base_set = set() - recurse_tree(self._inheritance_tree, self, base_set) - - return self._inheritance_tree - - # region finalized - - _finalized = False - - @property - def finalized(self): - """ - Has this class and its hierarchy been written out or not - :return: bool yes/no - :rtype: bool - """ - if self.vtbls: - return self._finalized and all( - vtbl.resolved_base is not None and vtbl.resolved_base._finalized for vtbl in self.vtbls) - else: - return self._finalized - - @finalized.setter - def finalized(self, value): - """ - Set if finalized or not - :param value: Finalized state - :type value: bool - :return: None - :rtype: None - """ - self._finalized = value - - # endregion - - # region finalize - - def finalize(self): - """ - Write out this class - :return: None - :rtype: None - """ - self._inherit_func_names_from_main_base() - self._comment_vtbls_with_inheritance_tree() - - self._write_vtbl_functions() - self._write_funcs() - - self._write_instances() - - self.finalized = True - - def _inherit_func_names_from_main_base(self): - """ - A base is guaranteed to be finalized before the child, - so a base has all the vfunc names of its base already. - :return: None - :rtype: None - """ - if self.vtbls and self.vtbls[0].resolved_base: - for idx, base_vfunc_name in self.vtbls[0].resolved_base.vfuncs.items(): - if idx in self.vfuncs: - print("Warning: 0x{0:X} \"{1}\" overwrites the name of inherited function \"{2}\"".format( - self.vtbls[0].ea, self.name, base_vfunc_name)) - pass - else: - self.vfuncs[idx] = base_vfunc_name - - def _comment_vtbls_with_inheritance_tree(self): - """ - Adds the inheritance tree as a comment to the start of the vtbl. - grandbase_name - base_name - self_name - :return: None - :rtype: None - """ - if self.vtbls: - for idx, vtbl in enumerate(self.vtbls): - comment = api.get_comment(vtbl.ea) or "" - - for pre, fill, node in RenderTree(self.inheritance_tree): - if comment: - comment += "\n" - comment += pre + api.format_class_name(node.name) - api.set_comment(vtbl.ea, comment) - - def write_vtbl_names(self): - """ - Write out the vtbl name. - :return: None - """ - api.set_addr_name(self.vtbls[0].ea, api.format_class_name_for_vtbl(self.name)) - for vtbl in self.vtbls[1:]: - api.set_addr_name(vtbl.ea, api.format_class_name_for_secondary_vtbl(self.name, - vtbl.base_name)) - - def _write_vtbl_functions(self): - """ - Write out the vtbl function names - :return: None - :rtype: None - """ - - def collect_vtbl_functions(ea, size, class_name, vfunc_names, base_class_names): - """ - :type ea: int - :type size: int - :type class_name: str - :type vfunc_names: dict[int, str] - :type base_class_names: list[str] - """ - vtbl_builder = [] - # Iterate through each offset - for func_idx in range(0, size): - vtbl_vfunc_ea = ea + func_idx * 8 - vfunc_ea = api.get_qword(vtbl_vfunc_ea) # type: int - - current_func_name = api.get_addr_name(vfunc_ea) # type: str - proposed_func_name = vfunc_names.get(func_idx, "vf{0}".format(func_idx)) - formatted_class_name = api.format_class_name(class_name) - - formatted_func_name = api.format_vfunc_name(vfunc_ea, current_func_name, proposed_func_name, - formatted_class_name, - base_class_names) - - if formatted_func_name == "": - pass - elif formatted_func_name is None: - print( - "Error: Function at 0x{0:X} had unexpected name \"{1}\" during naming of {2}.{3} (vtbl[{4}])" - .format(vfunc_ea, current_func_name, self.name, proposed_func_name, func_idx)) - else: - vtbl_builder.append((vfunc_ea, formatted_func_name)) - - return vtbl_builder - - if self.vtbls: - formatted_base_class_names = [api.format_class_name(node.name) for node in - PreOrderIter(self.inheritance_tree)] - formatted_base_class_names.remove(self.name) - for idx, vtbl in enumerate(self.vtbls): - if idx == 0: - funcs = collect_vtbl_functions(vtbl.ea, self.main_vtbl_size, self.name, self.vfuncs, - formatted_base_class_names) - else: - funcs = collect_vtbl_functions(vtbl.ea, vtbl.resolved_base.main_vtbl_size, - self.name + "___" + vtbl.resolved_base.name, - vtbl.resolved_base.vfuncs, formatted_base_class_names) - for (func_ea, func_name) in funcs: - api.set_addr_name(func_ea, func_name) - - def _write_funcs(self): - """ - Write the names of all non-vtbl funcs - :return: None - """ - for func_ea, proposed_func_name in self.funcs.items(): - current_func_name = api.get_addr_name(func_ea) # type: str - - func_name = api.format_func_name(func_ea, current_func_name, proposed_func_name, self.name) - if func_name == "": - pass - elif func_name is None: - print("Error: Function at 0x{0:X} had unexpected name \"{1}\" during naming of {2}.{3}" - .format(func_ea, current_func_name, self.name, proposed_func_name)) - else: - api.set_addr_name(func_ea, func_name) - - def _write_instances(self): - """ - Write the names of all instances - :return: None - """ - for (instance_ea, instance_name) in self.instances: - name = "g_{}_{}".format(self.name, instance_name) - api.set_addr_name(instance_ea, name) - - # endregion - - def __repr__(self): - return "<{0}(\"{1}\")>".format(self.__class__.__name__, self.name) - - -# endregion print("Executing") load_data() diff --git a/scripts/ida/rename.py b/scripts/ida/rename.py new file mode 100644 index 0000000000..4c224920ca --- /dev/null +++ b/scripts/ida/rename.py @@ -0,0 +1,60 @@ +from os.path import join, dirname +import sys + +for path in (dirname(__file__), join(dirname(__file__), "..")): + if path not in sys.path: + sys.path.append(path) + +from ida_wrapper import IdaInterface +from class_factory import apply_data + +lua_namespace = "Client::Game::Event::Lua" + +def apply_lua_funcs(): + """Relies on the Common::Lua::LuaState names from data.yml, so apply_data has to run first.""" + string_eas = ida.get_func_xrefs(ida.get_func_ea_by_name('Common::Lua::LuaState.SetStringField')) + func_eas = ida.get_func_xrefs(ida.get_func_ea_by_name('Common::Lua::LuaState.SetFunctionField')) + + string_eas = [ida.get_func_start(x) for x in string_eas] + + eas = {} + for x in func_eas: + regs = ida.get_registers(ida.heads_before(x, 4)) + regs['r8'] = ida.get_string_from_ptr(regs['r8']['val']) + eas.setdefault(ida.get_func_start(x), {'calls': []})['calls'].append({'off': x, 'regs': regs}) + + eas = {k: v for k,v in eas.items() if k in string_eas} + + for func_ea in eas: + eas[func_ea]['regs'] = ida.get_registers(ida.heads_until_call(func_ea, 'Common::Lua::LuaState.SetStringField'), ('mov',)) + + for func_ea in eas: + eas[func_ea]['name'] = eas[func_ea]['regs'][eas[func_ea]['regs']['r9']['val_text']]['val_text'] + eas[func_ea]['name_val'] = eas[func_ea]['regs'][eas[func_ea]['regs']['r9']['val_text']]['val'] + + for func_ea in eas: + if eas[func_ea]['name'] == 'r8': + eas[func_ea]['name'] = ida.get_func_name(func_ea).split('.')[0].split(':')[-1] + eas[func_ea]['name_val'] = func_ea + if eas[func_ea]['name'] == 'rax': + eas[func_ea]['name_val'] = ida.get_func_xrefs(func_ea)[0] + eas[func_ea]['name'] = ida.get_func_name(eas[func_ea]['name_val']).split('.')[0].split(':')[-1] + + for func_ea in eas: + name = f"{lua_namespace}::{eas[func_ea]['name']}" + ida.set_func_name(func_ea, f"{name}.SetupLuaClass") + for call in eas[func_ea]['calls']: + regs = call['regs'] + func_name = regs['r8'] + ida.set_func_name(regs['r9']['val'], f"{name}.{func_name}", unique=True) + +ida = IdaInterface() +ida.select_parser() +print("Loading data.yml") +ida.load_data_yml() + +print("Executing") +apply_data(ida, ida.data_yaml) +print("Naming lua functions") +apply_lua_funcs() +print("Done") From 5badfa309aea22aa16ae188b5c35a2b083fd0001 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:33:18 +0200 Subject: [PATCH 58/95] cleanup struct import to use more of the wrapper --- scripts/ida/struct_import.py | 51 ++++++++---------------------------- 1 file changed, 11 insertions(+), 40 deletions(-) diff --git a/scripts/ida/struct_import.py b/scripts/ida/struct_import.py index 30652da4e1..c55654ed1c 100644 --- a/scripts/ida/struct_import.py +++ b/scripts/ida/struct_import.py @@ -14,12 +14,6 @@ from ida_wrapper import IdaInterface from form_builder import DynamicForm from structs_schema import get_yaml -import ida_bytes -import ida_search -import ida_typeinf -import ida_hexrays -import idc -import idaapi src = SrcInterface() ida = IdaInterface() @@ -70,7 +64,7 @@ try: if form.Execute() == 1: _function_update_options = form.results()["function_types"] - idc.set_name(idaapi.get_imagebase(), form.results()["base_name"]) + ida.set_addr_name(ida.get_image_base(), form.results()["base_name"]) else: _function_update_options = { "virtual": False, @@ -110,23 +104,16 @@ ) def do_pattern(pattern, suffix, sheetSearchPattern, sheetSearchOffset): - ea = 0 - - while True: - ea = ida.search_binary(ea + 1, pattern, ida_search.SEARCH_DOWN) - - if ea == 0xFFFFFFFFFFFFFFFF: - break - + for ea in ida.search_binary_all(pattern, 1): if sheetSearchPattern == None: sheetEa = ea + sheetSearchOffset else: sheetEa = ( - ida.search_binary(ea, sheetSearchPattern, ida_search.SEARCH_DOWN) + ida.search_binary(ea, sheetSearchPattern) + sheetSearchOffset ) sheetIdx = ida.get_dword(sheetEa) - origName = idc.get_func_name(ea) + origName = ida.get_func_name(ea) # don't rename any funcs that are already named if origName[0:4] == "sub_": @@ -150,7 +137,7 @@ def do_pattern(pattern, suffix, sheetSearchPattern, sheetSearchOffset): while True: uniqueName = fnName + (f"_{uniquifier}" if uniquifier > 0 else "") - if idc.get_name_ea_simple(uniqueName) == idc.BADADDR and uniquifier > 0: + if ida.get_func_ea_by_name(uniqueName) == ida.BADADDR and uniquifier > 0: fnName = uniqueName break @@ -163,18 +150,9 @@ def do_pattern(pattern, suffix, sheetSearchPattern, sheetSearchOffset): if suffix == "SheetIndex": continue - tif, funcdata = ( - ida_typeinf.tinfo_t(), - ida_typeinf.func_type_data_t(), - ) - - ida_typeinf.guess_tinfo(tif, ea) - if not tif.get_func_details(funcdata): - ida_hexrays.decompile(ea) - ida_typeinf.guess_tinfo(tif, ea) - if not tif.get_func_details(funcdata): - print("Failed to get func details for %s @ %X" % (fnName, ea)) - continue + if not ida.can_guess_func_type(ea): + print("Failed to get func details for %s @ %X" % (fnName, ea)) + continue if suffix == "RowAndSubRowId": tif = ida.get_tinfo_from_type( @@ -189,15 +167,8 @@ def do_pattern(pattern, suffix, sheetSearchPattern, sheetSearchOffset): f"Component::Exd::Sheets::{exd_map[sheetIdx]} *__fastcall func(unsigned int rowIdOrIndex);" ) - ida_typeinf.apply_tinfo(ea, tif, ida_typeinf.TINFO_DEFINITE) - + ida.apply_tinfo(ea, tif) -def get_all_eas(pattern: str): - ea = ida.search_binary(0, pattern, ida_search.SEARCH_DOWN) - while ea != idc.BADADDR: - yield ea - ea = ea + 1 - ea = ida.search_binary(ea, pattern, ida_search.SEARCH_DOWN) print("Fixing up Excel functions") @@ -210,13 +181,13 @@ def get_all_eas(pattern: str): for pattern_key in exd_comment_patterns: - for ea in list(get_all_eas(pattern_key)): + for ea in list(ida.search_binary_all(pattern_key)): sheetIdx = ida.get_dword(ea + exd_comment_patterns[pattern_key]) sheetName = exd_map.get(sheetIdx) if sheetName is None: print(f"Sheet getter @ 0x{ea:X} references unknown sheet {sheetIdx}!") continue - ida_bytes.set_cmt(ea, "Sheet: {0} ({1})".format(sheetName, sheetIdx), 0) + ida.set_comment(ea, "Sheet: {0} ({1})".format(sheetName, sheetIdx)) if _function_update_options["virtual"]: for struct in import_structs.structs: From 4adf4ec244630152c769c416bf08b905eae6cfcd Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:33:30 +0200 Subject: [PATCH 59/95] add new full run file --- scripts/ida/full_run.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 scripts/ida/full_run.py diff --git a/scripts/ida/full_run.py b/scripts/ida/full_run.py new file mode 100644 index 0000000000..57de045963 --- /dev/null +++ b/scripts/ida/full_run.py @@ -0,0 +1,17 @@ +from os import path +import idautils +import ida_auto +import idc + +working_directory = path.dirname(path.realpath(__file__)) + +print("Loading rename.py") +exec(open(path.join(working_directory, "rename.py")).read()) + +print("Loading struct_import.py") +exec(open(path.join(working_directory, "struct_import.py")).read()) + +for s in idautils.Segments(): + start = idc.get_segm_start(s) + end = idc.get_segm_end(s) + ida_auto.plan_range(start, end) \ No newline at end of file From 3429c0c4201d9f80bedbe0068d213bfa6e927f9b Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:33:41 +0200 Subject: [PATCH 60/95] update the readme with new info --- scripts/README.md | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index 176730e650..d059416636 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,11 +1,10 @@ ## ffxiv_idarename.py -Despite the name being a misnomer at this point, this script supports both Ghidra and IDA. +Despite the name, this script is now only for Ghidra. The IDA version lives in `ida/rename.py`. This script ingests the `data.yml` file in the same directory, and renames various offsets as globals, functions, vtables and such. #### IDA dependency installation: -`pip install pyyaml anytree` or `python -m pip install pyyaml anytree` in whichever version of python you are currently using with IDA Pro. -If you are familiar with Poetry, the `idarename` extras package will do the same. -This could be either Python2 or Python3. +`pip install pyyaml anytree pythonnet` or `python -m pip install pyyaml anytree pythonnet` in whichever version of python you are currently using with IDA Pro. +If you are familiar with Poetry, the `idarename` extras package will do the same. Additionally, you will need to place "idauser.cfg" from this repository in "%AppData%\Hex-Rays\IDA Pro\". IDA doesn't allow you to use certain characters in names and this config changes that. @@ -24,23 +23,9 @@ This support **Python3** and **IDA** only. `pip install dacite pyyaml ruamel-yaml` or `python -m pip install dacite pyyaml ruamel-yaml` in whichever version of python you are currently using with IDA Pro. If you are familiar with Poetry, the `sigmaker` extras package will do the same. -## ffxiv_exdgetters.py -> [!WARNING] -> Does not work with IDA 9 due to changes made on the application - -This script ingests the `exh` files from the base game and renames various functions, including setting the return type of the functions to the propper sheet struct. - -## ffxiv_structimporter.py -> [!WARNING] -> Does not work with IDA 9 due to changes made on the application - -This script ingests the `ffxiv_structs.yml` file and forms the proper structs and assigns the member function return and param types correctly - -## ffxiv_fullrun_ida.py -> [!WARNING] -> Does not work with IDA 9 due to changes made on the application - -This script runs `ffxiv_idarename.py`, `ffxiv_exdgetters.py`, `ffxiv_structimporter.py` in sequence +## ida/ +### full_run.py +This script runs `rename.py` and `struct_import.py` in sequence ## classinformer.csv Once upon a time, someone did a bad thing and released FFXIV with the RTTI data intact. From 23d11708e2ea27831375da29e526c1ba2720735d Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:44:10 +0200 Subject: [PATCH 61/95] remove some files not used --- scripts/ffxiv_extract_uld_names.py | 76 ------------------------------ scripts/ida/ida_error.txt | 63 ------------------------- 2 files changed, 139 deletions(-) delete mode 100644 scripts/ffxiv_extract_uld_names.py delete mode 100644 scripts/ida/ida_error.txt diff --git a/scripts/ffxiv_extract_uld_names.py b/scripts/ffxiv_extract_uld_names.py deleted file mode 100644 index d706ba4e1a..0000000000 --- a/scripts/ffxiv_extract_uld_names.py +++ /dev/null @@ -1,76 +0,0 @@ -import idaapi -import ida_bytes -import ida_nalt -import ida_struct -import ida_enum -import ida_kernwin -import ida_search -import ida_ida -import ida_typeinf -import ida_hexrays -import ida_name -import os - -patterns = [ - "48 8D 15 ?? ?? ?? ?? 41 B9 ?? ?? ?? ?? 45 33 C0 E8 ?? ?? ?? ??", - "48 8D 15 ?? ?? ?? ?? 45 33 C0 48 8B CB 48 83 C4 20 5B E9 ?? ?? ?? ??", - "48 8D 15 ?? ?? ?? ?? 45 33 C0 E9 ?? ?? ?? ??", -] - -strings = [] - - -def search_binary(ea, pattern, flag): - return ida_search.find_binary( - ea, - flag & 1 and ida_ida.cvar.inf.max_ea or ida_ida.cvar.inf.min_ea, - pattern, - 16, - flag, - ) - - -def get_string(offset): - return ida_bytes.get_strlit_contents(offset, -1, ida_nalt.STRTYPE_TERMCHR).decode( - "UTF-8" - ) - - -def do_pattern(pattern): - ea = 0 - while True: - ea = search_binary( - ea + 1, - pattern, - ida_search.SEARCH_DOWN, - ) - - if ea == 0xFFFFFFFFFFFFFFFF: - break - - offset = ida_bytes.get_original_dword(ea + 3) - - string = get_string(offset + ea + 0x7) - - strings.append(string) - - -for pattern in patterns: - do_pattern(pattern) - -txtFile = os.path.join(os.path.dirname(os.path.realpath(__file__)), "uld_names.txt") - -if os.path.exists(txtFile): - os.remove(txtFile) - -strings.sort() - -filestr = "" - -for string in strings: - filestr = f'{filestr},\n"ui/uld/{string}.uld"' - -with open(txtFile, "w") as f: - f.write(filestr) - -print(strings) diff --git a/scripts/ida/ida_error.txt b/scripts/ida/ida_error.txt deleted file mode 100644 index 4fbd6799a2..0000000000 --- a/scripts/ida/ida_error.txt +++ /dev/null @@ -1,63 +0,0 @@ -Loading data.yml -Grabbing ExdSchema and Lumina -Requesting nugpkg Lumina with latest version -Requesting nugpkg BCnEncoder.Net with version 2.2.1 -Requesting nugpkg Microsoft.Extensions.ObjectPool with version 10.0.0 -Requesting nugpkg CommunityToolkit.HighPerformance with version 8.4.0 -Requesting nugpkg Microsoft.Bcl.HashCode with version 6.0.0 -Requesting nugpkg System.Memory with version 4.6.0 -Requesting nugpkg System.Runtime.CompilerServices.Unsafe with version 6.1.0 -Requesting nugpkg System.Threading.Tasks.Extensions with version 4.6.0 -Requesting nugpkg System.Buffers with version 4.6.0 -Requesting nugpkg System.Numerics.Vectors with version 4.6.0 -Downloading https://github.com/xivdev/EXDSchema/archive/refs/heads/latest.zip -Parsing ffxiv_structs.yml -Parsing combined Excel and FFXIV declarations -C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Tools\MSVC\14.51.36231\include\map:46:11 error Failed to prepare forward declaration for : name 'std::_Tmap_traits, std::allocator>, false>::value_compare' is not acceptable - -C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Tools\MSVC\14.51.36231\include\map:46:11 error Failed to prepare forward declaration for : name 'std::_Tmap_traits, std::allocator>, false>::value_compare' is not acceptable - -input.cc:99049:31 error Failed to prepare forward declaration for : name 'StdLinkedList::Node' is not acceptable - -C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Tools\MSVC\14.51.36231\include\map:46:11 error Failed to prepare forward declaration for : name 'std::_Tmap_traits, std::allocator>, false>::value_compare' is not acceptable - -C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Tools\MSVC\14.51.36231\include\map:46:11 error Failed to prepare forward declaration for : name 'std::_Tmap_traits, std::allocator>, false>::value_compare' is not acceptable - -C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Tools\MSVC\14.51.36231\include\map:46:11 error Failed to prepare forward declaration for : name 'std::_Tmap_traits, std::allocator>, false>::value_compare' is not acceptable - -C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Tools\MSVC\14.51.36231\include\map:46:11 error Failed to prepare forward declaration for : name 'std::_Tmap_traits, std::allocator>, false>::value_compare' is not acceptable - -:-1:-1 warning ignored: __cdecl is not supported by the current architecture -C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Tools\MSVC\14.51.36231\include\map:46:11 error Failed to prepare forward declaration for : name 'std::_Tmap_traits, std::allocator>, false>::value_compare' is not acceptable - -Fixing up Excel functions -Finding exd funcs of Row... please wait. -Finding exd funcs of Row... please wait. -Finding exd funcs of Row... please wait. -Finding exd funcs of Row... please wait. -Finding exd funcs of Row... please wait. -Finding exd funcs of Row... please wait. -Finding exd funcs of Row... please wait. -Finding exd funcs of RowAndSubRowId... please wait. -Func @ 0x140844120 references unknown sheet 1401! sheet dword offset = 0x14084413B -Func @ 0x140844170 references unknown sheet 1400! sheet dword offset = 0x14084418B -Func @ 0x1408441C0 references unknown sheet 1399! sheet dword offset = 0x1408441DB -Finding exd funcs of RowAndSubRowId... please wait. -Finding exd funcs of RowIndex... please wait. -Finding exd funcs of RowIndex... please wait. -Finding exd funcs of SheetIndex... please wait. -Finding exd funcs of RowCount... please wait. -An unexpected error occurred at 0x140844124: 1401 -An unexpected error occurred at 0x140844174: 1400 -An unexpected error occurred at 0x1408441c4: 1399 -Updating virtual functions for Animation::Playback::Control::Default::hkaDefaultAnimationControl::hkaDefaultAnimationControlListener -Error: Animation::Playback::Control::Default::hkaDefaultAnimationControl::hkaDefaultAnimationControlListener.LoopOverflowCallback not found and its VFT slot could not be resolved -Error: Animation::Playback::Control::Default::hkaDefaultAnimationControl::hkaDefaultAnimationControlListener.LoopUnderflowCallback not found and its VFT slot could not be resolved -Error: Animation::Playback::Control::Default::hkaDefaultAnimationControl::hkaDefaultAnimationControlListener.EasedInCallback not found and its VFT slot could not be resolved -Error: Animation::Playback::Control::Default::hkaDefaultAnimationControl::hkaDefaultAnimationControlListener.EasedOutCallback not found and its VFT slot could not be resolved -Error: Animation::Playback::Control::Default::hkaDefaultAnimationControl::hkaDefaultAnimationControlListener.dtor not found and its VFT slot could not be resolved -Updating virtual functions for Animation::Playback::Control::hkaAnimationControlListener -Error: Animation::Playback::Control::hkaAnimationControlListener.ControlDeletedCallback not found and its VFT slot could not be resolved -Error: Animation::Playback::Control::hkaAnimationControlListener.dtor not found and its VFT slot could not be resolved -Updating virtual functions for Application::Network::LobbyClient::LobbyRequestCallback -! failed to create pointer for 'Application::Network::LobbyClient::LobbyRequestCallback*': base type 'Application::Network::LobbyClient::LobbyRequestCallback' resolved to '?' (correct=True, forward=False) \ No newline at end of file From 54f294f36ce5fe9e1a63d8137277af738978d26e Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:44:34 +0200 Subject: [PATCH 62/95] move matcher into ida folder --- scripts/ffxiv_data_struct_matcher.py | 212 --------------------------- scripts/ida/data_struct_matcher.py | 28 ++++ scripts/ida/ida_wrapper.py | 17 ++- 3 files changed, 39 insertions(+), 218 deletions(-) delete mode 100644 scripts/ffxiv_data_struct_matcher.py create mode 100644 scripts/ida/data_struct_matcher.py diff --git a/scripts/ffxiv_data_struct_matcher.py b/scripts/ffxiv_data_struct_matcher.py deleted file mode 100644 index 0e393b5bc7..0000000000 --- a/scripts/ffxiv_data_struct_matcher.py +++ /dev/null @@ -1,212 +0,0 @@ -import idaapi -import idc -import ida_bytes -import ida_search -import ida_typeinf -import ida_funcs -import ida_name -import ida_kernwin -import os -import sys -from ida.ida_wrapper import IdaInterface -from structs_schema import * -from data_schema import * -from yaml import load - -if sys.version_info[0] >= 3: - long = int - -try: - from yaml import CSafeLoader as Loader -except ImportError: - from yaml import SafeLoader as Loader - - -def get_image_base(): - return idaapi.get_imagebase() - - -dataDir = os.path.dirname(os.path.realpath(__file__)) -structFileName = os.path.join(dataDir, "ffxiv_structs.yml") -dataFileName = os.path.join(dataDir, "data.yml") - -ida = IdaInterface() - -def get_structs() -> DefinedStructExport: - dic: dict[str, dict[str, list[dict[str, str | int | list[dict[str, str | int]]]]]] = load(open(structFileName), Loader=Loader) - enums = [] - structs = [] - for enum in dic["enums"]: - enums.append( - DefinedStructEnum( - enum["name"], - enum["type"], - enum["underlying"], - enum["namespace"], - enum["flags"], - enum["values"], - ) - ) - for struct in dic["structs"]: - fields = [] - virtual_functions = None - member_functions = [] - static_member_functions = None - static_members = None - for field in struct["fields"]: - base = field["base"] if "base" in field else False - if "size" in field: - fields.append( - DefinedStructFixedField( - field["name"], - field["type"], - field["offset"], - base, - field["size"], - ) - ) - elif "return_type" in field: - parameters = [] - for param in field["parameters"]: - parameters.append( - DefinedStructFuncParam(param["name"], param["type"]) - ) - fields.append( - DefinedStructFuncField( - field["name"], - field["type"], - field["offset"], - base, - field["return_type"], - parameters, - ) - ) - else: - fields.append( - DefinedStructField( - field["name"], field["type"], field["offset"], base - ) - ) - if "virtual_functions" in struct: - virtual_functions = [] - for vfunc in struct["virtual_functions"]: - parameters = ( - [ - DefinedStructFuncParam(param["name"], param["type"]) - for param in vfunc["parameters"] - ] - if "parameters" in vfunc - else None - ) - virtual_functions.append( - DefinedStructVFunc( - vfunc["name"], - vfunc["return_type"] if "return_type" in vfunc else None, - vfunc["offset"], - parameters, - ) - ) - for memfunc in struct["member_functions"]: - parameters = [] - for param in memfunc["parameters"]: - parameters.append(DefinedStructFuncParam(param["name"], param["type"])) - member_functions.append( - DefinedStructMemFunc( - memfunc["signature"], - memfunc["return_type"], - parameters, - memfunc["name"], - ) - ) - if "static_member_functions" in struct: - static_member_functions = [] - for smemfunc in struct["static_member_functions"]: - parameters = [] - for param in smemfunc["parameters"]: - parameters.append( - DefinedStructFuncParam(param["name"], param["type"]) - ) - static_member_functions.append( - DefinedStructMemFunc( - smemfunc["signature"], - smemfunc["return_type"], - parameters, - smemfunc["name"], - ) - ) - if "static_members" in struct: - static_members = [] - for sm in struct["static_members"]: - static_members.append( - DefinedStructStaticMember( - sm["name"], - sm["signature"], - sm["relative_follow_offsets"], - sm["return_type"], - sm["is_pointer"] if "is_pointer" in sm else False, - ) - ) - size = None - if "size" in struct: - size = struct["size"] - vtable_size = None - if "vtable_size" in struct: - vtable_size = struct["vtable_size"] - structs.append( - DefinedStruct( - struct["name"], - struct["type"], - struct["namespace"], - fields, - size, - vtable_size, - virtual_functions, - member_functions, - struct["union"], - static_member_functions, - static_members, - ) - ) - return DefinedStructExport(enums, structs) - - -def get_data(): - dic: dict[str, str | dict[int, str] | dict[str, dict[str, list[dict[str, int | bool | str]] | dict[int, str]]]] = load(open(dataFileName), Loader=Loader) - classes = [] - for className, classInstance in dic["classes"].items(): - if classInstance == None: - classes.append(DefinedDataClass(className)) - continue - instances = [] - if "instances" in classInstance: - for instance in classInstance["instances"]: - instances.append( - DefinedDataClassInstance( - instance["ea"], - instance["pointer"] if "pointer" in instance else False, - instance["name"] if "name" in instance else None, - ) - ) - classes.append(DefinedDataClass(className, instances)) - return DefinedData(classes) - - -def load_data(): - structs = get_structs().structs - for data in get_data().classes: - struct = next((struc for struc in structs if struc.type == data.name), None) - if struct is None or struct.static_members is None: - continue - for static_member in struct.static_members: - ea = ida.search_binary(0, static_member.signature, ida_search.SEARCH_DOWN) - for offset in static_member.relative_offsets: - ea = ea + offset - dword = ida.get_dword(ea) - ea = ea + 4 + dword - instance = next((inst for inst in data.instances if inst.ea == ea), None) - if instance is None: - continue - if ea == instance.ea and static_member.is_pointer != instance.pointer: - print(f"Found pointer discrepancy at {ea:X} on {data.name}") - -load_data() diff --git a/scripts/ida/data_struct_matcher.py b/scripts/ida/data_struct_matcher.py new file mode 100644 index 0000000000..a154dc3244 --- /dev/null +++ b/scripts/ida/data_struct_matcher.py @@ -0,0 +1,28 @@ +from os.path import join, dirname +import sys + +sys.path.append(join(dirname(__file__), "..")) + +from ida_wrapper import IdaInterface +from structs_schema import get_yaml + +ida = IdaInterface() +print("Loading data.yml") +ida.load_data_yml() + +print("Parsing ffxiv_structs.yml") +with open(ida.get_struct_path()) as f: + structs = {struct.type: struct for struct in get_yaml(f).structs} + +# data.yml addresses are relative to the standard image base +rebase_offset = ida.get_image_base() - 0x140000000 + +for class_name, class_data in ida.data_yaml.classes.items(): + struct = structs.get(class_name) + if struct is None or struct.static_members is None: + continue + for static_member in struct.static_members: + ea = ida.get_static_member_ea(static_member) + instance = next((inst for inst in class_data.instances if inst.ea + rebase_offset == ea), None) + if instance is not None and static_member.is_pointer != instance.pointer: + print(f"Found pointer discrepancy at {ea:X} on {class_name}") diff --git a/scripts/ida/ida_wrapper.py b/scripts/ida/ida_wrapper.py index 4ea42dbc70..fdd6e9c63a 100644 --- a/scripts/ida/ida_wrapper.py +++ b/scripts/ida/ida_wrapper.py @@ -513,16 +513,21 @@ def update_virt_func(self, virt_func: DefinedStructVFunc, struct: DefinedStruct) return self.apply_func_type(ea, func_name, virt_func.return_type, virt_func.parameters) - def update_static_member(self, static_member: DefinedStructStaticMember, struct: DefinedStruct): - ea = self.search_binary( - 0, static_member.signature, ida_search.SEARCH_DOWN - ) + def get_static_member_ea(self, static_member: DefinedStructStaticMember) -> int: + """Address the static member's signature resolves to, BADADDR if the signature is not found.""" + ea = self.search_binary(0, static_member.signature) if ea == idc.BADADDR: - print(f"Error: {static_member.signature} not found something is wrong") - return + return ea for follows in static_member.relative_offsets: ea = ea + follows ea = ea + 4 + self.get_dword(ea) + return ea + + def update_static_member(self, static_member: DefinedStructStaticMember, struct: DefinedStruct): + ea = self.get_static_member_ea(static_member) + if ea == idc.BADADDR: + print(f"Error: {static_member.signature} not found something is wrong") + return if ida_bytes.get_item_head(ea) != ea and static_member.return_type != struct.type: # points into an already typed item (ex. a field of the struct instance) return From bc479202ccd6172d7c7fecc78671a904205402d2 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:49:40 +0200 Subject: [PATCH 63/95] move wrapper files into a shared folder for later --- scripts/ffxiv_idarename.py | 13 +++++++++++++ scripts/ffxiv_structimporter.py | 12 ++++++++++++ scripts/ida/data_struct_matcher.py | 2 +- scripts/ida/rename.py | 2 +- scripts/ida/struct_import.py | 1 + scripts/{ => shared}/data_schema.py | 0 scripts/{ => shared}/exdschema_wrapper.py | 0 scripts/{ => shared}/lumina_wrapper.py | 0 scripts/{ => shared}/src_wrapper.py | 0 scripts/{ => shared}/structs_schema.py | 0 10 files changed, 28 insertions(+), 2 deletions(-) rename scripts/{ => shared}/data_schema.py (100%) rename scripts/{ => shared}/exdschema_wrapper.py (100%) rename scripts/{ => shared}/lumina_wrapper.py (100%) rename scripts/{ => shared}/src_wrapper.py (100%) rename scripts/{ => shared}/structs_schema.py (100%) diff --git a/scripts/ffxiv_idarename.py b/scripts/ffxiv_idarename.py index ef59e953ba..ec1e3c29c5 100644 --- a/scripts/ffxiv_idarename.py +++ b/scripts/ffxiv_idarename.py @@ -17,6 +17,19 @@ if sys.version_info[0] >= 3: long = int +# class_factory.py lives next to this script, the schema modules in ./shared +try: + script_dir = os.path.dirname(os.path.realpath(__file__)) +except NameError: + script_dir = os.path.dirname(str(sourceFile)) # noqa + +for path in (script_dir, os.path.join(script_dir, "shared")): + if path not in sys.path: + sys.path.append(path) + +from data_schema import get_yaml # noqa +from class_factory import apply_data # noqa + # region Api class BaseApi(object): diff --git a/scripts/ffxiv_structimporter.py b/scripts/ffxiv_structimporter.py index 59904a664e..2df03db4e2 100644 --- a/scripts/ffxiv_structimporter.py +++ b/scripts/ffxiv_structimporter.py @@ -10,8 +10,20 @@ from yaml import SafeLoader as Loader import os +import sys from abc import abstractmethod from time import time + +# The schema and wrapper modules live in ./shared +try: + script_dir = os.path.dirname(os.path.realpath(__file__)) +except NameError: + script_dir = os.path.dirname(str(sourceFile)) # noqa + +for path in (script_dir, os.path.join(script_dir, "shared")): + if path not in sys.path: + sys.path.append(os.path.join(script_dir, "shared")) + from structs_schema import * from src_wrapper import SrcInterface diff --git a/scripts/ida/data_struct_matcher.py b/scripts/ida/data_struct_matcher.py index a154dc3244..70ded7c7bb 100644 --- a/scripts/ida/data_struct_matcher.py +++ b/scripts/ida/data_struct_matcher.py @@ -1,7 +1,7 @@ from os.path import join, dirname import sys -sys.path.append(join(dirname(__file__), "..")) +sys.path.append(join(dirname(__file__), "..", "shared")) from ida_wrapper import IdaInterface from structs_schema import get_yaml diff --git a/scripts/ida/rename.py b/scripts/ida/rename.py index 4c224920ca..808af9ff26 100644 --- a/scripts/ida/rename.py +++ b/scripts/ida/rename.py @@ -1,7 +1,7 @@ from os.path import join, dirname import sys -for path in (dirname(__file__), join(dirname(__file__), "..")): +for path in (dirname(__file__), join(dirname(__file__), ".."), join(dirname(__file__), "..", "shared")): if path not in sys.path: sys.path.append(path) diff --git a/scripts/ida/struct_import.py b/scripts/ida/struct_import.py index c55654ed1c..c22f0d6276 100644 --- a/scripts/ida/struct_import.py +++ b/scripts/ida/struct_import.py @@ -2,6 +2,7 @@ import sys sys.path.append(join(dirname(__file__), "..")) +sys.path.append(join(dirname(__file__), "..", "shared")) from lumina_wrapper import get_excel_header_files from exdschema_wrapper import ( diff --git a/scripts/data_schema.py b/scripts/shared/data_schema.py similarity index 100% rename from scripts/data_schema.py rename to scripts/shared/data_schema.py diff --git a/scripts/exdschema_wrapper.py b/scripts/shared/exdschema_wrapper.py similarity index 100% rename from scripts/exdschema_wrapper.py rename to scripts/shared/exdschema_wrapper.py diff --git a/scripts/lumina_wrapper.py b/scripts/shared/lumina_wrapper.py similarity index 100% rename from scripts/lumina_wrapper.py rename to scripts/shared/lumina_wrapper.py diff --git a/scripts/src_wrapper.py b/scripts/shared/src_wrapper.py similarity index 100% rename from scripts/src_wrapper.py rename to scripts/shared/src_wrapper.py diff --git a/scripts/structs_schema.py b/scripts/shared/structs_schema.py similarity index 100% rename from scripts/structs_schema.py rename to scripts/shared/structs_schema.py From 9ed5efb2439ac19df7347533aba9e4073510819b Mon Sep 17 00:00:00 2001 From: wildwolf <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:18:11 +0200 Subject: [PATCH 64/95] update workflows to work with new folder structure --- .github/workflows/pr-change-check-patch-day.yaml | 6 +++--- .github/workflows/pr-change-check.yaml | 10 +++++----- .github/workflows/push-build.yaml | 4 ++-- .github/workflows/validate-data.yaml | 6 +++--- scripts/data-validator.js | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/pr-change-check-patch-day.yaml b/.github/workflows/pr-change-check-patch-day.yaml index 2a6cdc3593..d7c27db764 100644 --- a/.github/workflows/pr-change-check-patch-day.yaml +++ b/.github/workflows/pr-change-check-patch-day.yaml @@ -29,7 +29,7 @@ jobs: - name: Check error txt file for content id: check_error run: | - if [ -s ./ida/errors.txt ]; then + if [ -s ./scripts/errors.txt ]; then echo "error=true" >> $GITHUB_OUTPUT else echo "error=false" >> $GITHUB_OUTPUT @@ -40,7 +40,7 @@ jobs: with: script: | const fs = require('fs'); - const content = fs.readFileSync('./ida/errors.txt', 'utf8'); + const content = fs.readFileSync('./scripts/errors.txt', 'utf8'); const comments = (await github.rest.issues.listComments({ issue_number: context.payload.pull_request.number, owner: context.repo.owner, @@ -108,5 +108,5 @@ jobs: - uses: actions/upload-artifact@v4 with: name: structs.yml - path: ./ida/ffxiv_structs.yml + path: ./scripts/ffxiv_structs.yml retention-days: 7 diff --git a/.github/workflows/pr-change-check.yaml b/.github/workflows/pr-change-check.yaml index 94a8390916..87e92bd638 100644 --- a/.github/workflows/pr-change-check.yaml +++ b/.github/workflows/pr-change-check.yaml @@ -30,7 +30,7 @@ jobs: - name: Check error txt file for content id: check_error run: | - if [ -s ./ida/errors.txt ]; then + if [ -s ./scripts/errors.txt ]; then echo "error=true" >> $GITHUB_OUTPUT else echo "error=false" >> $GITHUB_OUTPUT @@ -41,7 +41,7 @@ jobs: with: script: | const fs = require('fs'); - const content = fs.readFileSync('./ida/errors.txt', 'utf8'); + const content = fs.readFileSync('./scripts/errors.txt', 'utf8'); const comments = (await github.rest.issues.listComments({ issue_number: context.payload.pull_request.number, owner: context.repo.owner, @@ -109,7 +109,7 @@ jobs: - uses: actions/upload-artifact@v4 with: name: structs.yml - path: ./ida/ffxiv_structs.yml + path: ./scripts/ffxiv_structs.yml retention-days: 7 build-pr: @@ -179,7 +179,7 @@ jobs: run: | dotnet tool install -g Microsoft.DotNet.ApiCompat.Tool dotnet run --project CompatChecker/CompatChecker.csproj -c Release -- ida/cs-main/FFXIVClientStructs.dll ida/cs-pr/FFXIVClientStructs.dll ida/errors.txt - if [ -s ./ida/errors.txt ]; then + if [ -s ./scripts/errors.txt ]; then echo "breaking=true" >> $GITHUB_OUTPUT else echo "breaking=false" >> $GITHUB_OUTPUT @@ -190,7 +190,7 @@ jobs: with: script: | const fs = require('fs'); - const content = fs.readFileSync('./ida/errors.txt', 'utf8'); + const content = fs.readFileSync('./scripts/errors.txt', 'utf8'); github.rest.issues.addLabels({ issue_number: context.payload.pull_request.number, owner: context.repo.owner, diff --git a/.github/workflows/push-build.yaml b/.github/workflows/push-build.yaml index b6617c8a11..5fa5008bb6 100644 --- a/.github/workflows/push-build.yaml +++ b/.github/workflows/push-build.yaml @@ -24,7 +24,7 @@ jobs: - name: Check error txt file for content id: check_error run: | - if [ -s ./ida/errors.txt ]; then + if [ -s ./scripts/errors.txt ]; then echo "error=true" >> $GITHUB_OUTPUT else echo "error=false" >> $GITHUB_OUTPUT @@ -33,4 +33,4 @@ jobs: if: steps.check_error.outputs.error == 'true' uses: peter-evans/commit-comment@v4 with: - body-path: './ida/errors.txt' + body-path: './scripts/errors.txt' diff --git a/.github/workflows/validate-data.yaml b/.github/workflows/validate-data.yaml index bd9bc63c47..a6b60eaba3 100644 --- a/.github/workflows/validate-data.yaml +++ b/.github/workflows/validate-data.yaml @@ -7,10 +7,10 @@ on: - synchronize - reopened paths: - - 'ida/data.yml' + - 'scripts/data.yml' push: paths: - - 'ida/data.yml' + - 'scripts/data.yml' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -28,4 +28,4 @@ jobs: run: | npm install -g js-yaml - name: Validate data.yml - run: NODE_PATH="$(npm root -g):$NODE_PATH" node ./ida/data-validator.js \ No newline at end of file + run: NODE_PATH="$(npm root -g):$NODE_PATH" node ./scripts/data-validator.js \ No newline at end of file diff --git a/scripts/data-validator.js b/scripts/data-validator.js index 02ee3b0c1f..6c4e884960 100644 --- a/scripts/data-validator.js +++ b/scripts/data-validator.js @@ -15,7 +15,7 @@ function toHex(str) { return !isNaN(str) ? toHexStringFromNumber(parseInt(str)) : str; } try { - const doc = yaml.load(fs.readFileSync('./ida/data.yml', 'utf8')); + const doc = yaml.load(fs.readFileSync('./scripts/data.yml', 'utf8')); if (versionRegex.test(doc.version) === false) errors.push('Invalid version format'); if (typeof doc.globals == "object") From dab64f3da0bf2ea28d82868641b45a9f0eca28e4 Mon Sep 17 00:00:00 2001 From: wildwolf <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:19:22 +0200 Subject: [PATCH 65/95] move the last shared script files into the shared folder --- scripts/ida/rename.py | 2 +- scripts/ida/struct_import.py | 1 - scripts/{ => shared}/class_factory.py | 0 scripts/{ => shared}/indent_writer.py | 0 4 files changed, 1 insertion(+), 2 deletions(-) rename scripts/{ => shared}/class_factory.py (100%) rename scripts/{ => shared}/indent_writer.py (100%) diff --git a/scripts/ida/rename.py b/scripts/ida/rename.py index 808af9ff26..c75054c4b5 100644 --- a/scripts/ida/rename.py +++ b/scripts/ida/rename.py @@ -1,7 +1,7 @@ from os.path import join, dirname import sys -for path in (dirname(__file__), join(dirname(__file__), ".."), join(dirname(__file__), "..", "shared")): +for path in (dirname(__file__), join(dirname(__file__), "..", "shared")): if path not in sys.path: sys.path.append(path) diff --git a/scripts/ida/struct_import.py b/scripts/ida/struct_import.py index c22f0d6276..39231a5819 100644 --- a/scripts/ida/struct_import.py +++ b/scripts/ida/struct_import.py @@ -1,7 +1,6 @@ from os.path import join, dirname import sys -sys.path.append(join(dirname(__file__), "..")) sys.path.append(join(dirname(__file__), "..", "shared")) from lumina_wrapper import get_excel_header_files diff --git a/scripts/class_factory.py b/scripts/shared/class_factory.py similarity index 100% rename from scripts/class_factory.py rename to scripts/shared/class_factory.py diff --git a/scripts/indent_writer.py b/scripts/shared/indent_writer.py similarity index 100% rename from scripts/indent_writer.py rename to scripts/shared/indent_writer.py From bcf2f203ca278638602ac0ea6040b66f604a91a2 Mon Sep 17 00:00:00 2001 From: wildwolf <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:19:51 +0200 Subject: [PATCH 66/95] get a testing version of ghidra scripts to work --- scripts/ffxiv_exdgetters.py | 377 --------------- scripts/ffxiv_idarename.py | 331 -------------- scripts/ffxiv_structimporter.py | 718 ----------------------------- scripts/ghidra/clang_wrapper.py | 306 +++++++++++++ scripts/ghidra/full_run.py | 19 + scripts/ghidra/ghidra_wrapper.py | 755 +++++++++++++++++++++++++++++++ scripts/ghidra/rename.py | 69 +++ scripts/ghidra/struct_import.py | 205 +++++++++ 8 files changed, 1354 insertions(+), 1426 deletions(-) delete mode 100644 scripts/ffxiv_exdgetters.py delete mode 100644 scripts/ffxiv_idarename.py delete mode 100644 scripts/ffxiv_structimporter.py create mode 100644 scripts/ghidra/clang_wrapper.py create mode 100644 scripts/ghidra/full_run.py create mode 100644 scripts/ghidra/ghidra_wrapper.py create mode 100644 scripts/ghidra/rename.py create mode 100644 scripts/ghidra/struct_import.py diff --git a/scripts/ffxiv_exdgetters.py b/scripts/ffxiv_exdgetters.py deleted file mode 100644 index 85e2c50c06..0000000000 --- a/scripts/ffxiv_exdgetters.py +++ /dev/null @@ -1,377 +0,0 @@ -# Automagically labels most exd getter functions along with a hint indicating which sheet/sheet id its fetching from -# @category __UserScripts -# @menupath Tools.Scripts.ffxiv_exdgetters -# @runtime PyGhidra - -from json import load -from os import getenv -from os.path import join -import sys -from luminapie.game_data import GameData, ParsedFileName -from luminapie.excel import ExcelListFile, ExcelHeaderFile -from abc import abstractmethod - - -class BaseApi: - @abstractmethod - def create_enum_struct(self, name, values, width=0): - # type: (str, dict[int, str], int) -> None - pass - - @abstractmethod - def create_struct(self, name, fields): - # type: (str, dict[str, str]) -> None - pass - - @abstractmethod - def set_func_name(self, ea, name, cmt): - # type: (int, str, str) -> None - pass - - @abstractmethod - def process_pattern(self, pattern): - # type: (str) -> None - pass - - -api = None - -if api is None: - try: - import idaapi - except ImportError as e: - print("Warning: Unable to load IDA failed with {0}".format(e)) - else: - raise ValueError("Script moved to new location in ./ida/struct_import.py") - -if api is None: - try: - import ghidra - import re - - try: - from ghidra.ghidra_builtins import * # ghidra-stubs - except ImportError: - pass - - from ghidra.program.model.data import * - from ghidra.program.model.listing import * - from ghidra.program.model.symbol import SourceType - from ghidra.app.util import SymbolPathParser - from java.util import ArrayList - - except ImportError: - print("Warning: Unable to load Ghidra") - else: - # noinspection PyUnresolvedReferences - class GhidraApi(BaseApi): - def create_enum_struct(self, name, values, width=0): - # type: (str, dict[int, str], int) -> None - path = self.get_datatype_path(name) - enum_dt = EnumDataType( - path.getCategoryPath(), path.getDataTypeName(), width or 8 - ) - is_sheets_enum = name == "Component::Exd::SheetsEnum" - for enum_value, enum_name in values.items(): - if monitor.isCancelled(): - break - if not is_sheets_enum and "_" in enum_name: - enum_name = "".join(enum_name.split("_")[1:]) - enum_dt.add(enum_name, enum_value) - if width == 0: - enum_dt.setLength(enum_dt.getMinimumPossibleLength()) - if enum_dt.getName(0) is None: - enum_dt.add("None", 0) - dt = currentProgram.getDataTypeManager().getDataType(path) - if dt is None: - currentProgram.getDataTypeManager().addDataType(enum_dt, None) - else: - dt.replaceWith(enum_dt) - - def create_struct(self, name, fields): - # type: (str, dict[str, str]) -> None - dt_path = self.get_datatype_path(name) - struct = StructureDataType( - dt_path.getCategoryPath(), dt_path.getDataTypeName(), 0 - ) - struct.setExplicitMinimumAlignment(4) - for [offset, [type_name, field_name]] in fields.items(): - if monitor.isCancelled(): - break - field_dt = currentProgram.getDataTypeManager().getDataType( - self.get_datatype_path(type_name) - ) - if field_dt is None: - print( - f"Warning: Data type {name} field {field_name} has missing type {type_name} ({dt_path})" - ) - continue - struct.insertAtOffset(offset, field_dt, -1, field_name, None) - dt = currentProgram.getDataTypeManager().getDataType(dt_path) - if dt is None: - currentProgram.getDataTypeManager().addDataType(struct, None) - else: - dt.replaceWith(struct) - - def set_func_name(self, ea, name, cmt): - # type: (int, str, str) -> None - func = getFunctionAt(ea) - if func is not None: - func.setName(None, SourceType.DEFAULT) - func.setName(name, SourceType.USER_DEFINED) - func.setComment(cmt) - - def process_pattern(self, pattern): - # type: (str) -> None - suffix, _ = exd_func_patterns[pattern] - pattern = "".join( - ["\\x" + x if x != "?" else "." for x in pattern.split(" ")] - ) - if suffix is not None: - print(f"Finding exd funcs of {suffix}... please wait.") - self.do_pattern(pattern, suffix) - - def do_pattern(self, pattern, suffix): - # type: (string, string) -> None - block = currentProgram.getMemory().getBlock(".text") - address_set = currentProgram.getAddressFactory().getAddressSet( - block.getStart(), block.getEnd() - ) - result_list = findBytes(address_set, pattern, 8192, 1) - for ea in result_list: - if monitor.isCancelled(): - break - sheet_index = self.get_sheet_index(ea) - sheet_name = exd_map.get(sheet_index) - if sheet_name is None: - print( - f"Warning: sheet index {sheet_index} is undefined at {ea}" - ) - continue - func_name = f"Component::Exd::ExdModule.Get{sheet_name}{suffix}" - self.set_func_name( - ea, func_name, f"Sheet: {sheet_name} ({sheet_index})" - ) - if suffix == "SheetIndex": - return - - return_type = PointerDataType(VoidDataType.dataType) - if suffix == "RowCount": - return_type = IntegerDataType() - else: - path = self.get_datatype_path(f"{exd_struct_map[sheet_index]}") - dt = currentProgram.getDataTypeManager().getDataType(path) - if dt is not None: - return_type = PointerDataType(dt) - - return_var = ReturnParameterImpl(return_type, currentProgram) - arg_vars = ArrayList() - arg_vars.add( - ParameterImpl( - "rowId", UnsignedIntegerDataType(), currentProgram - ) - ) - if suffix == "RowAndSubRowId": - arg_vars.add( - ParameterImpl( - "subRowId", UnsignedIntegerDataType(), currentProgram - ) - ) - elif suffix == "RowCount": - arg_vars.clear() - - update_type = Function.FunctionUpdateType.DYNAMIC_STORAGE_ALL_PARAMS - getFunctionAt(ea).updateFunction( - "__fastcall", - return_var, - arg_vars, - update_type, - False, - SourceType.USER_DEFINED, - ) - - def get_sheet_index(self, ea): - # type: (Address) -> int - func = getFunctionAt(ea) - if func is None and disassemble(ea): - func = createFunction(ea, None) - if func is None: - return -1 - min_address = func.getBody().getMinAddress().getOffset() - instructions = currentProgram.getListing().getInstructions( - func.getBody(), True - ) - for insn in instructions: - if monitor.isCancelled(): - break - if insn.getFlowType().isCall(): - return self.get_rdx_arg(insn, min_address) - return -1 - - def get_rdx_arg(self, insn, min_address): - # type: (Instruction, int) -> int - target = insn.getRegister("RDX") - while ( - insn is not None and insn.getMinAddress().getOffset() >= min_address - ): - if monitor.isCancelled(): - break - insn = insn.getPrevious() - if insn is None: - break - outp = insn.getResultObjects() - if outp.length == 0 or outp[0] != target: - continue - value = insn.getScalar(1) - if value is not None: - return value.getValue() - return -1 - - def get_datatype_path(self, name): - # type: (str) -> DataTypePath - if name == "__int8": - name = "char" - if name == "__int16": - name = "short" - if name == "__int32": - name = "int" - if name == "__int64": - name = "longlong" - if name == "unsigned __int8" or name == "unsigned char": - name = "byte" - if name == "unsigned __int16" or name == "unsigned short": - name = "ushort" - if name == "unsigned __int32" or name == "unsigned int": - name = "uint" - if name == "unsigned __int64" or name == "unsigned long long": - name = "ulonglong" - path_parts = SymbolPathParser.parse(name) - return DataTypePath("/" + "/".join(path_parts[:-1]), path_parts[-1]) - - def comment_rows(self, pattern: dict[str, int], values: dict[int, str]): - pass - - api = GhidraApi() - -if api is None: - print("Warning: No API available, exiting.") - exit(1) - - -if sys.platform.startswith("linux"): - launcher_ini = join(getenv("HOME"), ".xlcore", "launcher.ini") - game_path = None - - f = open(launcher_ini, "r") - try: - for line in f: - if line.startswith("GamePath="): - game_path = line.split("=", 1)[1].strip() - break - finally: - f.close() - - if not game_path: - raise ValueError("GamePath key not found in {0}".format(launcher_ini)) -else: - f = open(join(getenv("APPDATA"), "XIVLauncher", "launcherConfigV3.json"), "r") - try: - config = load(f) - finally: - f.close() - game_path = config["GamePath"] - -game_data = GameData(join(game_path, "game")) - -# nb: "pattern": ("func suffix", ("instance pointer sig" or None, offset from sig start or func start)) -exd_func_patterns: dict[str, tuple[str, tuple[str | None, int]]] = { - "48 83 EC 28 48 8B 05 ? ? ? ? 44 8B C1 BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ( - "Row", - ("48 8B 05 ? ? ? ? 44 8B C1 BA", 11), - ), - "48 83 EC 28 85 C9 74 20 48 8B 05 ? ? ? ? 44 8B C1 BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ( - "Row", - ("48 8B 05 ? ? ? ? 44 8B C1 BA", 11), - ), - "48 83 EC 28 48 8B 05 ? ? ? ? BA ? ? ? ? 44 0F B6 C1 48 8B 88 ? ? ? ? E8": ( - "Row", - ("48 8B 05 ? ? ? ? BA", 8), - ), - "48 83 EC 28 48 8B 05 ? ? ? ? BA ? ? ? ? 44 0F B7 C1 48 8B 88 ? ? ? ? E8": ( - "Row", - ("48 8B 05 ? ? ? ? BA", 8), - ), - "48 83 EC 28 48 8B 05 ? ? ? ? BA ? ? ? ? 44 0F B7 81 ? ? ? ? 48 8B 88 ? ? ? ? E8": ( - "Row", - ("48 8B 05 ? ? ? ? BA", 8), - ), - "40 53 48 83 EC 20 48 8B 05 ? ? ? ? 44 8B C1 8B DA BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ( - "Row", - ("44 8B C1 8B DA BA", 6), - ), - "48 83 EC 28 48 8B 05 ? ? ? ? 44 8B C1 84 D2 BA ? ? ? ? 48 8B 88 ? ? ? ? 74 07 E8": ( - "Row", - ("44 8B C1 84 D2 BA", 6), - ), - "48 83 EC 38 48 8B 05 ? ? ? ? 44 8B CA 44 8B C1 48 C7 44 24 ? ? ? ? ? BA ? ? ? ? 48 C7 44 24 ? ? ? ? ? 48 8B 88 ? ? ? ? E8": ( - "RowAndSubRowId", - ("C1 48 C7 44 24 ? ? ? ? ? BA", 11), - ), - "48 83 EC 38 48 8B 05 ? ? ? ? 44 8B C1 44 0F ? CA BA ? ? ? ? 48 C7 44 24 28 ? ? ? ? 48 C7 44 24 20 ? ? ? ? 48 8B 88 ? ? ? ? E8": ( - "RowAndSubRowId", - ("44 8B C1 44 0F ? CA BA", 8), - ), - "48 83 EC 28 48 8B 05 ? ? ? ? 44 8D 81 ? ? ? ? BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ( - "RowIndex", - ("05 ? ? ? ? 44 8D 81 ? ? ? ? BA", 13), - ), - "48 83 EC 28 8D 41 ? 3D ? ? ? ? 77 20 48 8B 05 ? ? ? ? 44 8B C1 BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ( - "RowIndex", - ("48 8B 05 ? ? ? ? 44 8B C1 BA", 11), - ), - "48 83 EC 28 48 8B 05 ? ? ? ? BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ( - "SheetIndex", - ("48 8B 05 ? ? ? ? BA", 8), - ), - "48 8B 05 ? ? ? ? BA ? ? ? ? 48 8B 88 ? ? ? ? E9 ? ? ? ?": ("RowCount", (None, 8)), -} - -exd_comment_patterns: dict[str, int] = { - "48 8B 05 ? ? ? ? BA ? ? ? ? 48 8B 88 40 2B 00 00": 8, - "48 8B 05 ? ? ? ? BA ? ? ? ? 44 0F": 8, - "48 8B 05 ? ? ? ? 44 8B C1 8B DA BA": 13, - "48 8B 05 ? ? ? ? 44 8B C1 BA": 11, - "48 8B 05 ? ? ? ? 44 8B CA 44 8B C1 48 C7 44 24 ? ? ? ? ? BA": 23, - "48 8B 05 ? ? ? ? 44 8B C1 84 D2 BA": 13, - "48 8B 05 ? ? ? ? 44 8B C1 44 0F ? CA BA": 15, - "48 8B 05 ? ? ? ? 44 8D 81 ? ? ? ? BA": 15, -} - -exd_map = ExcelListFile(game_data.get_file(ParsedFileName("exd/root.exl"))).dict -exd_struct_map: dict[str, str] = {} -exd_headers: tuple[dict[int, tuple[str, str]], dict[str, dict[int, str]], int] = {} - -api.create_enum_struct("Component::Exd::SheetsEnum", exd_map, 4) - -for key in exd_map: - print(f"Parsing schema for {exd_map[key]}.") - exd_headers[key] = ExcelHeaderFile( - game_data.get_file(ParsedFileName("exd/" + exd_map[key] + ".exh")), exd_map[key] - ).map_names(game_data.get_exd_schema(exd_map[key])) - -for key in exd_headers: - [exd_header, exd_header_enums, exd_header_count] = exd_headers[key] - struct_name = f"Component::Exd::Sheets::{exd_map[key]}" - print(f"Creating struct {struct_name}.") - exd_struct_map[key] = struct_name - for enum_key in exd_header_enums: - api.create_enum_struct( - f"{struct_name}::{enum_key}", exd_header_enums[enum_key], 1 - ) - api.create_struct(struct_name, exd_header) - -# if a new pattern is found, add it to the exd_func_patterns dict -for func_pattern in exd_func_patterns: - api.process_pattern(func_pattern) - -api.comment_rows(exd_comment_patterns, exd_map) diff --git a/scripts/ffxiv_idarename.py b/scripts/ffxiv_idarename.py deleted file mode 100644 index ec1e3c29c5..0000000000 --- a/scripts/ffxiv_idarename.py +++ /dev/null @@ -1,331 +0,0 @@ -# current exe version: 2020.12.29.0000.0000 -# @category __UserScripts -# @menupath Tools.Scripts.ffxiv_idarename -# @runtime PyGhidra - -from __future__ import print_function -import os - -try: - from typing import Any, Dict, List, Optional, Union # noqa -except ImportError: - pass - -import sys -from abc import abstractmethod - -if sys.version_info[0] >= 3: - long = int - -# class_factory.py lives next to this script, the schema modules in ./shared -try: - script_dir = os.path.dirname(os.path.realpath(__file__)) -except NameError: - script_dir = os.path.dirname(str(sourceFile)) # noqa - -for path in (script_dir, os.path.join(script_dir, "shared")): - if path not in sys.path: - sys.path.append(path) - -from data_schema import get_yaml # noqa -from class_factory import apply_data # noqa - -# region Api - -class BaseApi(object): - @property - @abstractmethod - def data_file_path(self): - """ - Get the Path to the data.yml File - :return: Path to the data.yml File - :rtype: str - """ - - @abstractmethod - def get_image_base(self): - """ - Get the image base ea - :return: Image base ea - :rtype: int - """ - - @abstractmethod - def is_offset(self, ea): - """ - Is the given address an offset to something else - :param ea: Effective address - :type ea: int - :return: Is offset or not - :rtype: bool - """ - - @abstractmethod - def xrefs_to(self, ea): - """ - Retrieve all xrefs to the given address - :param ea: Effective address - :type ea: int - :return: List of addresses - :rtype: List[int] - """ - - @abstractmethod - def get_qword(self, ea): - """ - Read a qword of data from an address - :param ea: Effective address - :type ea: int - :return: 64bits of data - :rtype: int - """ - - @abstractmethod - def get_addr_name(self, ea): - """ - Get the name of a given address - :param ea: Effective address - :type ea: int - :return: Name or empty - :rtype: str - """ - - @abstractmethod - def set_addr_name(self, ea, name): - """ - Set the name of a given address (function) - :param ea: Effective address - :type ea: int - :param name: Name to set - :type name: str - :return: Success/failure - :rtype: bool - """ - - @abstractmethod - def get_comment(self, ea): - """ - Get the comment at an address - :param ea: Effective address - :type ea: int - :return: Comment - :rtype: str - """ - - @abstractmethod - def set_comment(self, ea, comment): - """ - Add a comment to an address - :param ea: Effective address - :type ea: int - :param comment: Comment - :type comment: str - :return: None - """ - - def format_class_name_for_vtbl(self, class_name): - """ - Format a class name for special representation in the vtbl - By default, looks similar to vtbl_Component::Gui::AtkResNode - :param class_name: Name as contained in data.yml - :type class_name: str - :return: Formatted class name - :rtype: str - """ - return "vtbl_{0}".format(class_name) - - def format_class_name_for_secondary_vtbl(self, class_name, secondary_name): - """ - Format a class name for special representation in the vtbl - By default, looks similar to vtbl_classname_secondaryname - :param class_name: Name as contained in data.yml - :type class_name: str - :param secondary_name: Name as contained in data.yml - :type secondary_name: str - :return: Formatted class name - :rtype: str - """ - return "vtbl_{0}___{1}".format(class_name, secondary_name) - - def format_class_name(self, class_name): - """ - Format a class name for representation in the toolset - By default, looks similar to Component::Gui::AtkResNode - :param class_name: Name as contained in data.yml - :type class_name: str - :return: Formatted class name - :rtype: str - """ - return class_name - - @abstractmethod - def format_vfunc_name(self, ea, current_func_name, proposed_func_name, class_name, base_class_names): - """ - Name a function (vfunc) based on how it is currently named, what it is proposed to be named, - and the names of the current and base classes. If None is returned, it is assumed - that the current func was named something unexpectedly and will be warned about. Return - an empty string "" if no renaming should take place. - :param ea: Func effective address - :type ea: int - :param current_func_name: Current func name as in the disassembler - :type current_func_name: str - :param proposed_func_name: Either a name from data.yml or vfX where X is the vfunc index - :type proposed_func_name: str - :param class_name: Class name - :type class_name: str - :param base_class_names: base class names - :type base_class_names: List[str] - :return: Formatted vfunc name - :rtype: Optional[str] - """ - - @abstractmethod - def format_func_name(self, ea, current_func_name, proposed_func_name, class_name): - """ - Name a function based on how it is currently named, what it is proposed to be named, - and the name of the current class. If None is returned, it is assumed that the current - func was named something unexpectedly and will be warned about. Return an empty string "" - if no renaming should take place. - :param ea: Func effective address - :type ea: int - :param current_func_name: Current func name as in the disassembler - :type current_func_name: str - :param proposed_func_name: A name from data.yml - :type proposed_func_name: str - :param class_name: Class name - :type class_name: str - :return: Formatted func name - :rtype: Optional[str] - """ - - -api = None - -# region IDA Api - -if api is None: - try: - import idaapi # noqa - except ImportError: - print("Warning: Unable to load IDA") - else: - raise ValueError("Script moved to new location in ./ida/rename.py") - -# endregion -# region Ghidra Api - -if api is None: - try: - import ghidra - - from ghidra.program.model.data import CategoryPath # noqa - from ghidra.program.model.data import StructureDataType # noqa - from ghidra.program.model.data import PointerDataType # noqa - from ghidra.program.model.symbol import SourceType # noqa - except ImportError: - print("Warning: Unable to load Ghidra") - else: - def toAddr(offset): # noqa - # type: (int) -> Address - return currentProgram.getAddressFactory().getDefaultAddressSpace().getAddress(offset) # noqa - - # noinspection PyUnresolvedReferences - class GhidraApi(BaseApi): - @property - def data_file_path(self): - return os.path.join(os.path.dirname(str(sourceFile)), "data.yml") - - def get_image_base(self): - return currentProgram.getImageBase().getOffset() - - def is_offset(self, ea): - data = getDataAt(toAddr(ea)) - if not data: return False - return data.isPointer() - - def xrefs_to(self, ea): - return [xref.getFromAddress().getOffset() for xref in getReferencesTo(toAddr(ea))] - - def get_qword(self, ea): - return getLong(toAddr(ea)) - - def get_addr_name(self, ea): - sym = getSymbolAt(toAddr(ea)) - if not sym and getByte(toAddr(ea).subtract(1)) & 0xFF == 0xCC: - disassemble(toAddr(ea)) - createFunction(toAddr(ea), None) - sym = getSymbolAt(toAddr(ea)) - if not sym: - return "" - return sym.getName(True) - - def set_addr_name(self, ea, name): - createLabel(toAddr(ea), name, True, SourceType.ANALYSIS) - return True - # return createLabel(toAddr(ea), name, True).checkIsValid() - - def get_comment(self, ea): - return getEOLComment(toAddr(ea)) - - def set_comment(self, ea, comment): - if getEOLComment(toAddr(ea)) is None: - setEOLComment(toAddr(ea), comment) - - def format_vfunc_name(self, ea, current_func_name, proposed_func_name, class_name, base_class_names): - if current_func_name.startswith("thunk_"): # jump - current_func_name = current_func_name.lstrip("thunk_") - - # Previously renamed as a vfunc - if current_func_name.startswith(class_name): - # return the proposed func name in case it was updated since last run - current_class_name = current_func_name.rsplit(".", 1)[0] - return "{0}.{1}".format(current_class_name, proposed_func_name) - - # This should have been handled in the base class - if any(current_func_name.startswith(name) for name in base_class_names): - return "" - - if any(current_func_name.startswith(prefix) for prefix in ("FUN_", "LAB_", "SUB_", "LOC_", "DAT_")): - return "{0}.{1}".format(class_name, proposed_func_name) - - if current_func_name == "_purecall": - return "" - - return None - - def format_func_name(self, ea, current_func_name, proposed_func_name, class_name): - # override default thunk names and thunks of already named funcs with the default - func = getFunctionAt(toAddr(ea)) - if func and func.isThunk(): - current_func_name = "FUN_{0:x}".format(ea) - - proposed_qualified_func_name = "{0}.{1}".format(class_name, proposed_func_name) - if current_func_name == proposed_qualified_func_name: - return "" - - if any(current_func_name.startswith(prefix) for prefix in ("FUN_", "LAB_", "SUB_", "LOC_", "DAT_")): - return proposed_qualified_func_name - - return None - - - api = GhidraApi() - -# endregion - -if api is None: - raise Exception("Unable to load API (supported: IDA, Ghidra, Binary Ninja)") - - -# endregion - -def load_data(): - with open(api.data_file_path, "r") as fd: - data = get_yaml(fd) - - apply_data(api, data) - - -print("Executing") -load_data() -print("Done") diff --git a/scripts/ffxiv_structimporter.py b/scripts/ffxiv_structimporter.py deleted file mode 100644 index 2df03db4e2..0000000000 --- a/scripts/ffxiv_structimporter.py +++ /dev/null @@ -1,718 +0,0 @@ -# @category __UserScripts -# @menupath Tools.Scripts.ffxiv_structimport -# @runtime PyGhidra - -from yaml import load - -try: - from yaml import CSafeLoader as Loader -except ImportError: - from yaml import SafeLoader as Loader - -import os -import sys -from abc import abstractmethod -from time import time - -# The schema and wrapper modules live in ./shared -try: - script_dir = os.path.dirname(os.path.realpath(__file__)) -except NameError: - script_dir = os.path.dirname(str(sourceFile)) # noqa - -for path in (script_dir, os.path.join(script_dir, "shared")): - if path not in sys.path: - sys.path.append(os.path.join(script_dir, "shared")) - -from structs_schema import * -from src_wrapper import SrcInterface - - -predefine = """#define _HAS_ITERATOR_DEBUGGING 0 -#define _ITERATOR_DEBUG_LEVEL 0 -#include -#include -#include -#include -#include -#include -#include - -""" - -class BaseApi: - @abstractmethod - def can_run(self): - # type: () -> None - """ - Checks if exdgetters has run before this is allowed to continue - """ - - @abstractmethod - def create_enum_struct(self, enum): - # type: (DefinedStructEnum) -> None - """ - Create an enum in the database. - """ - - @abstractmethod - def delete_enum(self, enum): - # type: (DefinedStructEnum) -> None - """ - Delete an enum in the database. - """ - - @abstractmethod - def delete_struct(self, struct): - # type: (DefinedStruct) -> None - """ - Delete a struct in the database. - """ - - @abstractmethod - def create_struct(self, struct): - # type: (DefinedStruct) -> None - """ - Create a struct in the database. - """ - - @abstractmethod - def create_struct_members(self, struct): - # type: (DefinedStruct) -> None - """ - Create members for a struct in the database. - """ - - @abstractmethod - def create_vtable(self, struct): - # type: (DefinedStruct) -> None - """ - Create a vtable in the database. - """ - - @abstractmethod - def finalise_struct(self, struct): - # type: (DefinedStruct) -> None - """ - Finalise a struct in the database. - """ - - @abstractmethod - def create_union(self, struct): - # type: (DefinedStruct) -> None - """ - Create a union in the database. - """ - - @abstractmethod - def update_member_func(self, member_func, struct): - # type: (DefinedStructMemFunc, DefinedStruct) -> None - """ - Updates a member function in the database. - """ - - @abstractmethod - def update_virt_func(self, virt_func, struct): - # type: (DefinedStructVFunc, DefinedStruct) -> None - """ - Updates a virtual function in the database. - """ - - @abstractmethod - def update_static_member(self, static_member, struct): - # type: (DefinedStructStaticMember, DefinedStruct) -> None - """ - Updates a static member in the database. - """ - - @abstractmethod - def should_update_member_func(self): - # type: () -> bool - """ - Returns if the member function types should be updated. - """ - - @abstractmethod - def should_update_virt_func(self): - # type: () -> bool - """ - Returns if the virtual function types should be updated. - """ - - @property - @abstractmethod - def get_file_path(self): - """ - Retrieve the file path of the yaml file. - """ - - def get_yaml(self): - # type: () -> DefinedStructExport - with open(self.get_file_path, "r") as fd: - return get_yaml(fd) - - @abstractmethod - def preprocess_yaml(self, yml: DefinedStructExport): - """ - Preprocesses the YAML data before importing. - - For the IDA srclang importer, this fixes issues with generic base classes. - """ - - def load_data_yaml(self): - # type: () -> dict - path = os.path.join(os.path.dirname(self.get_file_path), "data.yml") - if not os.path.exists(path): - return None - with open(path, "r") as fd: - return load(fd, Loader=Loader) - -api = None - -if api is None: - try: - import idaapi - except ImportError: - print("Warning: Unable to load IDA") - else: - raise ("Script moved to new location in ./ida/struct_import.py") - -if api is None: - try: - import ghidra - import re - - try: - from ghidra.ghidra_builtins import * - except ImportError: - pass - - from yaml import SafeLoader as Loader - - from ghidra.program.model.data import * - from ghidra.program.model.listing import * - from ghidra.program.model.symbol import SourceType - from ghidra.app.util import SymbolPathParser - from java.util import ArrayList - - except ImportError: - print("Warning: Unable to load Ghidra") - else: - # noinspection PyUnresolvedReferences - - class GhidraApi(BaseApi): - def can_run(self): - return True - - def get_size_from_type(self, name): - # type: (str) -> int - dt = self.get_datatype(name) - if dt is not None: - return dt.getLength() - return 0 - - def fix_generic_name(self, name): - # type: (str) -> str - if "<" not in name: - return name - for match in re.finditer(r"unsigned _*[\w*]{3,}|[:\w*]{3,}", name): - tn = self.get_ghidra_type( - SymbolPathParser.parse(match.group(0)).getLast() - ) - name = name.replace(match.group(0), tn) - return name - - def get_ghidra_type(self, name): - # type: (str) -> str - if name == "__int8": - return "char" - elif name == "__int16": - return "short" - elif name == "__int64": - return "longlong" - elif name == "unsigned __int16": - return "ushort" - elif name == "unsigned int": - return "uint" - elif name == "unsigned __int64": - return "ulonglong" - elif name == "__int8*": - return "char*" - elif name == "__int16*": - return "short*" - elif name == "__int64*": - return "longlong*" - elif name == "unsigned __int16*": - return "ushort*" - elif name == "unsigned int*": - return "uint*" - elif name == "unsigned __int64*": - return "ulonglong*" - elif name == "__fastcall": - return "void*" - return name - - def get_category_path(self, typename): - # type: (str) -> CategoryPath - syms = SymbolPathParser.parse(typename) - return CategoryPath("/" + "/".join(syms.subList(0, syms.size() - 1))) - - def get_datatype(self, typename): - # type: (str) -> DataType - raw_type = self.get_ghidra_type(typename) - if not raw_type: - return raw_type - typename = raw_type.rstrip("*") - pointer_count = len(raw_type) - len(typename) - - syms = SymbolPathParser.parse(typename) - syms[-1] = self.fix_generic_name(syms.getLast()) - - dtm = currentProgram.getDataTypeManager() - dt = dtm.getDataType("/" + "/".join(syms)) - for i in range(pointer_count): - dt = dtm.getPointer(dt) - return dt - - def create_datatype(self, datatype): - # type: (DataType) -> DataType - dtm = currentProgram.getDataTypeManager() - old = dtm.getDataType(datatype.getDataTypePath()) - if old is not None: - old.replaceWith(datatype) - return old - else: - return dtm.addDataType(datatype, None) - - def create_function_def(self, func): - # type: (DefinedStructVFunc) -> FunctionDefinitionDataType - fd = FunctionDefinitionDataType(func.name) - return_type = self.get_datatype(func.return_type) - fd.setReturnType(return_type) - args = [] - for arg in func.parameters: - arg_type = self.get_datatype(arg.type) - ad = ParameterDefinitionImpl(arg.name, arg_type, None) - args.append(ad) - fd.setArguments(args) - return fd - - def get_func_by_name(self, name): - # type: (str) -> Function - funcs = getGlobalFunctions(name) - return funcs.first if not funcs.size() == 0 else None - - def create_memberfunc_args(self, member_func): - # type: (DefinedStructMemFunc) -> ArrayList - arg_vars = ArrayList() - for param in member_func.parameters: - dt = self.get_datatype(param.type) - if not dt: - return ArrayList() - arg_vars.add(ParameterImpl(param.name, dt, currentProgram)) - return arg_vars - - @property - def get_file_path(self): - return os.path.join( - os.path.dirname(str(sourceFile)), "ffxiv_structs.yml" - ) - - def create_enum_struct(self, enum): - # type: (DefinedStructEnum) -> None - if monitor.isCancelled(): - return - enum_size = self.get_size_from_type(enum.underlying) or 4 - dt = EnumDataType(enum.name, enum_size) - dt.setCategoryPath(self.get_category_path(enum.type)) - for value in enum.values: - if not dt.contains(enum.values[value]): - dt.add(value, enum.values[value]) - self.create_datatype(dt) - - def delete_enum(self, enum): - # type: (DefinedStructEnum) -> None - pass - - def delete_struct(self, struct): - # type: (DefinedStruct) -> None - pass - - def create_struct(self, struct): - # type: (DefinedStruct) -> None - if monitor.isCancelled(): - return - - name = struct.name - syms = SymbolPathParser.parse(struct.type) - if syms.size() > 0: - name = syms.getLast() - - name = self.fix_generic_name(name) - if struct.union: - dt = UnionDataType(name) - else: - dt = StructureDataType(name, struct.size or 0) - dt.setCategoryPath(self.get_category_path(struct.type)) - self.create_datatype(dt) - - def create_struct_members(self, struct): - # type: (DefinedStruct) -> None - dt = self.get_datatype(struct.type) - if dt is None: - return - - struct.fields.sort(key=lambda fld: fld.offset) - dtsize = dt.getLength() if not dt.isZeroLength() else 0 - if ( - dtsize == 0 - and struct.virtual_functions is not None - and not struct.union - ): - dt.growStructure(8) - - for field in struct.fields: - if monitor.isCancelled(): - return - - offset = field.offset - dtsize = dt.getLength() if not dt.isZeroLength() else 0 - - ft = self.get_datatype(field.type) - if ft is None: - continue - - if isinstance(field, DefinedStructFixedField): - ft = ArrayDataType(ft, int(field.size), ft.getLength() or -1) - - if not struct.union: - if dtsize <= offset and not struct.size: - dt.growStructure(((offset - dtsize) or 0) + ft.getLength()) - - if ( - dt.getLength() <= offset - or dt.getLength() < offset + ft.getLength() - ): - print( - f"Field {field.name} (off=0x{offset:X} size=0x{ft.getLength():X}) not within Struct {dt.getDataTypePath()} (size=0x{dt.getLength():X})" - ) - break - - dt.replaceAtOffset(offset, ft, -1, field.name, "") - else: - dt.add(ft, ft.getLength(), field.name, "") - - def create_vtable(self, struct): - # type: (DefinedStruct) -> None - if monitor.isCancelled(): - return - dtm = currentProgram.getDataTypeManager() - dt = self.get_datatype(struct.type) - - struct.virtual_functions.sort(key=lambda fn: fn.offset) - vt_type = StructureDataType("VTable", 0) - vt_type.setCategoryPath( - CategoryPath(dt.getCategoryPath(), [dt.getName()]) - ) - vt_type = self.create_datatype(vt_type) - if struct.fields != [] and struct.fields[0].offset == 0: - u_type = UnionDataType("Union") - u_type.setCategoryPath( - CategoryPath(dt.getCategoryPath(), [dt.getName()]) - ) - u_type.add(dtm.getPointer(vt_type), -1, "VTable", "") - comp = dt.getComponentContaining(0) - if comp and not Undefined.isUndefined(comp.getDataType()): - u_type.add( - comp.getDataType(), -1, comp.getFieldName(), "parent class" - ) - self.create_datatype(u_type) - - void_ptr = dtm.getPointer(VoidDataType.dataType) - for func in struct.virtual_functions: - if func.return_type and func.parameters: - func_def = self.create_function_def(func) - func_def.setCategoryPath( - CategoryPath(vt_type.getCategoryPath(), [vt_type.getName()]) - ) - vt_type.insertAtOffset( - func.offset, - dtm.getPointer(func_def), - -1, - func.name, - f"vf{int(func.offset / 8)}", - ) - else: - dtc = vt_type.getComponentAt(func.offset) - if dtc and Undefined.isUndefined(dtc.getDataType()): - vt_type.replaceAtOffset( - func.offset, - void_ptr, - -1, - func.name, - f"vf{int(func.offset / 8)}", - ) - else: - vt_type.insertAtOffset( - func.offset, - void_ptr, - -1, - func.name, - f"vf{int(func.offset / 8)}", - ) - - if struct.vtable_size: - vt_size = struct.vtable_size - vt_type.setLength(vt_size) - else: - vt_size = struct.virtual_functions[-1].offset - for offset in range(0, vt_size, 8): - dtc = vt_type.getComponentContaining(offset) - if not dtc or Undefined.isUndefined(dtc.getDataType()): - vt_type.replaceAtOffset( - offset, void_ptr, -1, f"vf{int(offset / 8)}", None - ) - - def finalise_struct(self, struct): - return - - def create_union(self, struct): - # type: (DefinedStruct) -> None - if monitor.isCancelled() or not struct.virtual_functions: - return - - dtm = currentProgram.getDataTypeManager() - void_ptr = dtm.getPointer(VoidDataType.dataType) - dt = self.get_datatype(struct.type) - u_type = self.get_datatype(struct.type + "::Union") - vt_type = self.get_datatype(struct.type + "::VTable") - - if vt_type: - dtc = dt.getComponentContaining(0) - while dtc and not Undefined.isUndefined(dtc.getDataType()): - if monitor.isCancelled(): - return - parent = dtc.getDataType() - parent_vt = dtm.getDataType( - CategoryPath(parent.getCategoryPath(), [parent.getName()]), - "VTable", - ) - if parent_vt: - if parent_vt.getLength() > vt_type.getLength(): - vt_type.replaceWith(parent_vt) - else: - for c in parent_vt.getComponents(): - if ( - vt_type.getComponentContaining(c.getOffset()) - .getDataType() - .equals(void_ptr) - ): - vt_type.replaceAtOffset( - c.getOffset(), - c.getDataType(), - -1, - c.getFieldName(), - c.getComment(), - ) - dtc = parent_vt.getComponentContaining(0) - else: - break - - if u_type and struct.fields != [] and struct.fields[0].offset == 0: - dt.replaceAtOffset( - 0, u_type, -1, "Union", "vtable and parent union" - ) - elif vt_type: - dt.replaceAtOffset(0, dtm.getPointer(vt_type), -1, "VTable", "") - - def update_member_func(self, member_func, struct): - # type: (DefinedStructMemFunc, DefinedStruct) -> None - if monitor.isCancelled(): - return - if not member_func.parameters: - return - func_name = f"{struct.type}.{member_func.name}" - func = self.get_func_by_name(func_name) - if not func: - return - arg_vars = self.create_memberfunc_args(member_func) - return_type = self.get_datatype(member_func.return_type) - if not return_type: - return - return_var = ReturnParameterImpl(return_type, currentProgram) - update_type = Function.FunctionUpdateType.DYNAMIC_STORAGE_ALL_PARAMS - func.updateFunction( - "__fastcall", - return_var, - arg_vars, - update_type, - False, - SourceType.USER_DEFINED, - ) - - def update_virt_func(self, virt_func, struct): - # type: (DefinedStructVFunc, DefinedStruct) -> None - if monitor.isCancelled(): - return - func_name = f"{struct.type}.{virt_func.name}" - func = self.get_func_by_name(func_name) - if not func: - return - arg_vars = self.create_memberfunc_args(virt_func) - return_type = self.get_datatype(virt_func.return_type) - if not return_type: - return - return_var = ReturnParameterImpl(return_type, currentProgram) - update_type = Function.FunctionUpdateType.DYNAMIC_STORAGE_ALL_PARAMS - func.updateFunction( - "__fastcall", - return_var, - arg_vars, - update_type, - False, - SourceType.USER_DEFINED, - ) - - def update_static_member(self, static_member, struct): - # type: (DefinedStructStaticMember, DefinedStruct) -> None - pass - - def should_update_member_func(self): - # type: () -> bool - return askYesNo("ffxiv_structimporter", "Update member function types?") - - def should_update_virt_func(self): - # type: () -> bool - return askYesNo( - "ffxiv_structimporter", "Update virtual function types?" - ) - - def preprocess_yaml(self, yaml: DefinedStructExport): - return - - api = GhidraApi() - - -if api is None: - raise Exception("Unable to load API (supported: IDA, Ghidra, Binary Ninja)") - -start_time = time() - - -def get_time(): - val = round(time() - start_time, 6).__str__() - while val.split(".")[-1].__len__() < 6: - val += "0" - return val - -def run(): - if not api.can_run(): - raise RuntimeError("This script depends on exdgetters. Run that script before retrying") - - update_virt_func = api.should_update_virt_func() - update_member_func = api.should_update_member_func() - - print("{0} Loading yaml".format(get_time())) - yaml = api.get_yaml() - - print("{0} Loading data yaml".format(get_time())) - api.data_yaml = api.load_data_yaml() - - if isinstance(api, IdaApi): - print("{0} Generating src file in memory".format(get_time())) - src_interface = SrcInterface() - header, vtables = src_interface.build_export_string(yaml) - - predef_header = """#define _HAS_ITERATOR_DEBUGGING 0 -#define _ITERATOR_DEBUG_LEVEL 0 -#include -#include -#include -#include -#include -#include -#include - -""" - - print("{0} Importing src file".format(get_time())) - api.select_parser() - api.parse_string(predef_header + header, vtables) - else: - print("{0} Deleting old structs".format(get_time())) - for struct in yaml.structs[::-1]: - api.delete_struct(struct) - - print("{0} Deleting old enums and creating new ones".format(get_time())) - for enum in yaml.enums: - api.delete_enum(enum) - - for enum in yaml.enums: - api.create_enum_struct(enum) - - print("{0} Creating new structs".format(get_time())) - for struct in yaml.structs: - api.create_struct(struct) - - print("{0} Creating members for structs".format(get_time())) - for struct in yaml.structs: - api.create_struct_members(struct) - - print("{0} Finalising structs".format(get_time())) - for struct in yaml.structs: - api.finalise_struct(struct) - - print("{0} Creating vtables for structs".format(get_time())) - for struct in yaml.structs: - if struct.virtual_functions: - api.create_vtable(struct) - - print("{0} Mapping unions/vtables for structs".format(get_time())) - for struct in yaml.structs: - api.create_union(struct) - - if update_virt_func: - for struct in yaml.structs: - if struct.virtual_functions: - print( - "{0} Updating virtual functions for {1}".format( - get_time(), struct.type - ) - ) - for virt_func in struct.virtual_functions: - if virt_func is None: - continue - - if virt_func.return_type != None or virt_func.parameters != None: - api.update_virt_func(virt_func, struct) - - if update_member_func: - for struct in yaml.structs: - if struct.member_functions != []: - print( - "{0} Updating member functions for {1}".format( - get_time(), struct.type - ) - ) - for member_func in struct.member_functions: - api.update_member_func(member_func, struct) - - if struct.static_member_functions: - print( - "{0} Updating static member functions for {1}".format( - get_time(), struct.type - ) - ) - for member_func in struct.static_member_functions: - api.update_member_func(member_func, struct) - - if struct.static_members: - print( - "{0} Updating static members for {1}".format( - get_time(), struct.type - ) - ) - for member in struct.static_members: - api.update_static_member(member, struct) - -run() diff --git a/scripts/ghidra/clang_wrapper.py b/scripts/ghidra/clang_wrapper.py new file mode 100644 index 0000000000..1ccd6ad68f --- /dev/null +++ b/scripts/ghidra/clang_wrapper.py @@ -0,0 +1,306 @@ +import os +import re + +# the crash recovery of libclang takes the SIGSEGV the JVM raises on purpose and kills Ghidra, read when the index is created +os.environ.setdefault("LIBCLANG_DISABLE_CRASH_RECOVERY", "1") + +from clang.cindex import Config, Cursor, CursorKind, Index, Type, TypeKind +from structs_schema import ( + DefinedStruct, + DefinedStructEnum, + DefinedStructExport, + DefinedStructField, + DefinedStructFixedField, + DefinedStructFuncField, + DefinedStructFuncParam, +) + +main_file = "ffxiv_structs.hpp" +# an instantiation can use templates that are only pointed to themselves +max_instantiation_passes = 8 + +# named like ffxiv_structs.yml does, sized for x86_64-pc-windows-msvc +primitive_types = { + TypeKind.VOID: "void", + TypeKind.BOOL: "bool", + TypeKind.CHAR_S: "char", + TypeKind.CHAR_U: "byte", + TypeKind.SCHAR: "__int8", + TypeKind.UCHAR: "byte", + TypeKind.WCHAR: "wchar_t", + TypeKind.CHAR16: "unsigned __int16", + TypeKind.CHAR32: "unsigned int", + TypeKind.SHORT: "__int16", + TypeKind.USHORT: "unsigned __int16", + TypeKind.INT: "int", + TypeKind.UINT: "unsigned int", + TypeKind.LONG: "int", + TypeKind.ULONG: "unsigned int", + TypeKind.LONGLONG: "__int64", + TypeKind.ULONGLONG: "unsigned __int64", + TypeKind.FLOAT: "float", + TypeKind.DOUBLE: "double", + TypeKind.LONGDOUBLE: "double", + TypeKind.NULLPTR: "void*", +} + +record_kinds = (CursorKind.STRUCT_DECL, CursorKind.CLASS_DECL, CursorKind.UNION_DECL) +pointer_kinds = (TypeKind.POINTER, TypeKind.LVALUEREFERENCE, TypeKind.RVALUEREFERENCE) +function_kinds = (TypeKind.FUNCTIONPROTO, TypeKind.FUNCTIONNOPROTO) + + +def split_type_path(name: str) -> list[str]: + """Split a type on its namespaces, the ones inside template arguments are left alone.""" + parts, depth, start, index = [], 0, 0, 0 + while index < len(name): + if name[index] == "<": + depth += 1 + elif name[index] == ">": + depth -= 1 + elif depth == 0 and name[index : index + 2] == "::": + parts.append(name[start:index]) + start = index + 2 + index += 1 + index += 1 + parts.append(name[start:]) + return parts + + +class RecordCollector(object): + """Turns the records clang laid out into the same structs ffxiv_structs.yml is read as.""" + + def __init__(self): + self.enums: dict[str, DefinedStructEnum] = {} + # in dependency order, a struct comes after everything it holds by value + self.structs: dict[str, DefinedStruct] = {} + # only ever pointed to, so they are imported without their members + self.opaque: dict[str, bool] = {} + # typedef name to the type behind it, see get_template_alias in structs_schema + self.aliases: dict[str, str] = {} + # template instances that are only pointed to, clang did not lay those out + self.incomplete: set[str] = set() + # records that are pointed to, laid out once the struct pointing to them is done to keep the dependency order + self._deferred: list[tuple[Type, Cursor, str]] = [] + + def collect(self, cursor: Cursor): + """Everything declared in the parsed string, and whatever that needs from the headers it includes.""" + self._collect_declarations(cursor) + while self._deferred: + type, decl, name = self._deferred.pop() + if name not in self.structs: + self.add_record(type, decl, name) + + def _collect_declarations(self, cursor: Cursor): + for child in cursor.get_children(): + if child.location.file is None or child.location.file.name != main_file: + continue + if child.kind in (CursorKind.NAMESPACE, CursorKind.LINKAGE_SPEC): + self._collect_declarations(child) + elif child.kind in record_kinds and child.is_definition(): + if "(" not in child.type.spelling: + self.get_type_string(child.type, True, True) + self._collect_declarations(child) + elif child.kind == CursorKind.ENUM_DECL and child.is_definition(): + self.get_enum(child) + elif child.kind == CursorKind.TYPEDEF_DECL and child.spelling.startswith("__cs_"): + type_string = self.get_type_string(child.underlying_typedef_type, True, False) + if type_string is not None: + self.aliases[child.spelling] = type_string + + def get_export(self) -> DefinedStructExport: + opaque = [ + self.new_struct(name, [], None, union) + for name, union in self.opaque.items() + if name not in self.structs + ] + return DefinedStructExport(list(self.enums.values()), [*opaque, *self.structs.values()]) + + def new_struct(self, type_name: str, fields: list, size: int | None, union: bool) -> DefinedStruct: + parts = split_type_path(type_name) + return DefinedStruct( + parts[-1], type_name, ".".join(parts[:-1]), fields, size, + None, [], [], union, None, None, [], [], [], False, + ) + + def get_decl_name(self, decl: Cursor) -> str: + name = re.sub(r"\s+([*&])", r"\1", decl.type.spelling) + if "(" in name: + # ex. Foo::(unnamed union at file:9:175) + name = f"{name[:name.index('(')]}anon_{decl.location.line}_{decl.location.column}" + return name + + def get_type_string(self, type: Type, follow: bool, by_value: bool) -> str | None: + """ + :param follow: Lay out the records this type points to, they are only declared by name otherwise + :param by_value: The type is held and not pointed to, so its layout is needed + :return: None if the type can not be described + """ + type = type.get_canonical() + if type.kind in primitive_types: + return primitive_types[type.kind] + + if type.kind in pointer_kinds: + pointee = type.get_pointee().get_canonical() + if pointee.kind in function_kinds: + return "__fastcall" + return (self.get_type_string(pointee, follow, False) or "void") + "*" + + if type.kind in (TypeKind.CONSTANTARRAY, TypeKind.INCOMPLETEARRAY) and not by_value: + # a pointer to an array points at its first element + return self.get_type_string(type.element_type, follow, False) + + if type.kind == TypeKind.ENUM: + return self.get_enum(type.get_declaration()) + + if type.kind == TypeKind.RECORD: + return self.get_record(type, follow, by_value) + + return None + + def get_enum(self, decl: Cursor) -> str: + decl = decl.get_definition() or decl + name = self.get_decl_name(decl) + if name not in self.enums: + short_name = split_type_path(name)[-1] + values = { + child.spelling.removeprefix(f"{short_name}_"): child.enum_value + for child in decl.get_children() + if child.kind == CursorKind.ENUM_CONSTANT_DECL + } + underlying = primitive_types.get(decl.enum_type.get_canonical().kind, "int") + self.enums[name] = DefinedStructEnum( + short_name, name, ".".join(split_type_path(name)[:-1]), underlying, False, values + ) + return name + + def get_record(self, type: Type, follow: bool, by_value: bool) -> str: + decl = type.get_declaration() + decl = decl.get_definition() or decl + name = self.get_decl_name(decl) + if name in self.structs: + return name + + if type.get_size() >= 0 and by_value: + self.add_record(type, decl, name) + elif type.get_size() >= 0 and follow: + self._deferred.append((type, decl, name)) + else: + if follow and "<" in name: + self.incomplete.add(name) + self.opaque[name] = decl.kind == CursorKind.UNION_DECL + return name + + def add_record(self, type: Type, decl: Cursor, name: str): + # what a header of the compiler or windows points to is not followed, that would pull in most of it + follow = decl.location.file is not None and decl.location.file.name == main_file + fields = [] + + # clang does not report where a base is, they are laid out in order before the fields + offset = 0 + for child in decl.get_children(): + if child.kind != CursorKind.CXX_BASE_SPECIFIER: + continue + base = child.type.get_canonical() + if base.get_size() < 0 or self.is_empty(base): + continue + offset = -(-offset // base.get_align()) * base.get_align() + fields.append( + DefinedStructField(f"baseclass_{offset:X}", self.get_type_string(base, follow, True), offset, True, []) + ) + offset += base.get_size() + + for field in type.get_fields(): + bit_offset = field.get_field_offsetof() + if field.is_bitfield() or bit_offset < 0 or bit_offset % 8 != 0: + continue + defined_field = self.get_field(field, bit_offset // 8, follow) + if defined_field is not None: + fields.append(defined_field) + + self.structs[name] = self.new_struct(name, fields, type.get_size(), decl.kind == CursorKind.UNION_DECL) + + def is_empty(self, type: Type) -> bool: + """An empty base takes up no room in the struct inheriting it.""" + if next(iter(type.get_fields()), None) is not None: + return False + return all( + self.is_empty(child.type.get_canonical()) + for child in type.get_declaration().get_children() + if child.kind == CursorKind.CXX_BASE_SPECIFIER + ) + + def get_field(self, field: Cursor, offset: int, follow: bool): + name = field.spelling or f"anon_{offset:X}" + type = field.type.get_canonical() + count = None + while type.kind == TypeKind.CONSTANTARRAY: + # ghidra gets the flattened array, ex. int[2][3] as int[6] + count = (count or 1) * type.element_count + type = type.element_type.get_canonical() + if count == 0 or type.kind == TypeKind.INCOMPLETEARRAY: + return None + + if count is None and type.kind == TypeKind.POINTER and type.get_pointee().get_canonical().kind == TypeKind.FUNCTIONPROTO: + function = type.get_pointee().get_canonical() + names = [child.spelling for child in field.get_children() if child.kind == CursorKind.PARM_DECL] + parameters = [ + DefinedStructFuncParam( + names[index] if index < len(names) and names[index] else f"a{index + 1}", + self.get_type_string(argument, follow, False) or "__int64", + ) + for index, argument in enumerate(function.argument_types()) + ] + return_type = self.get_type_string(function.get_result(), follow, False) or "__int64" + return DefinedStructFuncField(name, "__fastcall", offset, False, [], return_type, parameters) + + type_string = self.get_type_string(type, follow, True) + if type_string is None: + # ex. a member pointer, keep its room in the struct + type_string, count = "byte", field.type.get_size() + if count <= 0: + return None + + if count is not None: + return DefinedStructFixedField(name, type_string, offset, False, [], count, False) + return DefinedStructField(name, type_string, offset, False, []) + + +class ClangInterface(object): + def __init__(self, args: list[str], library_file: str | None = None): + """ + :param args: Arguments for clang, the same ones the compiler would get + :param library_file: Path of the libclang to use, the one of the python package if omitted + """ + if library_file is not None and not Config.loaded: + Config.set_library_file(library_file) + self.args = args + + def parse_string(self, decl: str) -> tuple[DefinedStructExport, dict[str, str], int]: + """ + Let clang work out the layout of every struct, template instances included. + :return: The structs and enums, the typedefs for templated types, the number of parser errors + """ + index = Index.create() + forced: list[str] = [] + for _ in range(max_instantiation_passes): + # sizeof needs the complete type, that makes clang instantiate the templates only used behind a pointer + source = decl + "".join( + f"\nstatic const auto __cs_instantiate_{number} = sizeof(::{name});" + for number, name in enumerate(forced) + ) + unit = index.parse(main_file, args=self.args, unsaved_files=[(main_file, source)]) + collector = RecordCollector() + collector.collect(unit.cursor) + missing = sorted(collector.incomplete.difference(forced)) + if not missing: + break + forced.extend(missing) + + errors = [diagnostic for diagnostic in unit.diagnostics if diagnostic.severity >= 3] + for diagnostic in errors[:50]: + location = diagnostic.location + print(f"Error: clang: {location.file}:{location.line}:{location.column}: {diagnostic.spelling}") + if len(errors) > 50: + print(f"Error: clang: {len(errors) - 50} more error(s) not shown") + + return collector.get_export(), collector.aliases, len(errors) diff --git a/scripts/ghidra/full_run.py b/scripts/ghidra/full_run.py new file mode 100644 index 0000000000..16a97189bf --- /dev/null +++ b/scripts/ghidra/full_run.py @@ -0,0 +1,19 @@ +# Runs rename.py and struct_import.py in sequence +# @category __UserScripts +# @menupath Tools.Scripts.ffxiv_full_run +# @runtime PyGhidra + +from os import path + +try: + working_directory = path.dirname(path.realpath(__file__)) +except NameError: + working_directory = path.dirname(str(sourceFile)) # noqa + +print("Loading rename.py") +exec(open(path.join(working_directory, "rename.py")).read()) + +print("Loading struct_import.py") +exec(open(path.join(working_directory, "struct_import.py")).read()) + +analyzeChanges(currentProgram) # noqa diff --git a/scripts/ghidra/ghidra_wrapper.py b/scripts/ghidra/ghidra_wrapper.py new file mode 100644 index 0000000000..6fa8e0c074 --- /dev/null +++ b/scripts/ghidra/ghidra_wrapper.py @@ -0,0 +1,755 @@ +from ghidra.program.flatapi import FlatProgramAPI +from ghidra.program.model.data import ( + ArrayDataType, + BooleanDataType, + ByteDataType, + CategoryPath, + CharDataType, + DataType, + DataUtilities, + DoubleDataType, + EnumDataType, + FloatDataType, + FunctionDefinitionDataType, + IntegerDataType, + LongLongDataType, + ParameterDefinition, + ParameterDefinitionImpl, + ShortDataType, + SignedByteDataType, + StructureDataType, + UnionDataType, + UnsignedIntegerDataType, + UnsignedLongLongDataType, + UnsignedShortDataType, + VoidDataType, + WideCharDataType, +) +from ghidra.program.model.listing import Function, ParameterImpl, ReturnParameterImpl +from ghidra.program.model.symbol import SourceType +from java.util import ArrayList +from jpype import JArray, JByte, JLong + +from structs_schema import * +from data_schema import get_yaml, DefinedData, DefinedDataClassVtable +from clang_wrapper import ClangInterface, split_type_path + +from platform import system +from pathlib import Path +import os +import re + +DEFAULT_NAME_PREFIXES = ("FUN_", "LAB_", "SUB_", "LOC_", "DAT_") +# the fields SrcInterface.get_struct_pad fills the unknown parts of a struct with +PAD_FIELD_NAME = re.compile(r"field_[0-9A-F]+") + +def first_subfolder(base: Path) -> Path | None: + for p in sorted(base.iterdir()): + if p.is_dir(): + return p + return None + + +class GhidraInterface(object): + BADADDR = 0xFFFFFFFFFFFFFFFF + + data_yaml: DefinedData + parser_args: list[str] + # typedef name to the type behind it, see get_template_alias + template_aliases: dict[str, str] = {} + + def __init__(self, program, monitor): + """ + :param program: currentProgram of the running script + :param monitor: monitor of the running script + """ + # the script globals (currentProgram, getFunctionAt, ...) only exist in the script itself + self.program = program + self.monitor = monitor + self.flat = FlatProgramAPI(program, monitor) + self.dtm = program.getDataTypeManager() + self.listing = program.getListing() + self.memory = program.getMemory() + self.symbols = program.getSymbolTable() + void_ptr = self.dtm.getPointer(VoidDataType.dataType) + self.primitives = { + "void": VoidDataType.dataType, + "bool": BooleanDataType.dataType, + "char": CharDataType.dataType, + "wchar_t": WideCharDataType.dataType, + "byte": ByteDataType.dataType, + "float": FloatDataType.dataType, + "double": DoubleDataType.dataType, + "int": IntegerDataType.dataType, + "unsigned int": UnsignedIntegerDataType.dataType, + "__int8": SignedByteDataType.dataType, + "__int16": ShortDataType.dataType, + "__int32": IntegerDataType.dataType, + "__int64": LongLongDataType.dataType, + "unsigned __int8": ByteDataType.dataType, + "unsigned __int16": UnsignedShortDataType.dataType, + "unsigned __int32": UnsignedIntegerDataType.dataType, + "unsigned __int64": UnsignedLongLongDataType.dataType, + "int8_t": SignedByteDataType.dataType, + "int16_t": ShortDataType.dataType, + "int32_t": IntegerDataType.dataType, + "int64_t": LongLongDataType.dataType, + "uint8_t": ByteDataType.dataType, + "uint16_t": UnsignedShortDataType.dataType, + "uint32_t": UnsignedIntegerDataType.dataType, + "uint64_t": UnsignedLongLongDataType.dataType, + # function pointers are exported without their signature + "__fastcall": void_ptr, + } + + def get_base_path(self): + return os.path.join( + os.path.dirname(os.path.realpath(__file__)), ".." + ) + + def get_data_path(self): + return os.path.join(self.get_base_path(), "data.yml") + + def load_data_yml(self): + with open(self.get_data_path()) as f: + self.data_yaml = get_yaml(f) + + def get_struct_path(self): + return os.path.join(self.get_base_path(), "ffxiv_structs.yml") + + def to_addr(self, ea: int): + return self.flat.toAddr(JLong(ea)) + + def check_cancelled(self): + if self.monitor.isCancelled(): + raise RuntimeError("Cancelled by user") + + # region types + + def get_category_path(self, type_name: str): + """Category a type lives in, ex. Component::GUI::AtkResNode is AtkResNode in /Component/GUI.""" + category = CategoryPath.ROOT + for part in split_type_path(type_name)[:-1]: + category = CategoryPath(category, part) + return category + + def get_type_name(self, type_name: str) -> str: + return split_type_path(type_name)[-1] + + def get_named_type(self, name: str): + """Returns None if Ghidra does not know the type.""" + if name in self.primitives: + return self.primitives[name] + + if "<" in name and get_template_alias(name) in self.template_aliases: + # clang spells template names differently than the yml + name = self.template_aliases[get_template_alias(name)] + + dt = self.dtm.getDataType(self.get_category_path(name), self.get_type_name(name)) + if dt is None and "::" not in name: + # ex. windows types that came from an applied archive + found = ArrayList() + self.dtm.findDataTypes(name, found) + if found.size() > 0: + dt = found.get(0) + return dt + + def get_tinfo_from_type(self, raw_type: str, array_size=0): + """Returns None if the type could not be built.""" + type = raw_type.strip().rstrip("*").strip() + ptr_count = raw_type.count("*", len(type)) + + dt = self.get_named_type(type) + if dt is None: + if ptr_count == 0: + return None + # a pointer to something unknown is still a pointer + dt = VoidDataType.dataType + + for _ in range(ptr_count): + dt = self.dtm.getPointer(dt) + + if array_size > 0: + dt = ArrayDataType(dt, array_size, dt.getLength()) + + return dt + + def get_enum_id(self, name: str): + return self.get_named_type(name) + + def get_struct_id(self, name: str): + return self.get_named_type(name) + + def create_datatype(self, datatype): + """Add the type to the program, an existing one with the same path gets its content replaced.""" + old = self.dtm.getDataType(datatype.getDataTypePath()) + if old is not None: + old.replaceWith(datatype) + return old + return self.dtm.addDataType(datatype, None) + + def create_enum(self, enum: DefinedStructEnum): + underlying = self.get_named_type(enum.underlying) + size = underlying.getLength() if underlying is not None else 4 + dt = EnumDataType(self.get_category_path(enum.type), self.get_type_name(enum.type), size) + for name, value in enum.values.items(): + if value >= 1 << 63: + # java longs are signed + value -= 1 << 64 + try: + dt.add(name, value) + except Exception as e: + print(f"Error: enum {enum.type} can not hold {name} = {value}: {e}") + self.create_datatype(dt) + + def create_struct(self, struct: DefinedStruct): + """Create the struct without its members so every struct can be pointed to.""" + category, name = self.get_category_path(struct.type), self.get_type_name(struct.type) + if struct.union: + dt = UnionDataType(category, name) + else: + dt = StructureDataType(category, name, struct.size or 0) + self.create_datatype(dt) + + def create_struct_members(self, struct: DefinedStruct): + dt = self.get_named_type(struct.type) + if dt is None: + return + + for field in sorted(struct.fields, key=lambda fld: fld.offset): + if PAD_FIELD_NAME.fullmatch(field.name): + # undefined bytes are of more use in ghidra than a filler field + continue + + if isinstance(field, DefinedStructFuncField): + ft = self.dtm.getPointer(self.create_function_def(field, struct)) + else: + ft = self.get_tinfo_from_type(field.type) + if ft is None: + print(f"Error: field {field.name} of {struct.type} has unknown type {field.type!r}") + continue + + if isinstance(field, DefinedStructFixedField): + if int(field.size) < 1: + continue + ft = ArrayDataType(ft, int(field.size), ft.getLength() or -1) + + if struct.union: + dt.add(ft, ft.getLength(), field.name, "") + continue + + size = 0 if dt.isZeroLength() else dt.getLength() + end = field.offset + ft.getLength() + if size < end: + if struct.size: + print( + f"Error: field {field.name} (off=0x{field.offset:X} size=0x{ft.getLength():X}) not within struct {struct.type} (size=0x{size:X})" + ) + break + dt.growStructure(end - size) + + dt.replaceAtOffset(field.offset, ft, -1, field.name, "") + + def create_function_def(self, func: DefinedStructFuncField, struct: DefinedStruct): + """Signature of a function pointer field, ex. an entry of a vtable.""" + fd = FunctionDefinitionDataType( + CategoryPath(self.get_category_path(struct.type), self.get_type_name(struct.type)), func.name + ) + fd.setReturnType(self.get_tinfo_from_type(func.return_type) or DataType.DEFAULT) + args = [ + ParameterDefinitionImpl(arg.name, self.get_tinfo_from_type(arg.type) or DataType.DEFAULT, None) + for arg in func.parameters + ] + fd.setArguments(JArray(ParameterDefinition)(args)) + return fd + + def select_parser(self, msvc_path: str | None = None): + """ + :param msvc_path: Folder where ./vsdownload.sh of mstorsjo/msvc-wine was run, clang finds the headers itself on windows + """ + includes = [] + if system() != "Windows": + if msvc_path is None: + raise RuntimeError("Has to have msvc includes for the parsers to work with the project") + path = Path(msvc_path) + vc_path = first_subfolder(path / "vc" / "tools" / "msvc") / "include" + window_kit = first_subfolder(path / "kits" / "10" / "include") + includes = [f"-I{folder.as_posix()}" for folder in (vc_path, window_kit / "ucrt", window_kit / "um", window_kit / "shared")] + + self.parser_args = [ + "-x", "c++", "-target", "x86_64-pc-windows-msvc", "-fms-compatibility", "-fms-extensions", + "-U__int64", "-std=c++17", "-ferror-limit=0", *includes, + ] + + def parse_string(self, decl: str, vtables: list[str]) -> int: + """ + Ghidra can not parse C++, so clang lays out the declarations and the result is imported as data types. + :return: The number of parser errors reported by clang + """ + export, self.template_aliases, error_count = ClangInterface( + self.parser_args, os.environ.get("LIBCLANG_LIBRARY_FILE") + ).parse_string(decl) + if error_count > 0: + print(f"Error: clang parser reported {error_count} error(s) while importing declarations") + else: + print("Clang parser imported declarations without errors") + + self.import_structs(export) + + missing = [vtable for vtable in vtables if self.get_named_type(vtable) is None] + for vtable in missing: + print(f"Error: vtable '{vtable}' not found in the data types") + print(f"Imported {len(vtables) - len(missing)}/{len(vtables)} vtables ({len(missing)} failed)") + + return error_count + + def import_structs(self, export: DefinedStructExport) -> int: + """ + :param export: Structs in dependency order with their size and field offsets known + :return: The number of types that failed to import + """ + steps = [ + ("Creating enums", export.enums, self.create_enum), + ("Creating structs", export.structs, self.create_struct), + ("Creating members for structs", export.structs, self.create_struct_members), + ] + error_count = 0 + for message, items, step in steps: + print(message) + self.monitor.setMessage(message) + for item in items: + self.check_cancelled() + try: + step(item) + except Exception as e: + error_count += 1 + print(f"Error: {message.lower()} failed on {item.type}: {e}") + + if error_count > 0: + print(f"Error: {error_count} error(s) while importing declarations") + else: + print("Imported declarations without errors") + return error_count + + # endregion + + # region functions + + def get_func(self, ea: int, create: bool = False): + """Function starting at ea.""" + addr = self.to_addr(ea) + func = self.flat.getFunctionAt(addr) + if func is None and create: + if self.flat.getInstructionAt(addr) is None: + self.flat.disassemble(addr) + func = self.flat.createFunction(addr, None) + return func + + def has_default_name(self, ea: int) -> bool: + """Only true for functions Ghidra named itself.""" + func = self.get_func(ea, True) + return func is not None and func.getSymbol().getSource() == SourceType.DEFAULT + + def apply_func_type(self, ea: int, func_name: str, return_type: str, parameters: list[DefinedStructFuncParam]) -> bool: + """Returns False if one of the types is not known to Ghidra.""" + func = self.get_func(ea, True) + if func is None: + print(f"Error: {func_name} @ {ea:#x} is not a function") + return False + + rettype = self.get_tinfo_from_type(return_type) + if rettype is None: + print(f"Error: {func_name} @ {ea:#x} has unknown return type {return_type!r}") + return False + + args = ArrayList() + for param in parameters: + arg_type = self.get_tinfo_from_type(param.type) + if arg_type is None: + print(f"Error: {func_name} @ {ea:#x} has unknown type {param.type!r} on parameter {param.name!r}") + return False + args.add(ParameterImpl(param.name, arg_type, self.program)) + + try: + func.updateFunction( + "__fastcall", + ReturnParameterImpl(rettype, self.program), + args, + Function.FunctionUpdateType.DYNAMIC_STORAGE_ALL_PARAMS, + False, + SourceType.USER_DEFINED, + ) + except Exception as e: + print(f"Error: {func_name} @ {ea:#x} failed to apply its function type: {e}") + return False + return True + + def get_vft_from_data(self, class_name: str) -> DefinedDataClassVtable | None: + class_data = self.data_yaml.classes.get(class_name) + if class_data is None or not class_data.vtbls: + return None + + return class_data.vtbls[0] + + def get_vfunc_ea_from_data(self, class_name: str, offset: int) -> int: + vtable = self.get_vft_from_data(class_name) + if vtable is None: + return self.BADADDR + + vtable_ea = vtable.ea + self.get_image_base() - 0x140000000 + try: + return self.get_qword(vtable_ea + offset) + except Exception: + return self.BADADDR + + def set_func_name(self, ea, name, cmt=None, unique=False): + """ + :param unique: Only name functions Ghidra has not named yet, a number is appended if the name is taken by another address + """ + if unique: + if not self.has_default_name(ea): + return + base_name, count = name, 1 + while self.get_func_ea_by_name(name) not in (self.BADADDR, ea): + count += 1 + name = f"{base_name}{count}" + print(name) + + func = self.get_func(ea, True) + if func is None: + self.set_addr_name(ea, name) + return + func.setName(name, SourceType.USER_DEFINED) + if cmt is not None: + func.setComment(cmt) + + def get_func_name(self, ea: int) -> str: + """Name of the function containing ea.""" + func = self.flat.getFunctionContaining(self.to_addr(ea)) + return func.getName() if func is not None else "" + + def get_func_start(self, ea: int) -> int: + """Start of the function containing ea.""" + return self.flat.getFunctionContaining(self.to_addr(ea)).getEntryPoint().getOffset() + + def get_func_ea_by_name(self, name: str) -> int: + for symbol in self.symbols.getGlobalSymbols(name): + return symbol.getAddress().getOffset() + return self.BADADDR + + def get_func_xrefs(self, ea: int) -> list[int]: + """The addresses calling ea.""" + return [ + xref.getFromAddress().getOffset() + for xref in self.flat.getReferencesTo(self.to_addr(ea)) + if xref.getReferenceType().isCall() + ] + + def get_func_ea_by_sig(self, pattern: str) -> int: + ea = self.search_binary(0, pattern) + if ea == self.BADADDR: + return ea + + if self.get_func(ea) is not None: + return ea + + insn = self.flat.getInstructionAt(self.to_addr(ea)) + if insn is None: + return self.BADADDR + + flow = insn.getFlowType() + if flow.isCall() or flow.isJump(): + flows = insn.getFlows() + if len(flows) != 1: + print(f"Error: Can't follow {insn} @ {ea:#x}") + return self.BADADDR + return flows[0].getOffset() + + return self.BADADDR + + def update_member_func(self, member_func: DefinedStructMemFunc, struct: DefinedStruct) -> None: + func_name = f"{struct.type}.{member_func.name}" + ea = self.get_func_ea_by_name(func_name) + if ea == self.BADADDR: + ea = self.get_func_ea_by_sig(member_func.signature) + if ea == self.BADADDR: + print( + f"Error: {func_name} not found bad sig? {member_func.signature}" + ) + return + if self.has_default_name(ea): + self.set_func_name(ea, func_name) + self.apply_func_type(ea, func_name, member_func.return_type, member_func.parameters) + + def update_virt_func(self, virt_func: DefinedStructVFunc, struct: DefinedStruct): + func_name = f"{struct.type}.{virt_func.name}" + + ea = self.get_func_ea_by_name(func_name) + if ea == self.BADADDR: + if self.get_vft_from_data(struct.type) is None: + # no vtable address in data.yml, so there is no slot to look up + return + ea = self.get_vfunc_ea_from_data(struct.type, virt_func.offset) + if ea in (0, self.BADADDR): + print(f"Error: {func_name} not found and its VFT slot could not be resolved") + return + + normalized_name = self.get_addr_name(ea).removeprefix("thunk_") + + if normalized_name == "_purecall": + return + + primary_base = self.get_vft_from_data(struct.type).base + if primary_base: + base_ea = self.get_vfunc_ea_from_data(primary_base, virt_func.offset) + if base_ea == ea: + return + + if ( + normalized_name.endswith("." + virt_func.name) + and not normalized_name.startswith(struct.type + ".") + ): + return + self.apply_func_type(ea, func_name, virt_func.return_type, virt_func.parameters) + + def get_static_member_ea(self, static_member: DefinedStructStaticMember) -> int: + """Address the static member's signature resolves to, BADADDR if the signature is not found.""" + ea = self.search_binary(0, static_member.signature) + if ea == self.BADADDR: + return ea + for follows in static_member.relative_offsets: + ea = ea + follows + ea = ea + 4 + self.memory.getInt(self.to_addr(ea)) + return ea + + def update_static_member(self, static_member: DefinedStructStaticMember, struct: DefinedStruct): + ea = self.get_static_member_ea(static_member) + if ea == self.BADADDR: + print(f"Error: {static_member.signature} not found something is wrong") + return + addr = self.to_addr(ea) + data = self.listing.getDefinedDataContaining(addr) + if data is not None and data.getAddress().getOffset() != ea and static_member.return_type != struct.type: + # points into an already typed item (ex. a field of the struct instance) + return + return_type = static_member.return_type + if static_member.is_pointer: + return_type = return_type + "*" + dt = self.get_tinfo_from_type(return_type) + if dt is None: + print(f"Error: static member of {struct.type} @ {ea:#x} has unknown type {return_type!r}") + else: + try: + DataUtilities.createData( + self.program, addr, dt, -1, DataUtilities.ClearDataMode.CLEAR_ALL_CONFLICT_DATA + ) + except Exception as e: + print(f"Error: static member of {struct.type} @ {ea:#x} failed to apply {return_type!r}: {e}") + self.set_addr_name(ea, f"g_{struct.type}_{static_member.name}") + + # endregion + + # region addresses + + def get_image_base(self) -> int: + return self.program.getImageBase().getOffset() + + def is_offset(self, ea: int) -> bool: + data = self.flat.getDataAt(self.to_addr(ea)) + return bool(data) and data.isPointer() + + def xrefs_to(self, ea: int) -> list[int]: + return [xref.getFromAddress().getOffset() for xref in self.flat.getReferencesTo(self.to_addr(ea))] + + def get_qword(self, ea: int) -> int: + return self.memory.getLong(self.to_addr(ea)) & 0xFFFFFFFFFFFFFFFF + + def get_dword(self, ea: int) -> int: + return self.memory.getInt(self.to_addr(ea)) & 0xFFFFFFFF + + def get_addr_name(self, ea: int) -> str: + addr = self.to_addr(ea) + sym = self.flat.getSymbolAt(addr) + if not sym: + # code only reachable through a vtable is not always disassembled, it does follow the padding of the previous function + try: + is_padded = self.memory.getByte(addr.subtract(1)) & 0xFF == 0xCC + except Exception: + return "" + if is_padded: + self.get_func(ea, True) + sym = self.flat.getSymbolAt(addr) + if not sym: + return "" + return sym.getName(True) + + def set_addr_name(self, ea: int, name: str) -> bool: + self.flat.createLabel(self.to_addr(ea), name, True, SourceType.ANALYSIS) + return True + + def get_comment(self, ea: int): + return self.flat.getEOLComment(self.to_addr(ea)) + + def set_comment(self, ea: int, comment: str): + # the callers append to what is already there, keeping the first one makes a rerun not stack them + if self.flat.getEOLComment(self.to_addr(ea)) is None: + self.flat.setEOLComment(self.to_addr(ea), comment) + + def get_string_from_ptr(self, ea: int) -> str: + val = self.get_string(ea) + if val is None: + val = self.get_string(self.get_qword(ea)) + return val + + def get_string(self, ea: int) -> str | None: + """Returns None if there is no printable string at ea.""" + chars = bytearray() + try: + while len(chars) < 256: + char = self.memory.getByte(self.to_addr(ea + len(chars))) & 0xFF + if char == 0: + break + if not 0x20 <= char < 0x7F: + return None + chars.append(char) + except Exception: + return None + return chars.decode("ascii") if chars else None + + # endregion + + # region instructions + + def heads_before(self, ea: int, count: int): + """The instructions in front of ea, closest first, ex. the argument setup of a call.""" + insn = self.listing.getInstructionContaining(self.to_addr(ea)) + for _ in range(count): + insn = insn.getPrevious() + if insn is None: + return + yield insn.getAddress().getOffset() + + def heads_until_call(self, ea: int, func_name: str): + """The instructions from ea up to the call of the named function.""" + target = self.get_func_ea_by_name(func_name) + insn = self.listing.getInstructionAt(self.to_addr(ea)) + while insn is not None: + if insn.getFlowType().isCall() and any(flow.getOffset() == target for flow in insn.getFlows()): + return + yield insn.getAddress().getOffset() + insn = insn.getNext() + + def get_registers(self, eas, mnemonics=None) -> dict[str, dict[str, object]]: + """ + Collect what the instructions write to each register, keyed by lower case register name. + :param eas: Instruction addresses to look at, later ones win + :param mnemonics: Only look at these instructions, all of them if omitted + """ + regs = {} + for ea in eas: + insn = self.listing.getInstructionAt(self.to_addr(ea)) + if insn is None or insn.getNumOperands() < 2: + continue + if mnemonics is not None and str(insn.getMnemonicString()).lower() not in mnemonics: + continue + val_text = str(insn.getDefaultOperandRepresentation(1)).lower() + val = 0 + refs = insn.getOperandReferences(1) + if len(refs) > 0: + val = refs[0].getToAddress().getOffset() + if refs[0].getReferenceType().isRead(): + # the register is loaded from memory, ex. a pointer to the class name + val_text = self.get_string_from_ptr(val) + elif insn.getScalar(1) is not None: + val = insn.getScalar(1).getUnsignedValue() + regs[str(insn.getDefaultOperandRepresentation(0)).lower()] = {'val_text': val_text, 'val': val} + return regs + + # endregion + + # region naming + + def format_class_name_for_vtbl(self, class_name: str) -> str: + return f"vtbl_{class_name}" + + def format_class_name_for_secondary_vtbl(self, class_name: str, secondary_name: str) -> str: + return f"vtbl_{class_name}___{secondary_name}" + + def format_class_name(self, class_name: str) -> str: + return class_name + + def format_vfunc_name(self, ea: int, current_func_name: str, proposed_func_name: str, class_name: str, base_class_names: list[str]) -> str | None: + current_func_name = current_func_name.removeprefix("thunk_") # jump + + # Previously renamed as a vfunc + if current_func_name.startswith(class_name): + # return the proposed func name in case it was updated since last run + current_class_name = current_func_name.rsplit(".", 1)[0] + return f"{current_class_name}.{proposed_func_name}" + + # This should have been handled in the base class + if any(current_func_name.startswith(name) for name in base_class_names): + return "" + + if current_func_name.startswith(DEFAULT_NAME_PREFIXES): + return f"{class_name}.{proposed_func_name}" + + # Name it later in a child class when it gets overridden + if current_func_name == "_purecall": + return "" + + return None + + def format_func_name(self, ea: int, current_func_name: str, proposed_func_name: str, class_name: str) -> str | None: + # override default thunk names and thunks of already named funcs with the default + func = self.get_func(ea) + if func and func.isThunk(): + current_func_name = f"FUN_{ea:x}" + + proposed_qualified_func_name = f"{class_name}.{proposed_func_name}" + if current_func_name == proposed_qualified_func_name: + return "" + + if current_func_name.startswith(DEFAULT_NAME_PREFIXES): + return proposed_qualified_func_name + + return None + + # endregion + + # region search + + def get_search_range(self): + """The signatures are all code, so only .text is looked at when the program has one.""" + block = self.memory.getBlock(".text") + if block is not None: + return block.getStart(), block.getEnd() + return self.memory.getMinAddress(), self.memory.getMaxAddress() + + def search_binary(self, ea: int, pattern: str) -> int: + """ + :param pattern: Space separated hex bytes, ? or ?? for a wildcard + """ + start, end = self.get_search_range() + if ea > end.getOffset(): + return self.BADADDR + if ea > start.getOffset(): + start = self.to_addr(ea) + + tokens = pattern.split() + values = bytes(0 if "?" in token else int(token, 16) for token in tokens) + masks = bytes(0 if "?" in token else 0xFF for token in tokens) + found = self.memory.findBytes( + start, end, JArray(JByte)(values), JArray(JByte)(masks), True, self.monitor + ) + return found.getOffset() if found is not None else self.BADADDR + + def search_binary_all(self, pattern: str, ea: int = 0): + """Every address matching the pattern, starting from ea.""" + ea = self.search_binary(ea, pattern) + while ea != self.BADADDR: + self.check_cancelled() + yield ea + ea = self.search_binary(ea + 1, pattern) + + # endregion diff --git a/scripts/ghidra/rename.py b/scripts/ghidra/rename.py new file mode 100644 index 0000000000..976365132a --- /dev/null +++ b/scripts/ghidra/rename.py @@ -0,0 +1,69 @@ +# Names the globals, functions, vtables and such of data.yml +# @category __UserScripts +# @menupath Tools.Scripts.ffxiv_rename +# @runtime PyGhidra + +from os.path import join, dirname +import sys + +try: + script_dir = dirname(__file__) +except NameError: + script_dir = dirname(str(sourceFile)) # noqa + +for import_path in (script_dir, join(script_dir, "..", "shared")): + if import_path not in sys.path: + sys.path.append(import_path) + +from ghidra_wrapper import GhidraInterface +from class_factory import apply_data + +lua_namespace = "Client::Game::Event::Lua" + +def apply_lua_funcs(): + """Relies on the Common::Lua::LuaState names from data.yml, so apply_data has to run first.""" + string_eas = ghidra.get_func_xrefs(ghidra.get_func_ea_by_name('Common::Lua::LuaState.SetStringField')) + func_eas = ghidra.get_func_xrefs(ghidra.get_func_ea_by_name('Common::Lua::LuaState.SetFunctionField')) + + string_eas = [ghidra.get_func_start(x) for x in string_eas] + + eas = {} + for x in func_eas: + regs = ghidra.get_registers(ghidra.heads_before(x, 4)) + regs['r8'] = ghidra.get_string_from_ptr(regs['r8']['val']) + eas.setdefault(ghidra.get_func_start(x), {'calls': []})['calls'].append({'off': x, 'regs': regs}) + + eas = {k: v for k,v in eas.items() if k in string_eas} + + for func_ea in eas: + eas[func_ea]['regs'] = ghidra.get_registers(ghidra.heads_until_call(func_ea, 'Common::Lua::LuaState.SetStringField'), ('mov',)) + + for func_ea in eas: + eas[func_ea]['name'] = eas[func_ea]['regs'][eas[func_ea]['regs']['r9']['val_text']]['val_text'] + eas[func_ea]['name_val'] = eas[func_ea]['regs'][eas[func_ea]['regs']['r9']['val_text']]['val'] + + for func_ea in eas: + if eas[func_ea]['name'] == 'r8': + eas[func_ea]['name'] = ghidra.get_func_name(func_ea).split('.')[0].split(':')[-1] + eas[func_ea]['name_val'] = func_ea + if eas[func_ea]['name'] == 'rax': + eas[func_ea]['name_val'] = ghidra.get_func_xrefs(func_ea)[0] + eas[func_ea]['name'] = ghidra.get_func_name(eas[func_ea]['name_val']).split('.')[0].split(':')[-1] + + for func_ea in eas: + name = f"{lua_namespace}::{eas[func_ea]['name']}" + ghidra.set_func_name(func_ea, f"{name}.SetupLuaClass") + for call in eas[func_ea]['calls']: + regs = call['regs'] + func_name = regs['r8'] + ghidra.set_func_name(regs['r9']['val'], f"{name}.{func_name}", unique=True) + +ghidra = GhidraInterface(currentProgram, monitor) # noqa +print("Loading data.yml") +ghidra.load_data_yml() + +print("Executing") +apply_data(ghidra, ghidra.data_yaml) +print("Naming lua functions") +apply_lua_funcs() +print("Done") diff --git a/scripts/ghidra/struct_import.py b/scripts/ghidra/struct_import.py new file mode 100644 index 0000000000..ee34aca869 --- /dev/null +++ b/scripts/ghidra/struct_import.py @@ -0,0 +1,205 @@ +# Imports the Excel sheets and ffxiv_structs.yml as data types, and names and types the functions using them +# @category __UserScripts +# @menupath Tools.Scripts.ffxiv_struct_import +# @runtime PyGhidra + +from os.path import join, dirname +import sys + +try: + script_dir = dirname(__file__) +except NameError: + script_dir = dirname(str(sourceFile)) # noqa + +for import_path in (script_dir, join(script_dir, "..", "shared")): + if import_path not in sys.path: + sys.path.append(import_path) + +from java.util import ArrayList +from ghidra.util.exception import CancelledException + +from lumina_wrapper import get_excel_header_files +from exdschema_wrapper import ( + create_struct_from_header_and_schema, + get_exdschema_data, + exd_comment_patterns, + exd_func_patterns, +) +from src_wrapper import SrcInterface +from ghidra_wrapper import GhidraInterface +from structs_schema import get_yaml, DefinedStructFuncParam + +src = SrcInterface() +# one filler field per gap, clang gets slow on a struct with a field for every unknown qword +src.full_padding = False +ghidra = GhidraInterface(currentProgram, monitor) # noqa +print("Loading data.yml") +ghidra.load_data_yml() + +if not askYesNo( # noqa + "Check version match.", + f"Loaded with ver: {ghidra.data_yaml.version}\n" + "Check if this matches with what is reported in the version files of the game.", +): + raise RuntimeError("Version is not correct please update repo") + +msvc_path = None +if sys.platform != "win32": + msvc_path = str(askDirectory( # noqa + "Select msvc, the folder where ./vsdownload.sh of mstorsjo/msvc-wine was run", "Select" + )) +ghidra.select_parser(msvc_path) + +function_types = {"virtual": "Virtual functions", "member": "Member functions"} +try: + _selected = askChoices( # noqa + "Select function types to update", + "Types to update", + ArrayList(list(function_types.keys())), + ArrayList(list(function_types.values())), + ) + _function_update_options = {key: key in _selected for key in function_types} +except CancelledException: + _function_update_options = { + "virtual": False, + "member": False, + } + +print("Grabbing ExdSchema and Lumina") + +excel_header_files = get_excel_header_files() +exdschema = get_exdschema_data("latest") + +excel_structs, excel_map = create_struct_from_header_and_schema( + excel_header_files, exdschema +) + +exd_map = {excel_map[k]: k for k in excel_map} + +excel_header, excel_vtables = src.build_export_string(excel_structs, 4) + + +print("Parsing ffxiv_structs.yml") + +with open(ghidra.get_struct_path()) as f: + import_structs = get_yaml(f) + +struct_header, struct_vtables = src.build_export_string(import_structs, 1) + +print("Parsing combined Excel and FFXIV declarations") +ghidra.parse_string( + f"""{src.get_predef_header()} +{excel_header} +{struct_header} +""", + [*excel_vtables, *struct_vtables], +) + +def do_pattern(pattern, suffix, sheetSearchPattern, sheetSearchOffset): + for ea in ghidra.search_binary_all(pattern, 1): + if sheetSearchPattern == None: + sheetEa = ea + sheetSearchOffset + else: + sheetEa = ( + ghidra.search_binary(ea, sheetSearchPattern) + + sheetSearchOffset + ) + sheetIdx = ghidra.get_dword(sheetEa) + + # don't rename any funcs that are already named + if ghidra.has_default_name(ea): + if exd_map.get(sheetIdx) == None: + print( + f"Func @ 0x{ea:X} references unknown sheet {sheetIdx}! sheet dword offset = 0x{sheetEa:X}" + ) + continue + + sheetName = exd_map[sheetIdx] + + if suffix == None: + suffix = "" + + fnName = "Component::Exd::ExdModule_Get%s%s" % ( + exd_map[sheetIdx], + suffix, + ) + + uniquifier = 0 + while True: + uniqueName = fnName + (f"_{uniquifier}" if uniquifier > 0 else "") + + if ghidra.get_func_ea_by_name(uniqueName) == ghidra.BADADDR and uniquifier > 0: + fnName = uniqueName + break + + uniquifier += 1 + + ghidra.set_func_name( + ea, fnName, "Sheet: {0} ({1})".format(sheetName, sheetIdx) + ) + + if suffix == "SheetIndex": + continue + + sheetType = f"Component::Exd::Sheets::{exd_map[sheetIdx]}*" + if suffix == "RowAndSubRowId": + parameters = [ + DefinedStructFuncParam("rowId", "unsigned int"), + DefinedStructFuncParam("subRowId", "unsigned int"), + ] + elif suffix == "RowCount": + parameters = [] + else: + parameters = [DefinedStructFuncParam("rowIdOrIndex", "unsigned int")] + + ghidra.apply_func_type(ea, fnName, sheetType, parameters) + + +print("Fixing up Excel functions") + +for pattern in exd_func_patterns: + suffix, search = exd_func_patterns[pattern] + if suffix is not None: + print(f"Finding exd funcs of {suffix}... please wait.") + searchPattern, searchOffset = search + do_pattern(pattern, suffix, searchPattern, searchOffset) + + +for pattern_key in exd_comment_patterns: + for ea in list(ghidra.search_binary_all(pattern_key)): + sheetIdx = ghidra.get_dword(ea + exd_comment_patterns[pattern_key]) + sheetName = exd_map.get(sheetIdx) + if sheetName is None: + print(f"Sheet getter @ 0x{ea:X} references unknown sheet {sheetIdx}!") + continue + ghidra.set_comment(ea, "Sheet: {0} ({1})".format(sheetName, sheetIdx)) + +if _function_update_options["virtual"]: + for struct in import_structs.structs: + if struct.virtual_functions: + print(f"Updating virtual functions for {struct.type}") + for virt_func in struct.virtual_functions: + if virt_func is None: + continue + + if virt_func.return_type is not None and virt_func.parameters is not None: + ghidra.update_virt_func(virt_func, struct) + +print("Fixing up CS defined functions") + +if _function_update_options["member"]: + for struct in import_structs.structs: + if struct.member_functions != []: + print(f"Updating member functions for {struct.type}") + for member_func in struct.member_functions: + ghidra.update_member_func(member_func, struct) + + if struct.static_member_functions: + print(f"Updating static member functions for {struct.type}") + for member_func in struct.static_member_functions: + ghidra.update_member_func(member_func, struct) + + if struct.static_members: + print(f"Updating static members for {struct.type}") + for member in struct.static_members: + ghidra.update_static_member(member, struct) From 6e5070596b48bb1d1129cde6d48d4eb68952bddb Mon Sep 17 00:00:00 2001 From: wildwolf <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:20:14 +0200 Subject: [PATCH 67/95] move some scripts im unsure about to an old folder for later revision --- scripts/{ => old_scripts}/ffxiv_addonmagic.py | 0 scripts/{ => old_scripts}/ffxiv_sigmaker.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename scripts/{ => old_scripts}/ffxiv_addonmagic.py (100%) rename scripts/{ => old_scripts}/ffxiv_sigmaker.py (100%) diff --git a/scripts/ffxiv_addonmagic.py b/scripts/old_scripts/ffxiv_addonmagic.py similarity index 100% rename from scripts/ffxiv_addonmagic.py rename to scripts/old_scripts/ffxiv_addonmagic.py diff --git a/scripts/ffxiv_sigmaker.py b/scripts/old_scripts/ffxiv_sigmaker.py similarity index 100% rename from scripts/ffxiv_sigmaker.py rename to scripts/old_scripts/ffxiv_sigmaker.py From fdc39b92f3b3d545cf84b5fcfea6dfeeba24e777 Mon Sep 17 00:00:00 2001 From: wildwolf <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:32:23 +0200 Subject: [PATCH 68/95] add some readme updates --- Ghidra/Getting Started.md | 32 +++++++++++++++++++++----------- README.md | 4 ++-- scripts/README.md | 32 +++++++++++++++++++++++--------- 3 files changed, 46 insertions(+), 22 deletions(-) diff --git a/Ghidra/Getting Started.md b/Ghidra/Getting Started.md index e4414b3323..7e12e7cebd 100644 --- a/Ghidra/Getting Started.md +++ b/Ghidra/Getting Started.md @@ -40,18 +40,23 @@ Note that after each patch, you'll have to re-import, re-analyze, and re-run any

# Script dependency installation: -We use [..\ida\ffxiv_idarename.py](../ida/ffxiv_idarename.py) to apply data.yml. The same script works for both IDA and Ghidra, howver it's slightly more complicated in Ghidra as we need to add dependencies to the PyGhidra environment manually. +We use [rename.py](../scripts/ghidra/rename.py) to apply data.yml and [struct_import.py](../scripts/ghidra/struct_import.py) to import the structs. They follow the same flow as their IDA counterparts in `scripts/ida`, however it's slightly more complicated in Ghidra as we need to add dependencies to the PyGhidra environment manually. - Install a copy of [Python 3](https://www.python.org/downloads/).
-Note: it has to be major version 3, so something like [Python 3.14.3](https://www.python.org/downloads/release/python-3143/) works.
+Note: any Python version that is still listed on the [Python versions status page](https://devguide.python.org/versions/) is supported, so something like [Python 3.14.3](https://www.python.org/downloads/release/python-3143/) works.
+Ghidra only ships JPype wheels up to Python 3.13, on newer versions PyGhidra builds JPype from source and needs a C++ compiler for that (the MSVC Build Tools on Windows).
- **Check your PyGhidra log for which environment is being used.** For example, if you've using a "virtual environment", it should be something similar to either of the following two paths:
Windows: `%AppData%\ghidra\ghidra_VERSION\venv`
Linux: `/home/user/.config/ghidra/ghidra_VERSION/venv` - Execute the following in your terminal, using Python from the same environment as PyGhidra:
-venv: `.../venv/bin/python -m pip install pyyaml==6.0.3 anytree==2.13.0`
-system: `python -m pip install pyyaml==6.0.3 anytree==2.13.0` +venv: `.../venv/bin/python -m pip install pyyaml==6.0.3 anytree==2.13.0 pythonnet libclang`
+system: `python -m pip install pyyaml==6.0.3 anytree==2.13.0 pythonnet libclang`
+`pythonnet` is only used by struct_import.py to read the Excel headers with Lumina, it needs the [.NET 10 runtime](https://dotnet.microsoft.com/download/dotnet/10.0) installed, the same version FFXIVClientStructs uses.
+`libclang` is only used by struct_import.py to work out the layout of the structs. Outside of Windows it also needs the MSVC and Windows SDK headers, download them with [mstorsjo/msvc-wine](https://github.com/mstorsjo/msvc-wine)'s `vsdownload.sh`.
+The `libclang` package ships libclang 18 while the MSVC headers want clang 20 or newer, set the `LIBCLANG_LIBRARY_FILE` environment variable to a libclang 20 or newer (ex. `/usr/lib/llvm-20/lib/libclang-20.so.1`) to avoid the errors this reports.
+struct_import.py looks for the game in the [XIVLauncher](https://goatcorp.github.io/) or XLCore config, the default install folder and the Steam libraries, and asks for the folder holding both `game` and `boot` if it is not found. It also needs internet access to download Lumina and [EXDSchema](https://github.com/xivdev/EXDSchema). - Open the Script Manager

@@ -59,13 +64,13 @@ system: `python -m pip install pyyaml==6.0.3 anytree==2.13.0` - Click the plus to add a directory

-- Add both `FFXIVClientStructs/ida` (for ffxiv_idarename.py) and `FFXIVClientStructs/Ghidra/scripts` (for ffxiv_exdgetters and SigScanner) +- Add both `FFXIVClientStructs/scripts/ghidra` (for rename.py, struct_import.py and full_run.py) and `FFXIVClientStructs/Ghidra/scripts` (for SigScanner) After adding both, select `__UserScripts` and you should see all of the scripts. SigScanner should already have a key binding assigned of `Ctrl-Alt-S`. Assign any others you wish, and ensure the checkbox is ticked.

# The Scripts -## ffxiv_idarename.py +## rename.py First a quick intro. The analyzer reads through the code and attempts to determine things like what blocks of code are instructions vs pointers vs data, etc. When it identifies things, it'll add labels to them: - FUN_14#######: for what it believes to be functions - PTR_FUN_14#######: for what it believes to be pointers to functions @@ -81,7 +86,7 @@ As an example, if you look in Dalamud's Data window You'll see the vtbl has an address like`7FF6DDEAD480 ffxiv_dx11.exe+1A0D480`. In Ghidra, if you were to `Navigation -> Go To...` (or simply press `g`) and typed 141A0D480, you'd arrive at the vtable. -What you'll find in [data.yml](../ida/data.yml) are definitions like: +What you'll find in [data.yml](../scripts/data.yml) are definitions like: ```yaml Client::System::Framework::Framework: instances: @@ -101,7 +106,7 @@ What you'll find in [data.yml](../ida/data.yml) are definitions like: ... ``` -When you run the ffxiv_idarename.py script, it'll rename `FUN_1400901d0` to `Client::System::Framework::Framework.ctor` (per the first line under `funcs:`). It'll turn: +When you run the rename.py script, it'll rename `FUN_1400901d0` to `Client::System::Framework::Framework.ctor` (per the first line under `funcs:`). It'll turn:

Into: @@ -111,8 +116,8 @@ And much more. So go ahead and run it. Either by opening the script manager, r You can re-run this script as many times as you like, it'll just re-process data.yml, so it's much easier to update data.yml when you map out a new vtable and re-run than to try and set all the labels yourself. -## ffxiv_exdgetters.java -After having run ffxiv_idarename.py, we'll have a bunch of functions referencing Exd methods like: +## struct_import.py +After having run rename.py, we'll have a bunch of functions referencing Exd methods like: ```cpp undefined8 FUN_1406cb6c0(undefined4 param_1) { @@ -129,7 +134,12 @@ undefined8 FUN_1406cb6c0(undefined4 param_1) } ``` -The label `Component::Exd::ExdModule.GetRowBySheetIndexAndRowIndex` came from data.yml, but unfortunately it's still not obvious exactly what that method is fetching. The sheet indexes are fairly well-known, and this script parses these and relabels the function from `FUN_1406cb6c0` to `undefined8 Client::ExdData::GetClassJob(undefined4 param_1)`, which is much more useful. Go ahead and run this too. +The label `Component::Exd::ExdModule.GetRowBySheetIndexAndRowIndex` came from data.yml, but unfortunately it's still not obvious exactly what that method is fetching. The sheet indexes are fairly well-known, and this script parses these and relabels the function from `FUN_1406cb6c0` to `Component::Exd::ExdModule_GetClassJobRow_1`, returning a `Component::Exd::Sheets::ClassJob*`, which is much more useful. + +It also imports the Excel sheets and everything in [ffxiv_structs.yml](../scripts/ffxiv_structs.yml) as data types, and can apply the types of the virtual and member functions. It asks you to confirm the game version of data.yml, for the folder of the MSVC headers (outside of Windows) and which function types to update. Go ahead and run this too. + +## full_run.py +Runs rename.py and struct_import.py in sequence, then lets Ghidra analyze the changes. ## SigScanner.java In plugins (and FFXIVClientStructs) we use signatures rather than offsets. This is because offsets change with every patch, while signatures are far more durable. diff --git a/README.md b/README.md index 76e7174c1a..3a545caf6f 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ We make extensive use of [C# Source Generators](https://learn.microsoft.com/en-u ### Reverse Engineering Rename Database -A database and script(s) are maintained in the [ida](https://github.com/aers/FFXIVClientStructs/tree/main/ida) folder which can be used to import a large number of location names to IDA or Ghidra. This database is updated with every patch, although keep in mind this is volunteer work and some patches require more effort than others. There is more info in the readme in the folder itself. +A database and script(s) are maintained in the [scripts](https://github.com/aers/FFXIVClientStructs/tree/main/scripts) folder which can be used to import a large number of location names to IDA or Ghidra. This database is updated with every patch, although keep in mind this is volunteer work and some patches require more effort than others. There is more info in the readme in the folder itself. ### Credits @@ -96,7 +96,7 @@ namespace FFXIVClientStructs.FFXIV.Component.GUI; public unsafe partial struct AtkResNode : ICreatable { } ``` -Native game classes are represented as explicit layout structs. If the official name of the class is available (via [old RTTI](https://github.com/aers/FFXIVClientStructs/blob/main/ida/classinformer.csv)) use that for the name and namespace. For new classes or classes without virtual functions, make up a name that seems appropriate. +Native game classes are represented as explicit layout structs. If the official name of the class is available (via [old RTTI](https://github.com/aers/FFXIVClientStructs/blob/main/scripts/classinformer.csv)) use that for the name and namespace. For new classes or classes without virtual functions, make up a name that seems appropriate. If the struct has unsafe members, mark the struct unsafe rather than the individual members. If you are using a generator, the struct must also be partial. If you are unable to get the exact size, use your best estimate. diff --git a/scripts/README.md b/scripts/README.md index d059416636..06a9d3c7ae 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,7 +1,22 @@ -## ffxiv_idarename.py -Despite the name, this script is now only for Ghidra. The IDA version lives in `ida/rename.py`. -This script ingests the `data.yml` file in the same directory, and renames various offsets as globals, functions, vtables and such. - +## ida/ and ghidra/ +Both folders hold the same scripts for their disassembler, and share the files in `shared/`. + +### rename.py +This script ingests the `data.yml` file in the scripts directory, and renames various offsets as globals, functions, vtables and such. + +### struct_import.py +This script imports the Excel sheets and `ffxiv_structs.yml` as types, names the Excel getter functions and applies the function types. +Both generate the same C++ header and let clang work out the layout of every struct, template instances included. IDA uses its built in clang parser. Ghidra has no C++ parser, so `ghidra/clang_wrapper.py` reads the layouts back from libclang and `ghidra_wrapper.py` creates the data types from that. + +### full_run.py +This script runs `rename.py` and `struct_import.py` in sequence + +#### Shared prerequisites: +Both IDA and Ghidra need these for `struct_import.py`: +- Python, any version still listed on the [Python versions status page](https://devguide.python.org/versions/) is supported. +- The [.NET 10 runtime](https://dotnet.microsoft.com/download/dotnet/10.0), the same version FFXIVClientStructs uses. `pythonnet` loads [Lumina](https://github.com/NotAdam/Lumina) with it to read the Excel headers. +- The game installed. Its folder is read from the best query holding both `game` and `boot`. + #### IDA dependency installation: `pip install pyyaml anytree pythonnet` or `python -m pip install pyyaml anytree pythonnet` in whichever version of python you are currently using with IDA Pro. If you are familiar with Poetry, the `idarename` extras package will do the same. @@ -9,7 +24,10 @@ If you are familiar with Poetry, the `idarename` extras package will do the same Additionally, you will need to place "idauser.cfg" from this repository in "%AppData%\Hex-Rays\IDA Pro\". IDA doesn't allow you to use certain characters in names and this config changes that. #### Ghidra dependency installation: -See [Getting Started](../Ghidra/Getting%20Started.md) for information on how to set up PyGhidra. +See [Getting Started](../Ghidra/Getting%20Started.md) for information on how to set up PyGhidra. The same packages as IDA are needed in the PyGhidra environment, plus `libclang`. +Outside of Windows the MSVC and Windows SDK headers are needed as well, `struct_import.py` asks for the folder [mstorsjo/msvc-wine](https://github.com/mstorsjo/msvc-wine)'s `vsdownload.sh` was run in. +The `libclang` python package ships libclang 18, the MSVC headers want clang 20 or newer and report errors otherwise (the layouts still come out the same). +Set the `LIBCLANG_LIBRARY_FILE` environment variable to a libclang 20 or newer to avoid those, ex. `/usr/lib/llvm-20/lib/libclang-20.so.1`. ## ffxiv_sigmaker.py > [!WARNING] @@ -23,10 +41,6 @@ This support **Python3** and **IDA** only. `pip install dacite pyyaml ruamel-yaml` or `python -m pip install dacite pyyaml ruamel-yaml` in whichever version of python you are currently using with IDA Pro. If you are familiar with Poetry, the `sigmaker` extras package will do the same. -## ida/ -### full_run.py -This script runs `rename.py` and `struct_import.py` in sequence - ## classinformer.csv Once upon a time, someone did a bad thing and released FFXIV with the RTTI data intact. This is an export of that RTTI data using the ClassInformer IDA Pro plugin. From f91b8a2ddf87c044ac0c9eaef74707152b2717a4 Mon Sep 17 00:00:00 2001 From: wildwolf <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:33:25 +0200 Subject: [PATCH 69/95] add more generic solution to find the game path --- scripts/ghidra/struct_import.py | 13 +++- scripts/ida/struct_import.py | 24 ++++++- scripts/shared/lumina_wrapper.py | 113 +++++++++++++++++++++++++------ 3 files changed, 125 insertions(+), 25 deletions(-) diff --git a/scripts/ghidra/struct_import.py b/scripts/ghidra/struct_import.py index ee34aca869..a0f567a597 100644 --- a/scripts/ghidra/struct_import.py +++ b/scripts/ghidra/struct_import.py @@ -18,7 +18,7 @@ from java.util import ArrayList from ghidra.util.exception import CancelledException -from lumina_wrapper import get_excel_header_files +from lumina_wrapper import get_excel_header_files, find_game_path, is_game_path from exdschema_wrapper import ( create_struct_from_header_and_schema, get_exdschema_data, @@ -50,6 +50,15 @@ )) ghidra.select_parser(msvc_path) +game_path = find_game_path() +if game_path is None: + game_path = str(askDirectory( # noqa + "Select the game install, the folder holding both game and boot", "Select" + )) + if not is_game_path(game_path): + raise RuntimeError(f"No game or boot folder in {game_path}, select the folder holding both") +print(f"Using game install at {game_path}") + function_types = {"virtual": "Virtual functions", "member": "Member functions"} try: _selected = askChoices( # noqa @@ -67,7 +76,7 @@ print("Grabbing ExdSchema and Lumina") -excel_header_files = get_excel_header_files() +excel_header_files = get_excel_header_files(game_path) exdschema = get_exdschema_data("latest") excel_structs, excel_map = create_struct_from_header_and_schema( diff --git a/scripts/ida/struct_import.py b/scripts/ida/struct_import.py index 39231a5819..c34152e1fa 100644 --- a/scripts/ida/struct_import.py +++ b/scripts/ida/struct_import.py @@ -3,7 +3,7 @@ sys.path.append(join(dirname(__file__), "..", "shared")) -from lumina_wrapper import get_excel_header_files +from lumina_wrapper import get_excel_header_files, find_game_path, is_game_path from exdschema_wrapper import ( create_struct_from_header_and_schema, get_exdschema_data, @@ -36,6 +36,26 @@ form.Free() ida.select_parser() + +game_path = find_game_path() +if game_path is None: + game_fields = [ + {"type": "label", "text": "The game install was not found through XLCore, XIVLauncher or the default folder."}, + {"type": "space"}, + {"type": "folder", "key": "game_path", "label": "Folder holding both game and boot"}, + ] + form = DynamicForm(game_fields) + form.Compile() + try: + if form.Execute() != 1: + raise RuntimeError("No game install selected") + game_path = form.results()["game_path"] + finally: + form.Free() + if not is_game_path(game_path): + raise RuntimeError(f"No game or boot folder in {game_path}, select the folder holding both") +print(f"Using game install at {game_path}") + fields = [ {"type": "label", "text": "Select function types to update"}, {"type": "space"}, @@ -75,7 +95,7 @@ print("Grabbing ExdSchema and Lumina") -excel_header_files = get_excel_header_files() +excel_header_files = get_excel_header_files(game_path) exdschema = get_exdschema_data("latest") excel_structs, excel_map = create_struct_from_header_and_schema( diff --git a/scripts/shared/lumina_wrapper.py b/scripts/shared/lumina_wrapper.py index a99272488d..2848970ed8 100644 --- a/scripts/shared/lumina_wrapper.py +++ b/scripts/shared/lumina_wrapper.py @@ -1,14 +1,15 @@ from urllib.request import urlopen from zipfile import ZipFile from io import BytesIO -from os.path import join, basename, abspath +from os.path import join, basename, abspath, isdir, isfile from os import getenv, getcwd, makedirs, listdir -from sys import platform, path +from sys import path from shutil import rmtree from xml.dom import pulldom from pythonnet import load as dotnetload from json import load as jsonload from enum import IntEnum +import re from dataclasses import dataclass import gc @@ -16,28 +17,95 @@ rmtree(tempdirname, ignore_errors=True) makedirs(tempdirname, exist_ok=True) -if platform.startswith("linux"): - launcher_ini = join(getenv("HOME"), ".xlcore", "launcher.ini") - game_path = None +default_game_path = r"C:\Program Files (x86)\SquareEnix\FINAL FANTASY XIV - A Realm Reborn" +steam_install_folder = "FINAL FANTASY XIV Online" - f = open(launcher_ini, "r") - try: + +def is_game_path(game_path: str | None) -> bool: + """The game install holds both the game and the boot folder.""" + return bool(game_path) and isdir(join(game_path, "game", "sqpack")) and isdir(join(game_path, "boot")) + + +def _get_xlcore_game_path() -> str | None: + if getenv("HOME") is None: + return None + launcher_ini = join(getenv("HOME"), ".xlcore", "launcher.ini") + if not isfile(launcher_ini): + return None + with open(launcher_ini, "r") as f: for line in f: if line.startswith("GamePath="): - game_path = line.split("=", 1)[1].strip() - break - finally: - f.close() - - if not game_path: - raise ValueError("GamePath key not found in {0}".format(launcher_ini)) -else: - f = open(join(getenv("APPDATA"), "XIVLauncher", "launcherConfigV3.json"), "r") + return line.split("=", 1)[1].strip() + return None + + +def _get_xivlauncher_game_path() -> str | None: + if getenv("APPDATA") is None: + return None + launcher_config = join(getenv("APPDATA"), "XIVLauncher", "launcherConfigV3.json") + if not isfile(launcher_config): + return None + with open(launcher_config, "r") as f: + return jsonload(f).get("GamePath") + + +def _get_steam_paths() -> list[str]: + steam_paths = [] try: - config = jsonload(f) - finally: - f.close() - game_path = config["GamePath"] + import winreg + + with winreg.OpenKey(winreg.HKEY_CURRENT_USER, r"Software\Valve\Steam") as key: + steam_paths.append(winreg.QueryValueEx(key, "SteamPath")[0]) + except (ImportError, OSError): + pass + steam_paths.append(r"C:\Program Files (x86)\Steam") + if getenv("HOME") is not None: + steam_paths.extend( + join(getenv("HOME"), *folder) + for folder in ( + (".local", "share", "Steam"), + (".steam", "steam"), + (".var", "app", "com.valvesoftware.Steam", ".local", "share", "Steam"), + ) + ) + return steam_paths + + +def _get_steam_game_paths() -> list[str]: + """The game folder in every Steam library, libraries can be on other drives than Steam itself.""" + libraries = [] + for steam_path in _get_steam_paths(): + libraries.append(steam_path) + library_folders = join(steam_path, "steamapps", "libraryfolders.vdf") + if isfile(library_folders): + with open(library_folders, "r", encoding="utf-8") as f: + libraries.extend( + library.replace("\\\\", "\\") + for library in re.findall(r'"path"\s+"([^"]+)"', f.read()) + ) + return [join(library, "steamapps", "common", steam_install_folder) for library in dict.fromkeys(libraries)] + + +def find_game_path() -> str | None: + """ + Where XLCore or XIVLauncher have the game installed, else the default install folder or a Steam library. + :return: None if none of them hold the game, the scripts ask for the folder then + """ + for get_paths in ( + lambda: [_get_xlcore_game_path()], + lambda: [_get_xivlauncher_game_path()], + lambda: [default_game_path], + _get_steam_game_paths, + ): + try: + game_paths = get_paths() + except (OSError, ValueError): + # a config that can not be read is the same as not having one + continue + for game_path in game_paths: + if is_game_path(game_path): + return game_path + return None class ExcelColumnDataType(IntEnum): @@ -138,7 +206,10 @@ def get_nugpkg(package: str, version: str = ""): return get_nugpkg_deps(join(tempdirname, f"{package}.nuspec")) -def get_excel_header_files() -> dict[str, tuple[int, list[ExcelColumnDefinition]]]: +def get_excel_header_files(game_path: str) -> dict[str, tuple[int, list[ExcelColumnDefinition]]]: + """ + :param game_path: The game install, the folder holding both game and boot, see find_game_path + """ dotnetload("coreclr") def _isolated_clr(): import clr From a2a30f8ad68fc6986f41fe77eb0ff573bfaacb64 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:33:33 +0200 Subject: [PATCH 70/95] check if CSafeLoader works if not fall back to SafeLoader --- scripts/shared/data_schema.py | 5 +---- scripts/shared/exdschema_wrapper.py | 5 +---- scripts/shared/structs_schema.py | 5 +---- scripts/shared/yaml_loader.py | 11 +++++++++++ 4 files changed, 14 insertions(+), 12 deletions(-) create mode 100644 scripts/shared/yaml_loader.py diff --git a/scripts/shared/data_schema.py b/scripts/shared/data_schema.py index 56fde2c441..c8ccdf30dc 100644 --- a/scripts/shared/data_schema.py +++ b/scripts/shared/data_schema.py @@ -2,10 +2,7 @@ from typing import TypeAlias from yaml import load -try: - from yaml import CSafeLoader as Loader -except ImportError: - from yaml import SafeLoader as Loader +from yaml_loader import Loader @dataclass diff --git a/scripts/shared/exdschema_wrapper.py b/scripts/shared/exdschema_wrapper.py index f93113aa70..869c16437a 100644 --- a/scripts/shared/exdschema_wrapper.py +++ b/scripts/shared/exdschema_wrapper.py @@ -15,10 +15,7 @@ Fields, ) -try: - from yaml import CSafeLoader as Loader -except ImportError: - from yaml import SafeLoader as Loader +from yaml_loader import Loader exd_namespace = "Component::Exd::Sheets" diff --git a/scripts/shared/structs_schema.py b/scripts/shared/structs_schema.py index 43fb47dd05..ddd1ecefac 100644 --- a/scripts/shared/structs_schema.py +++ b/scripts/shared/structs_schema.py @@ -2,10 +2,7 @@ from typing import TypeAlias, Union from yaml import load -try: - from yaml import CSafeLoader as Loader -except ImportError: - from yaml import SafeLoader as Loader +from yaml_loader import Loader YamlStructValuesExport: TypeAlias = list[ dict[str, Union[str, int, "YamlStructValuesExport"]] diff --git a/scripts/shared/yaml_loader.py b/scripts/shared/yaml_loader.py new file mode 100644 index 0000000000..13f42660ee --- /dev/null +++ b/scripts/shared/yaml_loader.py @@ -0,0 +1,11 @@ +from yaml import load, SafeLoader + +try: + from yaml import CSafeLoader + + # When a host re-imports the yaml package in the same interpreter (e.g. a script host unloading modules between runs), + # the cached C extension keeps the old node classes and the resolver fails with "tag None", so check it still works + load("a: 1", CSafeLoader) + Loader = CSafeLoader +except Exception: + Loader = SafeLoader From 788a3d5ed806473c3075b2d05e58e5f494d7fdcd Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:35:19 +0200 Subject: [PATCH 71/95] use strict import from structs_schema in wrappers --- scripts/ghidra/ghidra_wrapper.py | 13 ++++++++++++- scripts/ida/ida_wrapper.py | 9 ++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/scripts/ghidra/ghidra_wrapper.py b/scripts/ghidra/ghidra_wrapper.py index 6fa8e0c074..813edc7619 100644 --- a/scripts/ghidra/ghidra_wrapper.py +++ b/scripts/ghidra/ghidra_wrapper.py @@ -30,7 +30,18 @@ from java.util import ArrayList from jpype import JArray, JByte, JLong -from structs_schema import * +from structs_schema import ( + DefinedStruct, + DefinedStructEnum, + DefinedStructExport, + DefinedStructFixedField, + DefinedStructFuncField, + DefinedStructFuncParam, + DefinedStructMemFunc, + DefinedStructStaticMember, + DefinedStructVFunc, + get_template_alias, +) from data_schema import get_yaml, DefinedData, DefinedDataClassVtable from clang_wrapper import ClangInterface, split_type_path diff --git a/scripts/ida/ida_wrapper.py b/scripts/ida/ida_wrapper.py index fdd6e9c63a..74f93c4d8f 100644 --- a/scripts/ida/ida_wrapper.py +++ b/scripts/ida/ida_wrapper.py @@ -10,7 +10,14 @@ import ida_funcs import ida_srclang import ida_hexrays -from structs_schema import * +from structs_schema import ( + DefinedStruct, + DefinedStructFuncParam, + DefinedStructMemFunc, + DefinedStructStaticMember, + DefinedStructVFunc, + get_template_alias, +) from form_builder import DynamicForm from data_schema import get_yaml, DefinedData, DefinedDataClassVtable From f01798d2ccf3097e0b7555013fa23aeeb585f863 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:35:46 +0200 Subject: [PATCH 72/95] make CExporter log namespaces in cpp format for enums --- CExporter/Exporter.TypeProcessing.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CExporter/Exporter.TypeProcessing.cs b/CExporter/Exporter.TypeProcessing.cs index 2cf3b17466..d5c0678e9c 100644 --- a/CExporter/Exporter.TypeProcessing.cs +++ b/CExporter/Exporter.TypeProcessing.cs @@ -449,7 +449,7 @@ private static ProcessedField ProcessParameter( EnumValues = type.GetFields().Where(t => t.GetCustomAttribute() == null && t.FieldType == type).ToDictionary(f => f.Name, f => f.GetRawConstantValue()!.ToString()!) }; - if (!quiet) Console.WriteLine($"Processed {processedEnum.EnumNamespace}::{processedEnum.EnumName} with {processedEnum.EnumValues.Count} fields"); + if (!quiet) Console.WriteLine($"Processed {processedEnum.EnumNamespace.Replace(".", "::")}::{processedEnum.EnumName} with {processedEnum.EnumValues.Count} fields"); return processedEnum; } else if (type.IsStruct()) { while (type.IsPointer) type = type.GetElementType()!; From fc5516dd334b367ec61e3eb175030e7d3a09c841 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:42:59 +0200 Subject: [PATCH 73/95] change from ffxiv_structs.yml to structs.yml for better clarity of rewrite --- .gitattributes | 2 +- .github/workflows/pr-change-check-patch-day.yaml | 2 +- .github/workflows/pr-change-check.yaml | 2 +- CExporter/Exporter.cs | 2 +- Ghidra/Getting Started.md | 2 +- scripts/README.md | 2 +- scripts/ghidra/clang_wrapper.py | 4 ++-- scripts/ghidra/ghidra_wrapper.py | 2 +- scripts/ghidra/struct_import.py | 4 ++-- scripts/ida/data_struct_matcher.py | 2 +- scripts/ida/ida_wrapper.py | 2 +- scripts/ida/struct_import.py | 2 +- 12 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.gitattributes b/.gitattributes index c2f642f443..fbbdaff7ad 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,4 +3,4 @@ *.png binary ida/ffxiv_client_structs*.h linguist-generated=true -ida/ffxiv_structs.yml linguist-generated=true +scripts/structs.yml linguist-generated=true diff --git a/.github/workflows/pr-change-check-patch-day.yaml b/.github/workflows/pr-change-check-patch-day.yaml index d7c27db764..ed61d79873 100644 --- a/.github/workflows/pr-change-check-patch-day.yaml +++ b/.github/workflows/pr-change-check-patch-day.yaml @@ -108,5 +108,5 @@ jobs: - uses: actions/upload-artifact@v4 with: name: structs.yml - path: ./scripts/ffxiv_structs.yml + path: ./scripts/structs.yml retention-days: 7 diff --git a/.github/workflows/pr-change-check.yaml b/.github/workflows/pr-change-check.yaml index 87e92bd638..c42921aef6 100644 --- a/.github/workflows/pr-change-check.yaml +++ b/.github/workflows/pr-change-check.yaml @@ -109,7 +109,7 @@ jobs: - uses: actions/upload-artifact@v4 with: name: structs.yml - path: ./scripts/ffxiv_structs.yml + path: ./scripts/structs.yml retention-days: 7 build-pr: diff --git a/CExporter/Exporter.cs b/CExporter/Exporter.cs index 395be45941..2b1fd23bd5 100644 --- a/CExporter/Exporter.cs +++ b/CExporter/Exporter.cs @@ -456,7 +456,7 @@ void orderInlineStructs(ProcessedStruct parent) { Structs = _structs.Select(t => t.FixOrder()).ToArray() }); - new FileInfo(Path.Join(dir.FullName, "ffxiv_structs.yml")).WriteFile(yaml); + new FileInfo(Path.Join(dir.FullName, "structs.yml")).WriteFile(yaml); } } diff --git a/Ghidra/Getting Started.md b/Ghidra/Getting Started.md index 7e12e7cebd..9d2ae194c5 100644 --- a/Ghidra/Getting Started.md +++ b/Ghidra/Getting Started.md @@ -136,7 +136,7 @@ undefined8 FUN_1406cb6c0(undefined4 param_1) The label `Component::Exd::ExdModule.GetRowBySheetIndexAndRowIndex` came from data.yml, but unfortunately it's still not obvious exactly what that method is fetching. The sheet indexes are fairly well-known, and this script parses these and relabels the function from `FUN_1406cb6c0` to `Component::Exd::ExdModule_GetClassJobRow_1`, returning a `Component::Exd::Sheets::ClassJob*`, which is much more useful. -It also imports the Excel sheets and everything in [ffxiv_structs.yml](../scripts/ffxiv_structs.yml) as data types, and can apply the types of the virtual and member functions. It asks you to confirm the game version of data.yml, for the folder of the MSVC headers (outside of Windows) and which function types to update. Go ahead and run this too. +It also imports the Excel sheets and everything in [structs.yml](../scripts/structs.yml) as data types, and can apply the types of the virtual and member functions. It asks you to confirm the game version of data.yml, for the folder of the MSVC headers (outside of Windows) and which function types to update. Go ahead and run this too. ## full_run.py Runs rename.py and struct_import.py in sequence, then lets Ghidra analyze the changes. diff --git a/scripts/README.md b/scripts/README.md index 06a9d3c7ae..4349a0785c 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -5,7 +5,7 @@ Both folders hold the same scripts for their disassembler, and share the files i This script ingests the `data.yml` file in the scripts directory, and renames various offsets as globals, functions, vtables and such. ### struct_import.py -This script imports the Excel sheets and `ffxiv_structs.yml` as types, names the Excel getter functions and applies the function types. +This script imports the Excel sheets and `structs.yml` as types, names the Excel getter functions and applies the function types. Both generate the same C++ header and let clang work out the layout of every struct, template instances included. IDA uses its built in clang parser. Ghidra has no C++ parser, so `ghidra/clang_wrapper.py` reads the layouts back from libclang and `ghidra_wrapper.py` creates the data types from that. ### full_run.py diff --git a/scripts/ghidra/clang_wrapper.py b/scripts/ghidra/clang_wrapper.py index 1ccd6ad68f..df162772ed 100644 --- a/scripts/ghidra/clang_wrapper.py +++ b/scripts/ghidra/clang_wrapper.py @@ -19,7 +19,7 @@ # an instantiation can use templates that are only pointed to themselves max_instantiation_passes = 8 -# named like ffxiv_structs.yml does, sized for x86_64-pc-windows-msvc +# named like structs.yml does, sized for x86_64-pc-windows-msvc primitive_types = { TypeKind.VOID: "void", TypeKind.BOOL: "bool", @@ -67,7 +67,7 @@ def split_type_path(name: str) -> list[str]: class RecordCollector(object): - """Turns the records clang laid out into the same structs ffxiv_structs.yml is read as.""" + """Turns the records clang laid out into the same structs structs.yml is read as.""" def __init__(self): self.enums: dict[str, DefinedStructEnum] = {} diff --git a/scripts/ghidra/ghidra_wrapper.py b/scripts/ghidra/ghidra_wrapper.py index 813edc7619..1f95c9f23a 100644 --- a/scripts/ghidra/ghidra_wrapper.py +++ b/scripts/ghidra/ghidra_wrapper.py @@ -126,7 +126,7 @@ def load_data_yml(self): self.data_yaml = get_yaml(f) def get_struct_path(self): - return os.path.join(self.get_base_path(), "ffxiv_structs.yml") + return os.path.join(self.get_base_path(), "structs.yml") def to_addr(self, ea: int): return self.flat.toAddr(JLong(ea)) diff --git a/scripts/ghidra/struct_import.py b/scripts/ghidra/struct_import.py index a0f567a597..2ea195f9d5 100644 --- a/scripts/ghidra/struct_import.py +++ b/scripts/ghidra/struct_import.py @@ -1,4 +1,4 @@ -# Imports the Excel sheets and ffxiv_structs.yml as data types, and names and types the functions using them +# Imports the Excel sheets and structs.yml as data types, and names and types the functions using them # @category __UserScripts # @menupath Tools.Scripts.ffxiv_struct_import # @runtime PyGhidra @@ -88,7 +88,7 @@ excel_header, excel_vtables = src.build_export_string(excel_structs, 4) -print("Parsing ffxiv_structs.yml") +print("Parsing structs.yml") with open(ghidra.get_struct_path()) as f: import_structs = get_yaml(f) diff --git a/scripts/ida/data_struct_matcher.py b/scripts/ida/data_struct_matcher.py index 70ded7c7bb..dbb66a3ab5 100644 --- a/scripts/ida/data_struct_matcher.py +++ b/scripts/ida/data_struct_matcher.py @@ -10,7 +10,7 @@ print("Loading data.yml") ida.load_data_yml() -print("Parsing ffxiv_structs.yml") +print("Parsing structs.yml") with open(ida.get_struct_path()) as f: structs = {struct.type: struct for struct in get_yaml(f).structs} diff --git a/scripts/ida/ida_wrapper.py b/scripts/ida/ida_wrapper.py index 74f93c4d8f..61e53bf0a4 100644 --- a/scripts/ida/ida_wrapper.py +++ b/scripts/ida/ida_wrapper.py @@ -75,7 +75,7 @@ def load_data_yml(self): self.data_yaml = get_yaml(f) def get_struct_path(self): - return os.path.join(self.get_base_path(), "ffxiv_structs.yml") + return os.path.join(self.get_base_path(), "structs.yml") def get_enum_id(self, name: str): return idc.get_enum(name) diff --git a/scripts/ida/struct_import.py b/scripts/ida/struct_import.py index c34152e1fa..2b4e609aa4 100644 --- a/scripts/ida/struct_import.py +++ b/scripts/ida/struct_import.py @@ -107,7 +107,7 @@ excel_header, excel_vtables = src.build_export_string(excel_structs, 4) -print("Parsing ffxiv_structs.yml") +print("Parsing structs.yml") with open(ida.get_struct_path()) as f: import_structs = get_yaml(f) From fee0b2867ade7da24c2dd2a26edeade016c18cbb Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:43:40 +0200 Subject: [PATCH 74/95] still retain the old gitattribute until file is removed --- .gitattributes | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitattributes b/.gitattributes index fbbdaff7ad..cb1ea3aa99 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,4 +3,5 @@ *.png binary ida/ffxiv_client_structs*.h linguist-generated=true +scripts/ffxiv_structs.yml linguist-generated=true scripts/structs.yml linguist-generated=true From 987a3a1c3e6a5b13862bea3ec3adc8bc5740ed30 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 14 Oct 2026 13:00:15 +0200 Subject: [PATCH 75/95] delete old excel logic --- scripts/luminapie/__init__.py | 0 scripts/luminapie/definitions.py | 110 ----------- scripts/luminapie/enums.py | 64 ------ scripts/luminapie/excel.py | 305 ----------------------------- scripts/luminapie/exdschema.py | 66 ------- scripts/luminapie/file_handlers.py | 39 ---- scripts/luminapie/game_data.py | 132 ------------- scripts/luminapie/se_crc.py | 74 ------- scripts/luminapie/sqpack.py | 225 --------------------- 9 files changed, 1015 deletions(-) delete mode 100644 scripts/luminapie/__init__.py delete mode 100644 scripts/luminapie/definitions.py delete mode 100644 scripts/luminapie/enums.py delete mode 100644 scripts/luminapie/excel.py delete mode 100644 scripts/luminapie/exdschema.py delete mode 100644 scripts/luminapie/file_handlers.py delete mode 100644 scripts/luminapie/game_data.py delete mode 100644 scripts/luminapie/se_crc.py delete mode 100644 scripts/luminapie/sqpack.py diff --git a/scripts/luminapie/__init__.py b/scripts/luminapie/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/scripts/luminapie/definitions.py b/scripts/luminapie/definitions.py deleted file mode 100644 index a2f9342da5..0000000000 --- a/scripts/luminapie/definitions.py +++ /dev/null @@ -1,110 +0,0 @@ -def get_definition(schema): - # type: (dict[str, str]) -> Definition - if "type" in schema: - if schema["type"] == "array": - return RepeatDefinition(schema) - return Definition(schema) - - -class Definition: - def __init__(self, obj): - # type: (dict[str, str]) -> None - self.name = obj["name"] - if "pendingName" in obj: - self.pendingName = obj["pendingName"] - - def get_name(self): - # type: () -> str - if hasattr(self, "pendingName"): - return self.pendingName - else: - return self.name - - def __repr__(self): - # type: () -> str - return self.name - - -class RepeatDefinition(Definition): - def __init__(self, obj): - # type: (dict[str, str]) -> None - super().__init__(obj) - self.obj = obj - self.count = obj["count"] - self.inner_defs = [] - self.process_inner() - - def process_inner(self): - # type: () -> None - if "fields" in self.obj: - for field in self.obj["fields"]: - if "name" in field: - self.inner_defs.append(get_definition(field)) - else: - self.inner_defs.append(Definition({"name": ""})) - if self.inner_defs == []: - self.inner_defs.append(Definition({"name": ""})) - - def flatten(self, extern): - # type: (str) -> list[Definition] - defs = [] - extern = extern + self.name - for i in range(0, int(self.count)): - for inner in self.inner_defs: - if isinstance(inner, RepeatDefinition): - defs.extend(inner.flatten(extern + i.__str__())) - else: - defs.append(Definition({"name": extern + i.__str__() + inner.name})) - return defs - - def __repr__(self): - # type: () -> str - return self.flatten("") - - -class SemanticVersion: - """Represents a semantic version string that can compare versions""" - - def __init__(self, year, month, date, patch, build=0): - # type: (int, int, int, int, int) -> None - self.year = year - self.month = month - self.date = date - self.patch = patch - self.build = build - - def __lt__(self, other): - # type: (SemanticVersion) -> bool - return ( - self.year < other.year - or self.month < other.month - or self.date < other.date - or self.patch < other.patch - or self.build < other.build - ) - - def __repr__(self): - # type: () -> str - return "{0}.{1}.{2}.{3}.{4}".format( - self.year, - self.month.__str__().rjust(2, "0"), - self.date.__str__().rjust(2, "0"), - self.patch.__str__().rjust(4, "0"), - self.build.__str__().rjust(4, "0"), - ) - - def __eq__(self, __value): - # type: (object) -> bool - if not isinstance(__value, SemanticVersion): - return False - return ( - self.year == __value.year - and self.month == __value.month - and self.date == __value.date - and self.patch == __value.patch - and self.build == __value.build - ) - - def __hash__(self): - # type: () -> int - return hash(repr(self)) diff --git a/scripts/luminapie/enums.py b/scripts/luminapie/enums.py deleted file mode 100644 index 44b90a4ed8..0000000000 --- a/scripts/luminapie/enums.py +++ /dev/null @@ -1,64 +0,0 @@ -import enum - - -class SqPackCatergories(enum.IntEnum): - COMMON = 0x0 - BGCOMMON = 0x1 - BG = 0x2 - CUT = 0x3 - CHARA = 0x4 - SHADER = 0x5 - UI = 0x6 - SOUND = 0x7 - VFX = 0x8 - UI_SCRIPT = 0x9 - EXD = 0xA - GAME_SCRIPT = 0xB - MUSIC = 0xC - SQPACK_TEST = 0x12 - DEBUG = 0x13 - - -class SqPackPlatformId(enum.IntEnum): - Win32 = 0x0 - PS3 = 0x1 - PS4 = 0x2 - - -class SqPackFileType(enum.IntEnum): - Empty = 1 - Standard = 2 - Model = 3 - Texture = 4 - - -class DatBlockType(enum.IntEnum): - Compressed = 4713 - Uncompressed = 32000 - - -class ExcelColumnDataType(enum.IntEnum): - String = 0x0 - Bool = 0x1 - Int8 = 0x2 - UInt8 = 0x3 - Int16 = 0x4 - UInt16 = 0x5 - Int32 = 0x6 - UInt32 = 0x7 - # unused? - Unk = 0x8 - Float32 = 0x9 - Int64 = 0xA - UInt64 = 0xB - # unused? - Unk2 = 0xC - # 0 is read like data & 1, 1 is like data & 2, 2 = data & 4, etc... - PackedBool0 = 0x19 - PackedBool1 = 0x1A - PackedBool2 = 0x1B - PackedBool3 = 0x1C - PackedBool4 = 0x1D - PackedBool5 = 0x1E - PackedBool6 = 0x1F - PackedBool7 = 0x20 diff --git a/scripts/luminapie/excel.py b/scripts/luminapie/excel.py deleted file mode 100644 index 9e77754f92..0000000000 --- a/scripts/luminapie/excel.py +++ /dev/null @@ -1,305 +0,0 @@ -from luminapie.enums import ExcelColumnDataType -from luminapie.definitions import Definition - - -class ExcelListFile: - def __init__(self, data): - # type: (list[bytes]) -> None - self.data = b"".join(data).split("\r\n".encode("utf-8")) - self.parse() - - def parse(self): - # type: () -> None - self.header = self.data[0].decode("utf-8").split(",") - self.version = int(self.header[1]) - self.data = self.data[1:] - self.dict: dict[int, str] = {} - for line in [x.decode("utf-8") for x in self.data]: - if line == "": - continue - linearr = line.split(",") - if linearr[1] == "-1": - continue - self.dict[int(linearr[1])] = linearr[0] - - def __repr__(self): - # type: () -> str - return "ExcelListFile: {0}, {1}".format(self.header, self.dict) - - -class ExcelHeader: - def __init__(self, data): - # type: (bytes) -> None - self.data = data - self.parse() - - def parse(self): - # type: () -> None - self.magic = self.data[0:4] - self.version = int.from_bytes(self.data[4:6], "big") - self.data_offset = int.from_bytes(self.data[6:8], "big") - self.column_count = int.from_bytes(self.data[8:10], "big") - self.page_count = int.from_bytes(self.data[10:12], "big") - self.language_count = int.from_bytes(self.data[12:14], "big") - self.unknown1 = int.from_bytes(self.data[14:16], "big") - self.unknown2 = self.data[17] - self.variant = self.data[18] - self.unknown3 = int.from_bytes(self.data[19:20], "big") - self.row_count = int.from_bytes(self.data[20:24], "big") - self.unknown4 = [ - int.from_bytes(self.data[24:28], "big"), - int.from_bytes(self.data[28:32], "big"), - ] - - def __repr__(self): - # type: () -> str - return "Header: {0}, version: {1}, data_offset: {2}, column_count: {3}, page_count: {4}, language_count: {5}, unknown1: {6}, unknown2: {7}, variant: {8}, unknown3: {9}, row_count: {10}, unknown4: {11}".format( - self.magic, - self.version, - self.data_offset, - self.column_count, - self.page_count, - self.language_count, - self.unknown1, - self.unknown2, - self.variant, - self.unknown3, - self.row_count, - self.unknown4, - ) - - -class ExcelColumnDefinition: - def __init__(self, data): - # type: (bytes) -> None - self.data = data - self.parse() - - def parse(self): - # type: () -> None - self.type = ExcelColumnDataType(int.from_bytes(self.data[0:2], "big")) - self.offset = int.from_bytes(self.data[2:4], "big") - - def __lt__(self, other): - # type: (ExcelColumnDefinition) -> bool - return self.offset < other.offset - - def __eq__(self, other): - return self.offset == other.offset and self.type == other.offset - - def __repr__(self): - # type: () -> str - return "Column: {0}, offset: {1}".format(self.type.name, self.offset) - - -class ExcelDataPagination: - def __init__(self, data): - # type: (bytes) -> None - self.data = data - self.parse() - - def parse(self): - # type: () -> None - self.start_id = int.from_bytes(self.data[0:2], "big") - self.row_count = int.from_bytes(self.data[2:4], "big") - - def __repr__(self): - # type: () -> str - return "Pagination: {0:x}, count: {1}".format(self.start_id, self.row_count) - - -class ExcelHeaderFile: - def __init__(self, data, name): - # type: (list[bytes], str) -> None - self.data = data[0] - self.column_definitions: list[ExcelColumnDefinition] = [] - self.pagination: list[ExcelDataPagination] = [] - self.languages: list[int] = [] - self.header: ExcelHeader = None - self.name = name - self.parse() - - def parse(self): - # type: () -> None - self.header = ExcelHeader(self.data[0:32]) - if self.header.magic != b"EXHF": - raise Exception("Invalid EXHF header") - self.column_definitions: list[ExcelColumnDefinition] = [] - for i in range(self.header.column_count): - self.column_definitions.append( - ExcelColumnDefinition(self.data[32 + (i * 4) : 32 + ((i + 1) * 4)]) - ) - self.column_definitions = sorted(self.column_definitions) - self.pagination: list[ExcelDataPagination] = [] - for i in range(self.header.page_count): - self.pagination.append( - ExcelDataPagination( - self.data[ - 32 - + (self.header.column_count * 4) - + (i * 4) : 32 - + (self.header.column_count * 4) - + ((i + 1) * 4) - ] - ) - ) - self.languages: list[int] = [] - for i in range(self.header.language_count): - self.languages.append( - self.data[ - 32 - + (self.header.column_count * 4) - + (self.header.page_count * 4) - + i - ] - ) - - def map_names(self, names: list[Definition]) -> tuple[dict[int, tuple[str, str]], dict[str, dict[int, str]], int]: - """ - Map the column definitions to the names in the header. - - Args: - names: A list of names of the columns in the header. - - Returns: - A tuple of a dictionary mapping column offsets to tuples of the column type - and name, and the size of the structure. - """ - mapped: dict[int, tuple[str, str]] = {} - enumMapped: dict[str, dict[int, str]] = {} - largest_offset_index: int = 0 - for i in range(self.header.column_count): - if ( - self.column_definitions[i].offset - > self.column_definitions[largest_offset_index].offset - ): - largest_offset_index = i - - size = self.column_definitions[ - largest_offset_index - ].offset + column_data_type_to_size( - self.column_definitions[largest_offset_index].type - ) - - if self.header.column_count != len(names): - for i in range(self.header.column_count): - mapped[self.column_definitions[i].offset] = ( - column_data_type_to_c_type(self.column_definitions[i].type), - "field_{0}".format(i), - ) - else: - for i in range(self.header.column_count): - col_def = self.column_definitions[i] - if ( - col_def.offset in mapped - and mapped[col_def.offset] is not None - ): - if (col_def.type in {ExcelColumnDataType.PackedBool0, ExcelColumnDataType.PackedBool1, ExcelColumnDataType.PackedBool2, ExcelColumnDataType.PackedBool3, ExcelColumnDataType.PackedBool4, ExcelColumnDataType.PackedBool5, ExcelColumnDataType.PackedBool6, ExcelColumnDataType.PackedBool7}): - name = "PackedBool{0:X}".format(col_def.offset) - if (name in enumMapped): - enumMapped["PackedBool{0:X}".format(col_def.offset)][(1 << col_def.type - ExcelColumnDataType.PackedBool0)] = "{0}_{1}".format(name, names[i].get_name()) - else: - # this should never be hit but just to be safe - enumMapped[name] = {(1 << col_def.type - ExcelColumnDataType.PackedBool0): "{0}_{1}".format(name, names[i].get_name())} - else: - [_, name] = mapped[col_def.offset] - if name.split("_")[0] == "Unknown": - continue - if ( - column_data_type_to_c_type(col_def.type) - != "unsigned __int8" - ): - continue - else: - mapped[col_def.offset] = ( - column_data_type_to_c_type(col_def.type), - names[i].get_name(), - ) - else: - if(col_def.type in {ExcelColumnDataType.PackedBool0, ExcelColumnDataType.PackedBool1, ExcelColumnDataType.PackedBool2, ExcelColumnDataType.PackedBool3, ExcelColumnDataType.PackedBool4, ExcelColumnDataType.PackedBool5, ExcelColumnDataType.PackedBool6, ExcelColumnDataType.PackedBool7}): - name = "PackedBool{0:X}".format(col_def.offset) - mapped[col_def.offset] = ( - f"Component::Exd::Sheets::{self.name}::{name}", - name - ) - enumMapped[name] = {(1 << col_def.type - ExcelColumnDataType.PackedBool0): "{0}_{1}".format(name, names[i].get_name())} - else: - mapped[self.column_definitions[i].offset] = ( - column_data_type_to_c_type(self.column_definitions[i].type), - names[i].get_name(), - ) - mapped = dict(sorted(mapped.items())) - enumMapped = dict(sorted(enumMapped.items())) - return [mapped, enumMapped, size] - - -def column_data_type_to_c_type(column_data_type): - # type: (ExcelColumnDataType) -> str - if column_data_type == ExcelColumnDataType.Bool: - return "bool" - elif column_data_type == ExcelColumnDataType.Int8: - return "__int8" - elif column_data_type == ExcelColumnDataType.UInt8: - return "unsigned __int8" - elif column_data_type == ExcelColumnDataType.Int16: - return "__int16" - elif column_data_type == ExcelColumnDataType.UInt16: - return "unsigned __int16" - elif column_data_type == ExcelColumnDataType.Int32: - return "__int32" - elif column_data_type == ExcelColumnDataType.UInt32: - return "unsigned __int32" - elif column_data_type == ExcelColumnDataType.Float32: - return "float" - elif column_data_type == ExcelColumnDataType.Int64: - return "__int64" - elif column_data_type == ExcelColumnDataType.UInt64: - return "unsigned __int64" - elif ( - column_data_type == ExcelColumnDataType.PackedBool0 - or column_data_type == ExcelColumnDataType.PackedBool1 - or column_data_type == ExcelColumnDataType.PackedBool2 - or column_data_type == ExcelColumnDataType.PackedBool3 - or column_data_type == ExcelColumnDataType.PackedBool4 - or column_data_type == ExcelColumnDataType.PackedBool5 - or column_data_type == ExcelColumnDataType.PackedBool6 - or column_data_type == ExcelColumnDataType.PackedBool7 - ): - return "unsigned __int8" - elif column_data_type == ExcelColumnDataType.String: - return "unsigned __int32" - - -def column_data_type_to_size(column_data_type): - # type: (ExcelColumnDataType) -> int - if ( - column_data_type == ExcelColumnDataType.Bool - or column_data_type == ExcelColumnDataType.Int8 - or column_data_type == ExcelColumnDataType.UInt8 - or column_data_type == ExcelColumnDataType.PackedBool0 - or column_data_type == ExcelColumnDataType.PackedBool1 - or column_data_type == ExcelColumnDataType.PackedBool2 - or column_data_type == ExcelColumnDataType.PackedBool3 - or column_data_type == ExcelColumnDataType.PackedBool4 - or column_data_type == ExcelColumnDataType.PackedBool5 - or column_data_type == ExcelColumnDataType.PackedBool6 - or column_data_type == ExcelColumnDataType.PackedBool7 - ): - return 1 - elif ( - column_data_type == ExcelColumnDataType.Int16 - or column_data_type == ExcelColumnDataType.UInt16 - ): - return 2 - elif ( - column_data_type == ExcelColumnDataType.Int32 - or column_data_type == ExcelColumnDataType.UInt32 - or column_data_type == ExcelColumnDataType.Float32 - or column_data_type == ExcelColumnDataType.String - ): - return 4 - elif ( - column_data_type == ExcelColumnDataType.Int64 - or column_data_type == ExcelColumnDataType.UInt64 - ): - return 8 diff --git a/scripts/luminapie/exdschema.py b/scripts/luminapie/exdschema.py deleted file mode 100644 index fc2e798b7c..0000000000 --- a/scripts/luminapie/exdschema.py +++ /dev/null @@ -1,66 +0,0 @@ -from urllib.request import urlopen, Request -from urllib.error import HTTPError, URLError -from luminapie.definitions import ( - Definition, - RepeatDefinition, - get_definition, - SemanticVersion, -) -from yaml import load, Loader -from zipfile import ZipFile -from tempfile import TemporaryFile - - -def get_url(url, supress=False): - # type: (str, bool) -> bytes | None - req = Request(url) - try: - resp = urlopen(req) - return resp.read() - except HTTPError as e: - if not supress: - print("HTTP Error code: ", e.code, " for url: ", url) - return None - except URLError as e: - if not supress: - print("HTTP Reason: ", e.reason, " for url: ", url) - return None - - -# TODO: Add the ability to use previous version schemas as well -def get_definitions(schema): - # type: (SemanticVersion) -> dict[str, list[Definition]] - exd_schema_map = {} - with TemporaryFile() as f: - f.write( - get_url( - "https://github.com/xivdev/EXDSchema/archive/refs/heads/latest.zip", - True, - ) - ) - f.seek(0) - schema_zip = ZipFile(f) - - for file in schema_zip.namelist(): - if file.endswith(".yml") and ".github" not in file: - schema_yml = load(schema_zip.read(file), Loader=Loader) - if "pendingFields" in schema_yml: - exd_schema_map[file.rsplit(".", 1)[0].rsplit("/")[1]] = schema_yml[ - "pendingFields" - ] - else: - exd_schema_map[file.rsplit(".", 1)[0].rsplit("/")[1]] = schema_yml[ - "fields" - ] - - defs_map = {} - for exd in exd_schema_map: - defs = [] - for field in exd_schema_map[exd]: - defin = get_definition(field) - if isinstance(defin, RepeatDefinition): - defs.extend(defin.flatten("")) - else: - defs.append(defin) - defs_map[exd] = defs - return defs_map diff --git a/scripts/luminapie/file_handlers.py b/scripts/luminapie/file_handlers.py deleted file mode 100644 index 27a2421018..0000000000 --- a/scripts/luminapie/file_handlers.py +++ /dev/null @@ -1,39 +0,0 @@ -import os - - -def get_game_data_folders(root): - # type: (str) -> str - for folder in os.listdir(os.path.join(root, "sqpack")): - if os.path.isdir(os.path.join(root, "sqpack", folder)): - yield folder - - -def get_files(path): - # type: (str) -> list[bytes] - files: list[bytes] = [] - for dir_path, dir_names, file_names in os.walk(path): - files.extend(os.path.join(dir_path, file) for file in file_names) - - return files - - -def get_sqpack_files(root, path): - # type: (str, str) -> str - for file in get_files(os.path.join(root, "sqpack", path)): - ext = file.split(".")[-1] - if ext.startswith("dat"): - yield file - - -def get_sqpack_index(root, path): - # type: (str, str) -> str - for file in get_files(os.path.join(root, "sqpack", path)): - if file.endswith(".index"): - yield file - - -def get_sqpack_index2(root, path): - # type: (str, str) -> str - for file in get_files(os.path.join(root, "sqpack", path)): - if file.endswith(".index2"): - yield file diff --git a/scripts/luminapie/game_data.py b/scripts/luminapie/game_data.py deleted file mode 100644 index 05ca000be9..0000000000 --- a/scripts/luminapie/game_data.py +++ /dev/null @@ -1,132 +0,0 @@ -from luminapie.sqpack import SqPack, SqPackIndexHashTable -from luminapie.file_handlers import get_game_data_folders, get_sqpack_index -from luminapie.se_crc import Crc32 -from luminapie.exdschema import get_definitions -from luminapie.definitions import SemanticVersion -import os - -crc = Crc32() - - -class Repository: - def __init__(self, name, root): - # type: (str, str) -> None - self.root = root - self.name = name - self.sqpacks: list[SqPack] = [] - self.index: dict[int, tuple[SqPackIndexHashTable, SqPack]] = {} - self.expansion_id = 0 - self.get_expansion_id() - - def get_expansion_id(self): - # type: () -> None - if self.name.startswith("ex"): - self.expansion_id = int(self.name.removeprefix("ex")) - - def parse_version(self): - # type: () -> None - versionPath = "" - if self.name == "ffxiv": - versionPath = os.path.join(self.root, "ffxivgame.ver") - else: - versionPath = os.path.join( - self.root, "sqpack", self.name, self.name + ".ver" - ) - if os.path.exists(versionPath): - with open(versionPath, "r") as f: - self.version = SemanticVersion( - *(int(v) for v in f.read().strip().split(".")) - ) - else: - self.version = SemanticVersion(0, 0, 0, 0) - - def setup_indexes(self): - # type: () -> None - for file in get_sqpack_index(self.root, self.name): - self.sqpacks.append(SqPack(self.root, file)) - - for sqpack in self.sqpacks: - sqpack.discover_data_files() - for indexes in sqpack.hash_table: - self.index[indexes.hash] = [indexes, sqpack] - - def get_index(self, hash): - # type: (int) -> tuple[SqPackIndexHashTable, SqPack] - return self.index[hash] - - def get_file(self, hash): - # type: (int) -> bytes - index, sqpack = self.get_index(hash) - id = index.data_file_id() - offset = index.data_file_offset() - return SqPack(self.root, sqpack.data_files[id]).read_file(offset) - - def __repr__(self): - # type: () -> str - return "Repository: {0} ({1}) - {2}".format( - self.name, self.version, self.expansion_id - ) - - -class GameData: - def __init__(self, root, load_schema=True): - # type: (str, bool) -> None - self.root = root - self.repositories: dict[int, Repository] = {} - self.load_schema = load_schema - self.setup() - - def get_repo_index(self, folder): - # type: (str) -> int - if folder == "ffxiv": - return 0 - else: - return int(folder.removeprefix("ex")) - - def setup(self): - # type: () -> None - for folder in get_game_data_folders(self.root): - self.repositories[self.get_repo_index(folder)] = Repository( - folder, self.root - ) - - for folder in self.repositories: - repo = self.repositories[folder] - repo.parse_version() - repo.setup_indexes() - - if self.load_schema: - self.schema = get_definitions(self.repositories[0].version) - - def get_file(self, file): - # type: (ParsedFileName) -> bytes - return self.repositories[self.get_repo_index(file.repo)].get_file(file.index) - - def get_exd_schema(self, key): - # type: (str) -> list[Definition] - if key not in self.schema: - return [] - return self.schema[key] - - def __repr__(self): - # type: () -> str - return "Repositories: {0}".format(self.repositories) - - -class ParsedFileName: - def __init__(self, path): - # type: (str) -> None - self.path = path.lower().strip() - parts = self.path.split("/") - self.category = parts[0] - self.index = crc.calc_index(self.path) - self.index2 = crc.calc_index2(self.path) - self.repo = parts[1] - if self.repo[0] != "e" or self.repo[1] != "x" or not self.repo[2].isdigit(): - self.repo = "ffxiv" - - def __repr__(self): - # type: () -> str - return "ParsedFileName: {0}, category: {1}, index: {2:X}, index2: {3:X}, repo: {4}".format( - self.path, self.category, self.index, self.index2, self.repo - ) diff --git a/scripts/luminapie/se_crc.py b/scripts/luminapie/se_crc.py deleted file mode 100644 index 6d3729222d..0000000000 --- a/scripts/luminapie/se_crc.py +++ /dev/null @@ -1,74 +0,0 @@ -class Crc32: - def __init__(self): - # type: () -> None - self.poly = 0xEDB88320 - self.table = [0] * 256 * 16 - for i in range(256): - res = i - for j in range(16): - for k in range(8): - if res & 1 == 1: - res = self.poly ^ (res >> 1) - else: - res = res >> 1 - self.table[i + j * 256] = res - - def calc(self, value): - # type: (bytes) -> int - start = 0 - size = len(value) - crc_local = 4294967295 ^ 0 - while size >= 16: - a = ( - self.table[(3 * 256) + value[start + 12]] - ^ self.table[(2 * 256) + value[start + 13]] - ^ self.table[(1 * 256) + value[start + 14]] - ^ self.table[(0 * 256) + value[start + 15]] - ) - b = ( - self.table[(7 * 256) + value[start + 8]] - ^ self.table[(6 * 256) + value[start + 9]] - ^ self.table[(5 * 256) + value[start + 10]] - ^ self.table[(4 * 256) + value[start + 11]] - ) - c = ( - self.table[(11 * 256) + value[start + 4]] - ^ self.table[(10 * 256) + value[start + 5]] - ^ self.table[(9 * 256) + value[start + 6]] - ^ self.table[(8 * 256) + value[start + 7]] - ) - d = ( - self.table[(15 * 256) + (self.byte(crc_local) ^ value[start])] - ^ self.table[(14 * 256) + (self.byte(crc_local, 1) ^ value[start + 1])] - ^ self.table[(13 * 256) + (self.byte(crc_local, 2) ^ value[start + 2])] - ^ self.table[(12 * 256) + (self.byte(crc_local, 3) ^ value[start + 3])] - ) - crc_local = d ^ c ^ b ^ a - start += 16 - size -= 16 - - while size > 0: - crc_local = self.table[(crc_local ^ value[start]) & 0xFF] ^ (crc_local >> 8) - start += 1 - size -= 1 - - return ~(crc_local ^ 4294967295) % (1 << 32) - - def byte(self, number, i=0): - # type: (int, int) -> int - return (number & (0xFF << (i * 8))) >> (i * 8) - - def calc_index(self, path): - # type: (str) -> int - path_parts = path.split("/") - filename = path_parts[-1] - folder = path.rstrip(filename).rstrip("/") - - foldercrc = self.calc(folder.encode("utf-8")) - filecrc = self.calc(filename.encode("utf-8")) - - return foldercrc << 32 | filecrc - - def calc_index2(self, path): - # type: (str) -> int - return self.calc(path.encode("utf-8")) diff --git a/scripts/luminapie/sqpack.py b/scripts/luminapie/sqpack.py deleted file mode 100644 index ef6be0ad5a..0000000000 --- a/scripts/luminapie/sqpack.py +++ /dev/null @@ -1,225 +0,0 @@ -from luminapie.enums import SqPackFileType, SqPackPlatformId -from io import BufferedReader -import os -import zlib -from luminapie.file_handlers import get_sqpack_files - - -class SqPackFileInfo: - def __init__(self, bytes, offset): - # type: (bytes, int) -> None - self.header_size = int.from_bytes(bytes[0:4], byteorder="little") - self.type = SqPackFileType(int.from_bytes(bytes[4:8], byteorder="little")) - self.raw_file_size = int.from_bytes(bytes[8:12], byteorder="little") - self.unknown = [ - int.from_bytes(bytes[12:16], byteorder="little"), - int.from_bytes(bytes[16:20], byteorder="little"), - ] - self.number_of_blocks = int.from_bytes(bytes[20:24], byteorder="little") - self.offset = offset - - -class DatStdFileBlockInfos: - def __init__(self, bytes): - # type: (bytes) -> None - self.offset = int.from_bytes(bytes[0:4], byteorder="little") - self.compressed_size = int.from_bytes(bytes[4:6], byteorder="little") - self.uncompressed_size = int.from_bytes(bytes[6:8], byteorder="little") - - -class DatBlockHeader: - def __init__(self, bytes): - # type: (bytes) -> None - self.size = int.from_bytes(bytes[0:4], byteorder="little") - self.unknown1 = int.from_bytes(bytes[4:8], byteorder="little") - self.block_data_size = int.from_bytes(bytes[8:12], byteorder="little") - self.dat_block_type = int.from_bytes(bytes[12:16], byteorder="little") - - def __repr__(self): - # type: () -> str - return "Size: {0} Unknown1: {1} DatBlockType: {2} BlockDataSize: {3}".format( - self.size, self.unknown1, self.dat_block_type, self.block_data_size - ) - - -class SqPackHeader: - def __init__(self, file): - # type: (BufferedReader) -> None - self.magic = file.read(8) - self.platform_id = SqPackPlatformId( - int.from_bytes(file.read(1), byteorder="little") - ) - self.unknown = file.read(3) - if self.platform_id != SqPackPlatformId.PS3: - self.size = int.from_bytes(file.read(4), byteorder="little") - self.version = int.from_bytes(file.read(4), byteorder="little") - self.type = int.from_bytes(file.read(4), byteorder="little") - else: - raise Exception("PS3 is not supported") - - def __repr__(self): - # type: () -> str - return "Magic: {0} Platform: {1} Size: {2} Version: {3} Type: {4}".format( - self.magic, self.platform_id, self.size, self.version, self.type - ) - - -class SqPackIndexHeader: - def __init__(self, bytes: bytes): - # type: (bytes) -> None - self.size = int.from_bytes(bytes[0:4], byteorder="little") - self.version = int.from_bytes(bytes[4:8], byteorder="little") - self.index_data_offset = int.from_bytes(bytes[8:12], byteorder="little") - self.index_data_size = int.from_bytes(bytes[12:16], byteorder="little") - self.index_data_hash = bytes[16:80] - self.number_of_data_file = int.from_bytes(bytes[80:84], byteorder="little") - self.synonym_data_offset = int.from_bytes(bytes[84:88], byteorder="little") - self.synonym_data_size = int.from_bytes(bytes[88:92], byteorder="little") - self.synonym_data_hash = bytes[92:156] - self.empty_block_data_offset = int.from_bytes( - bytes[156:160], byteorder="little" - ) - self.empty_block_data_size = int.from_bytes(bytes[160:164], byteorder="little") - self.empty_block_data_hash = bytes[164:228] - self.dir_index_data_offset = int.from_bytes(bytes[228:232], byteorder="little") - self.dir_index_data_size = int.from_bytes(bytes[232:236], byteorder="little") - self.dir_index_data_hash = bytes[236:300] - self.index_type = int.from_bytes(bytes[300:304], byteorder="little") - self.reserved = bytes[304:960] - self.hash = bytes[960:1024] - - def __repr__(self): - # type: () -> str - return "Size: {0} Version: {1} Index Data Offset: {2} Index Data Size: {3} Index Data Hash: {4} Number Of Data File: {5} Synonym Data Offset: {6} Synonym Data Size: {7} Synonym Data Hash: {8} Empty Block Data Offset: {9} Empty Block Data Size: {10} Empty Block Data Hash: {11} Dir Index Data Offset: {12} Dir Index Data Size: {13} Dir Index Data Hash: {14} Index Type: {15} Reserved: {16} Hash: {17}".format( - self.size, - self.version, - self.index_data_offset, - self.index_data_size, - self.index_data_hash, - self.number_of_data_file, - self.synonym_data_offset, - self.synonym_data_size, - self.synonym_data_hash, - self.empty_block_data_offset, - self.empty_block_data_size, - self.empty_block_data_hash, - self.dir_index_data_offset, - self.dir_index_data_size, - self.dir_index_data_hash, - self.index_type, - self.reserved, - self.hash, - ) - - -class SqPackIndexHashTable: - def __init__(self, bytes): - # type: (bytes) -> None - self.hash = int.from_bytes(bytes[0:8], byteorder="little") - self.data = int.from_bytes(bytes[8:12], byteorder="little") - self.padding = int.from_bytes(bytes[12:16], byteorder="little") - - def is_synonym(self): - # type: () -> bool - return (self.data & 0b1) == 0b1 - - def data_file_id(self): - # type: () -> int - return (self.data & 0b1110) >> 1 - - def data_file_offset(self): - # type: () -> int - return (self.data & ~0xF) * 0x08 - - def __repr__(self): - # type: () -> str - return "Hash: {0} Data: {1} Padding: {2} Is Synonym: {3} Data File ID: {4} Data File Offset: {5}".format( - self.hash, - self.data, - self.padding, - self.is_synonym(), - self.data_file_id(), - self.data_file_offset(), - ) - - -class SqPack: - def __init__(self, root, path): - # type: (str, str) -> None - self.root = root - self.path = path - self.file = open(path, "rb") - self.header = SqPackHeader(self.file) - - def get_index_header(self): - # type: () -> SqPackIndexHeader - self.file.seek(self.header.size) - return SqPackIndexHeader(self.file.read(1024)) - - def get_index_hash_table(self, index_header): - # type: (SqPackIndexHeader) -> list[SqPackIndexHashTable] - self.file.seek(index_header.index_data_offset) - entry_count = index_header.index_data_size // 16 - return [SqPackIndexHashTable(self.file.read(16)) for _ in range(entry_count)] - - def load_index_header(self): - # type: () -> None - self.index_header = self.get_index_header() - - def load_hash_table(self): - # type: () -> None - self.hash_table = self.get_index_hash_table(self.index_header) - - def discover_data_files(self): - # type: () -> None - self.load_index_header() - self.load_hash_table() - self.data_files: list[str] = [] - for file in get_sqpack_files( - self.root, self.path.rsplit(os.sep, 1)[0].split(os.sep)[-1] - ): - for i in range(0, self.index_header.number_of_data_file): - name = self.path.rsplit(".", 1)[0] + ".dat" + str(i) - if file == name: - self.data_files.append(file) - - def read_file(self, offset): - # type: (int) -> list[bytes] - if self.path.rsplit(".", 1)[1][0:3] != "dat": - raise Exception("Not a data file") - self.file.seek(offset) - file_info_bytes = self.file.read(24) - file_info = SqPackFileInfo(file_info_bytes, offset) - data: list[bytes] = [] - if file_info.type == SqPackFileType.Empty: - raise Exception(f"File located at 0x{hex(offset)} is empty.") - elif file_info.type == SqPackFileType.Standard: - data = self.read_standard_file(file_info) - else: - raise Exception("Type: " + str(file_info.type) + " not implemented.") - return data - - def read_standard_file(self, file_info): - # type: (SqPackFileInfo) -> list[bytes] - block_bytes = self.file.read(file_info.number_of_blocks * 8) - data: list[bytes] = [] - for i in range(file_info.number_of_blocks): - block = DatStdFileBlockInfos(block_bytes[i * 8 : i * 8 + 8]) - self.file.seek(file_info.offset + file_info.header_size + block.offset) - block_header = DatBlockHeader(self.file.read(16)) - if block_header.dat_block_type == 32000: - data.append(self.file.read(block_header.block_data_size)) - else: - data.append( - zlib.decompress( - self.file.read(block_header.block_data_size), wbits=-15 - ) - ) - - return data - - def __repr__(self): - # type: () -> str - return "Path: {0} Header: {1}".format( - os.path.join(self.root, "sqpack", self.path), self.header - ) From c1feb6366c7cf71c27b33720aebe32dfa3dfbb43 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 14 Oct 2026 13:52:00 +0200 Subject: [PATCH 76/95] fix some stuff for clang compile that could break --- scripts/ghidra/ghidra_wrapper.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/scripts/ghidra/ghidra_wrapper.py b/scripts/ghidra/ghidra_wrapper.py index 1f95c9f23a..011c2cdafe 100644 --- a/scripts/ghidra/ghidra_wrapper.py +++ b/scripts/ghidra/ghidra_wrapper.py @@ -54,11 +54,14 @@ # the fields SrcInterface.get_struct_pad fills the unknown parts of a struct with PAD_FIELD_NAME = re.compile(r"field_[0-9A-F]+") -def first_subfolder(base: Path) -> Path | None: - for p in sorted(base.iterdir()): - if p.is_dir(): - return p - return None +def newest_version_folder(base: Path) -> Path: + """The highest version folder in base, ex. vc/tools/msvc/14.51.36231 or kits/10/include/10.0.26100.0""" + if not base.is_dir(): + raise RuntimeError(f"{base} is missing, select the folder msvc-wine's install.sh was run on") + versions = [p for p in base.iterdir() if p.is_dir() and re.fullmatch(r"[\d.]+", p.name)] + if not versions: + raise RuntimeError(f"No version folder in {base}") + return max(versions, key=lambda p: tuple(int(part) for part in p.name.split(".") if part)) class GhidraInterface(object): @@ -276,20 +279,25 @@ def create_function_def(self, func: DefinedStructFuncField, struct: DefinedStruc def select_parser(self, msvc_path: str | None = None): """ - :param msvc_path: Folder where ./vsdownload.sh of mstorsjo/msvc-wine was run, clang finds the headers itself on windows + :param msvc_path: Folder msvc-wine's install.sh was run on, clang finds the headers itself on windows """ includes = [] if system() != "Windows": if msvc_path is None: raise RuntimeError("Has to have msvc includes for the parsers to work with the project") path = Path(msvc_path) - vc_path = first_subfolder(path / "vc" / "tools" / "msvc") / "include" - window_kit = first_subfolder(path / "kits" / "10" / "include") - includes = [f"-I{folder.as_posix()}" for folder in (vc_path, window_kit / "ucrt", window_kit / "um", window_kit / "shared")] + vc_path = newest_version_folder(path / "vc" / "tools" / "msvc") / "include" + window_kit = newest_version_folder(path / "kits" / "10" / "include") + # as system headers like clang adds them on windows, otherwise the warnings it treats as errors in them count + for folder in (vc_path, window_kit / "ucrt", window_kit / "um", window_kit / "shared"): + includes += ["-isystem", folder.as_posix()] self.parser_args = [ "-x", "c++", "-target", "x86_64-pc-windows-msvc", "-fms-compatibility", "-fms-extensions", - "-U__int64", "-std=c++17", "-ferror-limit=0", *includes, + "-U__int64", "-std=c++17", "-ferror-limit=0", + # the libclang package is older than what newer msvc headers expect + "-D_ALLOW_COMPILER_AND_STL_VERSION_MISMATCH", "-D_MSVC_STL_USE_ABORT_AS_DOOM_FUNCTION", + *includes, ] def parse_string(self, decl: str, vtables: list[str]) -> int: From fffa4df20bb72a409fc77a2bf8bfda5272aba849 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 14 Oct 2026 13:52:45 +0200 Subject: [PATCH 77/95] make vtable generation be a nested structure --- scripts/shared/src_wrapper.py | 118 ++++++++++++++++------------------ 1 file changed, 55 insertions(+), 63 deletions(-) diff --git a/scripts/shared/src_wrapper.py b/scripts/shared/src_wrapper.py index 6f31b412b0..68aa6e7eb5 100644 --- a/scripts/shared/src_wrapper.py +++ b/scripts/shared/src_wrapper.py @@ -665,7 +665,7 @@ def collect_inline_enum_definitions(struct: DefinedStruct): def collect_vtable_names(struct: DefinedStruct): # a template vtable only exists in IDA as its instantiations if struct.virtual_functions and not struct.template_types: - vtable_name = f"{struct.type}_vtbl" + vtable_name = f"{struct.type}::VTable" if vtable_name not in seen_vtable_names: seen_vtable_names.add(vtable_name) vtable_names.append(vtable_name) @@ -673,7 +673,7 @@ def collect_vtable_names(struct: DefinedStruct): collect_vtable_names(inline_struct) for struct in ordered_structs: - forward, define, vtable, vtable_full_name = self.build_struct_string( + forward, define = self.build_struct_string( struct, struct_lookup, enum_lookup, @@ -688,7 +688,6 @@ def collect_vtable_names(struct: DefinedStruct): collect_inline_enum_definitions(struct) forward_chunks.append((struct.namespace, forward)) definition_chunks.append((struct.namespace, define)) - definition_chunks.append((struct.namespace, vtable)) def forward_chunk_key(key: tuple[str, IndentWriter]) -> str: (namespace, _) = key @@ -779,23 +778,19 @@ def build_struct_string( declspec_align: int = 0, struct_definition_lookup: dict[str, DefinedStruct] | None = None, opaque_pointer_types: set[str] | None = None, - ) -> tuple[IndentWriter, IndentWriter, IndentWriter, str | None]: - """Returns (forward_declarations, struct_definition, vtable_definition, vtable_full_name or None).""" + ) -> tuple[IndentWriter, IndentWriter]: + """Returns (forward_declarations, struct_definition), the vtable is defined inside the struct as VTable.""" struct_string_forward: IndentWriter = IndentWriter() struct_string_define: IndentWriter = IndentWriter() - struct_string_vtable: IndentWriter = IndentWriter() template_struct_lookup = template_struct_lookup or {} opaque_pointer_types = opaque_pointer_types or set() template_declaration = "" - template_suffix = "" if struct.template_types: template_declaration = ( f"template" ) - template_suffix = f"<{', '.join(struct.template_types)}>" short_name = self.get_short_name(struct.type) - vtable_full_name: str | None = None if declspec_align > 0: declspec = f"__declspec(align({declspec_align})) " @@ -805,11 +800,6 @@ def build_struct_string( if template_declaration: struct_string_forward += template_declaration struct_string_forward += f"struct {struct.name};" - - if struct.virtual_functions: - if template_declaration: - struct_string_forward += template_declaration - struct_string_forward += f"struct {short_name}_vtbl;" if template_declaration: struct_string_define += template_declaration @@ -866,15 +856,23 @@ def append_chunk(chunk: IndentWriter | str): for inline_struct in struct.inline_structs ] - for inline_forward, _, _, _ in inline_struct_chunks: + for inline_forward, _ in inline_struct_chunks: append_chunk(inline_forward) - for _, inline_define, inline_vtable, _ in inline_struct_chunks: + for _, inline_define in inline_struct_chunks: append_chunk(inline_define) - append_chunk(inline_vtable) - if struct.virtual_functions and prev_size == 0: - struct_string_define += f"{short_name}_vtbl{template_suffix}* __vtable;" - prev_size = 8 + if struct.virtual_functions: + struct_string_define += "struct VTable {" + struct_string_define.indent() + for vfunc_line in self.build_vtable_lines(struct): + struct_string_define += vfunc_line + struct_string_define.unindent() + # with a base at offset 0 the pointer comes from the base + if prev_size == 0: + struct_string_define += "} *__vtable;" + prev_size = 8 + else: + struct_string_define += "};" field_name_counts: dict[str, int] = {} field_name_last_indices: dict[str, int] = {} @@ -963,51 +961,45 @@ def append_chunk(chunk: IndentWriter | str): struct_string_define.unindent() struct_string_define += "};" - if struct.virtual_functions: - if template_declaration: - struct_string_vtable += template_declaration - struct_string_vtable += f"struct {short_name}_vtbl {{" - struct_string_vtable.indent() - previous_offset = 0 - vfunc_name_counts: dict[str, int] = {} - vfunc_preferred_indices: dict[str, int] = {} - for vfunc_index, vfunc in enumerate(struct.virtual_functions): - vfunc_name_counts[vfunc.name] = vfunc_name_counts.get(vfunc.name, 0) + 1 - preferred_index = vfunc_preferred_indices.get(vfunc.name) - if preferred_index is None or ( - vfunc.return_type - and vfunc.parameters - and not ( - struct.virtual_functions[preferred_index].return_type - and struct.virtual_functions[preferred_index].parameters - ) - ): - vfunc_preferred_indices[vfunc.name] = vfunc_index - - emitted_vfunc_names = set() - for vfunc_index, vfunc in enumerate(struct.virtual_functions): - while previous_offset < vfunc.offset: - struct_string_vtable += f"void* vf{int(previous_offset/8)};" - previous_offset += 8 - - vfunc_name = vfunc.name - if ( - vfunc_name_counts[vfunc.name] > 1 - and vfunc_preferred_indices[vfunc.name] != vfunc_index - ): - vfunc_name = f"vf{vfunc.offset // 8}" - while vfunc_name in emitted_vfunc_names: - vfunc_name += "_" - emitted_vfunc_names.add(vfunc_name) - struct_string_vtable += self.build_vfunc_signature( - vfunc, vfunc_name + return (struct_string_forward, struct_string_define) + + def build_vtable_lines(self, struct: DefinedStruct) -> list[str]: + """One line per vtable slot, the gaps filled with void* vfN.""" + lines: list[str] = [] + previous_offset = 0 + vfunc_name_counts: dict[str, int] = {} + vfunc_preferred_indices: dict[str, int] = {} + for vfunc_index, vfunc in enumerate(struct.virtual_functions): + vfunc_name_counts[vfunc.name] = vfunc_name_counts.get(vfunc.name, 0) + 1 + preferred_index = vfunc_preferred_indices.get(vfunc.name) + if preferred_index is None or ( + vfunc.return_type + and vfunc.parameters + and not ( + struct.virtual_functions[preferred_index].return_type + and struct.virtual_functions[preferred_index].parameters ) - previous_offset = vfunc.offset + 8 - struct_string_vtable.unindent() - struct_string_vtable += "};" - vtable_full_name = f"{struct.type}_vtbl" + ): + vfunc_preferred_indices[vfunc.name] = vfunc_index - return (struct_string_forward, struct_string_define, struct_string_vtable, vtable_full_name) + emitted_vfunc_names = set() + for vfunc_index, vfunc in enumerate(struct.virtual_functions): + while previous_offset < vfunc.offset: + lines.append(f"void* vf{int(previous_offset/8)};") + previous_offset += 8 + + vfunc_name = vfunc.name + if ( + vfunc_name_counts[vfunc.name] > 1 + and vfunc_preferred_indices[vfunc.name] != vfunc_index + ): + vfunc_name = f"vf{vfunc.offset // 8}" + while vfunc_name in emitted_vfunc_names: + vfunc_name += "_" + emitted_vfunc_names.add(vfunc_name) + lines.append(self.build_vfunc_signature(vfunc, vfunc_name)) + previous_offset = vfunc.offset + 8 + return lines def get_predef_header(self): return """#define _HAS_ITERATOR_DEBUGGING 0 From 7a6644403b0ebc99504a19cd3829e256dc0ced5f Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 14 Oct 2026 13:56:29 +0200 Subject: [PATCH 78/95] make temp dir only live while the script is executing --- scripts/shared/lumina_wrapper.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/scripts/shared/lumina_wrapper.py b/scripts/shared/lumina_wrapper.py index 2848970ed8..6d08bbc9fb 100644 --- a/scripts/shared/lumina_wrapper.py +++ b/scripts/shared/lumina_wrapper.py @@ -13,10 +13,6 @@ from dataclasses import dataclass import gc -tempdirname = join(getcwd(), "luminatmp") -rmtree(tempdirname, ignore_errors=True) -makedirs(tempdirname, exist_ok=True) - default_game_path = r"C:\Program Files (x86)\SquareEnix\FINAL FANTASY XIV - A Realm Reborn" steam_install_folder = "FINAL FANTASY XIV Online" @@ -177,7 +173,7 @@ def __repr__(self): return self.get_base_type_string() -def get_nugpkg_and_extract(package: str, version: str = ""): +def get_nugpkg_and_extract(tempdirname: str, package: str, version: str = ""): with urlopen( f"https://www.nuget.org/api/v2/package/{package}/{version}" ) as response: @@ -199,10 +195,10 @@ def get_nugpkg_deps(nuspec_path: str) -> list[tuple[str, str]]: return deps -def get_nugpkg(package: str, version: str = ""): +def get_nugpkg(tempdirname: str, package: str, version: str = ""): ver_str = f"version {version}" if version != "" else "latest version" print(f"Requesting nugpkg {package} with {ver_str}") - get_nugpkg_and_extract(package, version) + get_nugpkg_and_extract(tempdirname, package, version) return get_nugpkg_deps(join(tempdirname, f"{package}.nuspec")) @@ -210,18 +206,20 @@ def get_excel_header_files(game_path: str) -> dict[str, tuple[int, list[ExcelCol """ :param game_path: The game install, the folder holding both game and boot, see find_game_path """ + tempdirname = join(getcwd(), "luminatmp") + makedirs(tempdirname, exist_ok=True) dotnetload("coreclr") def _isolated_clr(): import clr - deps = get_nugpkg("Lumina") + deps = get_nugpkg(tempdirname, "Lumina") deps_gotten: dict[str, str] = {} while len(deps) > 0: local_deps = [] for package, version in deps: if package not in deps_gotten or deps_gotten[package] != version: - local_deps.extend(get_nugpkg(package, version)) + local_deps.extend(get_nugpkg(tempdirname, package, version)) deps_gotten[package] = version deps = local_deps @@ -286,4 +284,8 @@ def _isolated_extract(context): return excel_header_files # pythonnet.unload() would break every later use of clr in the same process - return _isolated_clr() + excel_header_files = _isolated_clr() + + rmtree(tempdirname, ignore_errors=True) + + return excel_header_files From 63fcc16fabc393772fe006e8ae2f8f1e1791f70e Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 14 Oct 2026 13:57:31 +0200 Subject: [PATCH 79/95] update some readme stuffs --- Ghidra/Getting Started.md | 6 +++--- scripts/README.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Ghidra/Getting Started.md b/Ghidra/Getting Started.md index 9d2ae194c5..843a19c3dd 100644 --- a/Ghidra/Getting Started.md +++ b/Ghidra/Getting Started.md @@ -51,8 +51,8 @@ Windows: `%AppData%\ghidra\ghidra_VERSION\venv`
Linux: `/home/user/.config/ghidra/ghidra_VERSION/venv` - Execute the following in your terminal, using Python from the same environment as PyGhidra:
-venv: `.../venv/bin/python -m pip install pyyaml==6.0.3 anytree==2.13.0 pythonnet libclang`
-system: `python -m pip install pyyaml==6.0.3 anytree==2.13.0 pythonnet libclang`
+venv: `.../venv/bin/python -m pip install pyyaml anytree pythonnet libclang`
+system: `python -m pip install pyyaml anytree pythonnet libclang`
`pythonnet` is only used by struct_import.py to read the Excel headers with Lumina, it needs the [.NET 10 runtime](https://dotnet.microsoft.com/download/dotnet/10.0) installed, the same version FFXIVClientStructs uses.
`libclang` is only used by struct_import.py to work out the layout of the structs. Outside of Windows it also needs the MSVC and Windows SDK headers, download them with [mstorsjo/msvc-wine](https://github.com/mstorsjo/msvc-wine)'s `vsdownload.sh`.
The `libclang` package ships libclang 18 while the MSVC headers want clang 20 or newer, set the `LIBCLANG_LIBRARY_FILE` environment variable to a libclang 20 or newer (ex. `/usr/lib/llvm-20/lib/libclang-20.so.1`) to avoid the errors this reports.
@@ -64,7 +64,7 @@ struct_import.py looks for the game in the [XIVLauncher](https://goatcorp.github - Click the plus to add a directory

-- Add both `FFXIVClientStructs/scripts/ghidra` (for rename.py, struct_import.py and full_run.py) and `FFXIVClientStructs/Ghidra/scripts` (for SigScanner) +- Add `FFXIVClientStructs/scripts/ghidra` (for rename.py, struct_import.py and full_run.py) After adding both, select `__UserScripts` and you should see all of the scripts. SigScanner should already have a key binding assigned of `Ctrl-Alt-S`. Assign any others you wish, and ensure the checkbox is ticked.

diff --git a/scripts/README.md b/scripts/README.md index 4349a0785c..e206831281 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -25,9 +25,9 @@ Additionally, you will need to place "idauser.cfg" from this repository in "%App #### Ghidra dependency installation: See [Getting Started](../Ghidra/Getting%20Started.md) for information on how to set up PyGhidra. The same packages as IDA are needed in the PyGhidra environment, plus `libclang`. -Outside of Windows the MSVC and Windows SDK headers are needed as well, `struct_import.py` asks for the folder [mstorsjo/msvc-wine](https://github.com/mstorsjo/msvc-wine)'s `vsdownload.sh` was run in. -The `libclang` python package ships libclang 18, the MSVC headers want clang 20 or newer and report errors otherwise (the layouts still come out the same). -Set the `LIBCLANG_LIBRARY_FILE` environment variable to a libclang 20 or newer to avoid those, ex. `/usr/lib/llvm-20/lib/libclang-20.so.1`. +Outside of Windows the MSVC and Windows SDK headers are needed as well, `struct_import.py` asks for the folder [mstorsjo/msvc-wine](https://github.com/mstorsjo/msvc-wine)'s `vsdownload.py` downloaded to and `install.sh` was then run on, it picks the newest MSVC and Windows SDK version in it. +The `libclang` python package ships libclang 18 while newer MSVC headers want clang 20 or newer, the Ghidra scripts define `_ALLOW_COMPILER_AND_STL_VERSION_MISMATCH` and `_MSVC_STL_USE_ABORT_AS_DOOM_FUNCTION` so they parse anyway. +The `LIBCLANG_LIBRARY_FILE` environment variable can point to another libclang, ex. `/usr/lib/llvm-20/lib/libclang-20.so.1`. ## ffxiv_sigmaker.py > [!WARNING] From 6145a2f09e379dda814886e35292550cd1b16e6e Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 14 Oct 2026 14:01:24 +0200 Subject: [PATCH 80/95] update string for msvc-wine selection --- scripts/ghidra/struct_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ghidra/struct_import.py b/scripts/ghidra/struct_import.py index 2ea195f9d5..00a153e502 100644 --- a/scripts/ghidra/struct_import.py +++ b/scripts/ghidra/struct_import.py @@ -46,7 +46,7 @@ msvc_path = None if sys.platform != "win32": msvc_path = str(askDirectory( # noqa - "Select msvc, the folder where ./vsdownload.sh of mstorsjo/msvc-wine was run", "Select" + "Select msvc, the folder mstorsjo/msvc-wine install.sh was run on", "Select" )) ghidra.select_parser(msvc_path) From e63ade24a93961330229a0fc3b50cfe83c133366 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:13:09 +0200 Subject: [PATCH 81/95] have a version cache for excel headers --- .gitignore | 1 + scripts/ghidra/struct_import.py | 20 ++++++----- scripts/ida/struct_import.py | 40 +++++++++++----------- scripts/shared/lumina_wrapper.py | 57 ++++++++++++++++++++++++++++++-- scripts/shared/yaml_loader.py | 7 +++- 5 files changed, 94 insertions(+), 31 deletions(-) diff --git a/.gitignore b/.gitignore index 3af96880f5..84273833bb 100644 --- a/.gitignore +++ b/.gitignore @@ -436,5 +436,6 @@ FodyWeavers.xsd # test folders scripts/luminatmp/ +scripts/excel_headers/ tests/ luminatmp/ \ No newline at end of file diff --git a/scripts/ghidra/struct_import.py b/scripts/ghidra/struct_import.py index 00a153e502..488947c8d3 100644 --- a/scripts/ghidra/struct_import.py +++ b/scripts/ghidra/struct_import.py @@ -50,14 +50,16 @@ )) ghidra.select_parser(msvc_path) -game_path = find_game_path() -if game_path is None: - game_path = str(askDirectory( # noqa - "Select the game install, the folder holding both game and boot", "Select" - )) - if not is_game_path(game_path): - raise RuntimeError(f"No game or boot folder in {game_path}, select the folder holding both") -print(f"Using game install at {game_path}") +def ask_game_path() -> str: + game_path = find_game_path() + if game_path is None: + game_path = str(askDirectory( # noqa + "Select the game install, the folder holding both game and boot", "Select" + )) + if not is_game_path(game_path): + raise RuntimeError(f"No game or boot folder in {game_path}, select the folder holding both") + print(f"Using game install at {game_path}") + return game_path function_types = {"virtual": "Virtual functions", "member": "Member functions"} try: @@ -76,7 +78,7 @@ print("Grabbing ExdSchema and Lumina") -excel_header_files = get_excel_header_files(game_path) +excel_header_files = get_excel_header_files(ghidra.data_yaml.version, ask_game_path) exdschema = get_exdschema_data("latest") excel_structs, excel_map = create_struct_from_header_and_schema( diff --git a/scripts/ida/struct_import.py b/scripts/ida/struct_import.py index 2b4e609aa4..5fd8a55866 100644 --- a/scripts/ida/struct_import.py +++ b/scripts/ida/struct_import.py @@ -37,24 +37,26 @@ ida.select_parser() -game_path = find_game_path() -if game_path is None: - game_fields = [ - {"type": "label", "text": "The game install was not found through XLCore, XIVLauncher or the default folder."}, - {"type": "space"}, - {"type": "folder", "key": "game_path", "label": "Folder holding both game and boot"}, - ] - form = DynamicForm(game_fields) - form.Compile() - try: - if form.Execute() != 1: - raise RuntimeError("No game install selected") - game_path = form.results()["game_path"] - finally: - form.Free() - if not is_game_path(game_path): - raise RuntimeError(f"No game or boot folder in {game_path}, select the folder holding both") -print(f"Using game install at {game_path}") +def ask_game_path() -> str: + game_path = find_game_path() + if game_path is None: + game_fields = [ + {"type": "label", "text": "The game install was not found through XLCore, XIVLauncher or the default folder."}, + {"type": "space"}, + {"type": "folder", "key": "game_path", "label": "Folder holding both game and boot"}, + ] + form = DynamicForm(game_fields) + form.Compile() + try: + if form.Execute() != 1: + raise RuntimeError("No game install selected") + game_path = form.results()["game_path"] + finally: + form.Free() + if not is_game_path(game_path): + raise RuntimeError(f"No game or boot folder in {game_path}, select the folder holding both") + print(f"Using game install at {game_path}") + return game_path fields = [ {"type": "label", "text": "Select function types to update"}, @@ -95,7 +97,7 @@ print("Grabbing ExdSchema and Lumina") -excel_header_files = get_excel_header_files(game_path) +excel_header_files = get_excel_header_files(ida.data_yaml.version, ask_game_path) exdschema = get_exdschema_data("latest") excel_structs, excel_map = create_struct_from_header_and_schema( diff --git a/scripts/shared/lumina_wrapper.py b/scripts/shared/lumina_wrapper.py index 6d08bbc9fb..ca4d8ad898 100644 --- a/scripts/shared/lumina_wrapper.py +++ b/scripts/shared/lumina_wrapper.py @@ -1,13 +1,16 @@ from urllib.request import urlopen from zipfile import ZipFile from io import BytesIO -from os.path import join, basename, abspath, isdir, isfile +from os.path import join, basename, abspath, dirname, isdir, isfile from os import getenv, getcwd, makedirs, listdir from sys import path from shutil import rmtree from xml.dom import pulldom from pythonnet import load as dotnetload from json import load as jsonload +from yaml import load as yamlload, dump as yamldump +from yaml_loader import Loader, Dumper +from typing import Callable from enum import IntEnum import re from dataclasses import dataclass @@ -15,6 +18,8 @@ default_game_path = r"C:\Program Files (x86)\SquareEnix\FINAL FANTASY XIV - A Realm Reborn" steam_install_folder = "FINAL FANTASY XIV Online" +# one file per data.yml version, so Lumina only reads the game once for each version +excel_header_folder = abspath(join(dirname(__file__), "..", "excel_headers")) def is_game_path(game_path: str | None) -> bool: @@ -202,7 +207,55 @@ def get_nugpkg(tempdirname: str, package: str, version: str = ""): return get_nugpkg_deps(join(tempdirname, f"{package}.nuspec")) -def get_excel_header_files(game_path: str) -> dict[str, tuple[int, list[ExcelColumnDefinition]]]: +def get_excel_header_file_path(version: str) -> str: + return join(excel_header_folder, f"{version}.yml") + + +def _load_excel_header_files(file_path: str) -> dict[str, tuple[int, list[ExcelColumnDefinition]]]: + with open(file_path, "r", encoding="utf-8") as f: + excel_header_files = yamlload(f, Loader) + return { + name: (id, [ExcelColumnDefinition(type, offset) for type, offset in columns]) + for name, (id, columns) in excel_header_files.items() + } + + +def _save_excel_header_files(file_path: str, excel_header_files: dict[str, tuple[int, list[ExcelColumnDefinition]]]): + makedirs(excel_header_folder, exist_ok=True) + with open(file_path, "w", encoding="utf-8") as f: + # Sheet: [id, [[type, offset], ...]] + yamldump( + { + name: [id, [[int(column.type), column.offset] for column in columns]] + for name, (id, columns) in excel_header_files.items() + }, + f, Dumper, default_flow_style=None, sort_keys=False, width=1 << 16, + ) + + +def get_excel_header_files( + version: str | None, get_game_path: Callable[[], str] +) -> dict[str, tuple[int, list[ExcelColumnDefinition]]]: + """ + The file saved for the version is used if there is one, otherwise Lumina reads the game and the result is saved. + :param version: The version of data.yml, the headers are read from the game every time if None + :param get_game_path: Returns the game install, the folder holding both game and boot, only called when Lumina has to read the game + """ + if not version: + return _read_excel_header_files(get_game_path()) + + file_path = get_excel_header_file_path(version) + if isfile(file_path): + print(f"Using Excel headers from {file_path}") + return _load_excel_header_files(file_path) + + excel_header_files = _read_excel_header_files(get_game_path()) + _save_excel_header_files(file_path, excel_header_files) + print(f"Saved Excel headers to {file_path}") + return excel_header_files + + +def _read_excel_header_files(game_path: str) -> dict[str, tuple[int, list[ExcelColumnDefinition]]]: """ :param game_path: The game install, the folder holding both game and boot, see find_game_path """ diff --git a/scripts/shared/yaml_loader.py b/scripts/shared/yaml_loader.py index 13f42660ee..1cf7bac66a 100644 --- a/scripts/shared/yaml_loader.py +++ b/scripts/shared/yaml_loader.py @@ -1,4 +1,4 @@ -from yaml import load, SafeLoader +from yaml import load, SafeLoader, SafeDumper try: from yaml import CSafeLoader @@ -9,3 +9,8 @@ Loader = CSafeLoader except Exception: Loader = SafeLoader + +try: + from yaml import CSafeDumper as Dumper +except ImportError: + Dumper = SafeDumper From 3e7fb977fd45ad13f748a51ded2c73062c2fbd46 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:21:31 +0200 Subject: [PATCH 82/95] trim function comments to be more on point --- scripts/ghidra/clang_wrapper.py | 20 ++----- scripts/ghidra/ghidra_wrapper.py | 49 +++++----------- scripts/ghidra/rename.py | 2 +- scripts/ida/form_builder.py | 2 +- scripts/ida/ida_wrapper.py | 20 +++---- scripts/ida/rename.py | 2 +- scripts/shared/class_factory.py | 96 +++++++------------------------- scripts/shared/lumina_wrapper.py | 19 ++----- scripts/shared/src_wrapper.py | 6 +- scripts/shared/structs_schema.py | 2 +- 10 files changed, 58 insertions(+), 160 deletions(-) diff --git a/scripts/ghidra/clang_wrapper.py b/scripts/ghidra/clang_wrapper.py index df162772ed..f634267b48 100644 --- a/scripts/ghidra/clang_wrapper.py +++ b/scripts/ghidra/clang_wrapper.py @@ -50,7 +50,7 @@ def split_type_path(name: str) -> list[str]: - """Split a type on its namespaces, the ones inside template arguments are left alone.""" + """Split a type on its namespaces, not the ones in template arguments.""" parts, depth, start, index = [], 0, 0, 0 while index < len(name): if name[index] == "<": @@ -83,7 +83,7 @@ def __init__(self): self._deferred: list[tuple[Type, Cursor, str]] = [] def collect(self, cursor: Cursor): - """Everything declared in the parsed string, and whatever that needs from the headers it includes.""" + """Everything declared in the string and what it needs from the headers.""" self._collect_declarations(cursor) while self._deferred: type, decl, name = self._deferred.pop() @@ -130,11 +130,7 @@ def get_decl_name(self, decl: Cursor) -> str: return name def get_type_string(self, type: Type, follow: bool, by_value: bool) -> str | None: - """ - :param follow: Lay out the records this type points to, they are only declared by name otherwise - :param by_value: The type is held and not pointed to, so its layout is needed - :return: None if the type can not be described - """ + """The type as structs.yml spells it, None if it can not be described.""" type = type.get_canonical() if type.kind in primitive_types: return primitive_types[type.kind] @@ -267,19 +263,13 @@ def get_field(self, field: Cursor, offset: int, follow: bool): class ClangInterface(object): def __init__(self, args: list[str], library_file: str | None = None): - """ - :param args: Arguments for clang, the same ones the compiler would get - :param library_file: Path of the libclang to use, the one of the python package if omitted - """ + """Uses the libclang of the python package unless library_file is given.""" if library_file is not None and not Config.loaded: Config.set_library_file(library_file) self.args = args def parse_string(self, decl: str) -> tuple[DefinedStructExport, dict[str, str], int]: - """ - Let clang work out the layout of every struct, template instances included. - :return: The structs and enums, the typedefs for templated types, the number of parser errors - """ + """The layout of every struct, the template typedefs and the error count.""" index = Index.create() forced: list[str] = [] for _ in range(max_instantiation_passes): diff --git a/scripts/ghidra/ghidra_wrapper.py b/scripts/ghidra/ghidra_wrapper.py index 011c2cdafe..0155444308 100644 --- a/scripts/ghidra/ghidra_wrapper.py +++ b/scripts/ghidra/ghidra_wrapper.py @@ -55,7 +55,7 @@ PAD_FIELD_NAME = re.compile(r"field_[0-9A-F]+") def newest_version_folder(base: Path) -> Path: - """The highest version folder in base, ex. vc/tools/msvc/14.51.36231 or kits/10/include/10.0.26100.0""" + """The highest version folder in base.""" if not base.is_dir(): raise RuntimeError(f"{base} is missing, select the folder msvc-wine's install.sh was run on") versions = [p for p in base.iterdir() if p.is_dir() and re.fullmatch(r"[\d.]+", p.name)] @@ -73,10 +73,7 @@ class GhidraInterface(object): template_aliases: dict[str, str] = {} def __init__(self, program, monitor): - """ - :param program: currentProgram of the running script - :param monitor: monitor of the running script - """ + """Takes currentProgram and monitor of the running script.""" # the script globals (currentProgram, getFunctionAt, ...) only exist in the script itself self.program = program self.monitor = monitor @@ -141,7 +138,7 @@ def check_cancelled(self): # region types def get_category_path(self, type_name: str): - """Category a type lives in, ex. Component::GUI::AtkResNode is AtkResNode in /Component/GUI.""" + """The category a type lives in, ex. /Component/GUI.""" category = CategoryPath.ROOT for part in split_type_path(type_name)[:-1]: category = CategoryPath(category, part) @@ -195,7 +192,7 @@ def get_struct_id(self, name: str): return self.get_named_type(name) def create_datatype(self, datatype): - """Add the type to the program, an existing one with the same path gets its content replaced.""" + """Add the type, replacing the one with the same path.""" old = self.dtm.getDataType(datatype.getDataTypePath()) if old is not None: old.replaceWith(datatype) @@ -217,7 +214,7 @@ def create_enum(self, enum: DefinedStructEnum): self.create_datatype(dt) def create_struct(self, struct: DefinedStruct): - """Create the struct without its members so every struct can be pointed to.""" + """The struct without its members, so it can be pointed to.""" category, name = self.get_category_path(struct.type), self.get_type_name(struct.type) if struct.union: dt = UnionDataType(category, name) @@ -265,7 +262,7 @@ def create_struct_members(self, struct: DefinedStruct): dt.replaceAtOffset(field.offset, ft, -1, field.name, "") def create_function_def(self, func: DefinedStructFuncField, struct: DefinedStruct): - """Signature of a function pointer field, ex. an entry of a vtable.""" + """Signature of a function pointer field.""" fd = FunctionDefinitionDataType( CategoryPath(self.get_category_path(struct.type), self.get_type_name(struct.type)), func.name ) @@ -278,9 +275,7 @@ def create_function_def(self, func: DefinedStructFuncField, struct: DefinedStruc return fd def select_parser(self, msvc_path: str | None = None): - """ - :param msvc_path: Folder msvc-wine's install.sh was run on, clang finds the headers itself on windows - """ + """msvc_path is where msvc-wine was installed, not needed on windows.""" includes = [] if system() != "Windows": if msvc_path is None: @@ -301,10 +296,7 @@ def select_parser(self, msvc_path: str | None = None): ] def parse_string(self, decl: str, vtables: list[str]) -> int: - """ - Ghidra can not parse C++, so clang lays out the declarations and the result is imported as data types. - :return: The number of parser errors reported by clang - """ + """Import the declarations as clang lays them out, returns the error count.""" export, self.template_aliases, error_count = ClangInterface( self.parser_args, os.environ.get("LIBCLANG_LIBRARY_FILE") ).parse_string(decl) @@ -323,10 +315,7 @@ def parse_string(self, decl: str, vtables: list[str]) -> int: return error_count def import_structs(self, export: DefinedStructExport) -> int: - """ - :param export: Structs in dependency order with their size and field offsets known - :return: The number of types that failed to import - """ + """Import the structs in dependency order, returns the failure count.""" steps = [ ("Creating enums", export.enums, self.create_enum), ("Creating structs", export.structs, self.create_struct), @@ -422,9 +411,7 @@ def get_vfunc_ea_from_data(self, class_name: str, offset: int) -> int: return self.BADADDR def set_func_name(self, ea, name, cmt=None, unique=False): - """ - :param unique: Only name functions Ghidra has not named yet, a number is appended if the name is taken by another address - """ + """With unique only default names are replaced, a taken name gets a number.""" if unique: if not self.has_default_name(ea): return @@ -532,7 +519,7 @@ def update_virt_func(self, virt_func: DefinedStructVFunc, struct: DefinedStruct) self.apply_func_type(ea, func_name, virt_func.return_type, virt_func.parameters) def get_static_member_ea(self, static_member: DefinedStructStaticMember) -> int: - """Address the static member's signature resolves to, BADADDR if the signature is not found.""" + """Where the static member's signature resolves to, BADADDR if not found.""" ea = self.search_binary(0, static_member.signature) if ea == self.BADADDR: return ea @@ -640,7 +627,7 @@ def get_string(self, ea: int) -> str | None: # region instructions def heads_before(self, ea: int, count: int): - """The instructions in front of ea, closest first, ex. the argument setup of a call.""" + """The instructions before ea, closest first.""" insn = self.listing.getInstructionContaining(self.to_addr(ea)) for _ in range(count): insn = insn.getPrevious() @@ -659,11 +646,7 @@ def heads_until_call(self, ea: int, func_name: str): insn = insn.getNext() def get_registers(self, eas, mnemonics=None) -> dict[str, dict[str, object]]: - """ - Collect what the instructions write to each register, keyed by lower case register name. - :param eas: Instruction addresses to look at, later ones win - :param mnemonics: Only look at these instructions, all of them if omitted - """ + """What the instructions write to each register, later ones win.""" regs = {} for ea in eas: insn = self.listing.getInstructionAt(self.to_addr(ea)) @@ -739,16 +722,14 @@ def format_func_name(self, ea: int, current_func_name: str, proposed_func_name: # region search def get_search_range(self): - """The signatures are all code, so only .text is looked at when the program has one.""" + """Only .text if the program has one, the signatures are all code.""" block = self.memory.getBlock(".text") if block is not None: return block.getStart(), block.getEnd() return self.memory.getMinAddress(), self.memory.getMaxAddress() def search_binary(self, ea: int, pattern: str) -> int: - """ - :param pattern: Space separated hex bytes, ? or ?? for a wildcard - """ + """The pattern is space separated hex bytes, ? or ?? for a wildcard.""" start, end = self.get_search_range() if ea > end.getOffset(): return self.BADADDR diff --git a/scripts/ghidra/rename.py b/scripts/ghidra/rename.py index 976365132a..22729ed757 100644 --- a/scripts/ghidra/rename.py +++ b/scripts/ghidra/rename.py @@ -21,7 +21,7 @@ lua_namespace = "Client::Game::Event::Lua" def apply_lua_funcs(): - """Relies on the Common::Lua::LuaState names from data.yml, so apply_data has to run first.""" + """Needs the LuaState names, so apply_data has to run first.""" string_eas = ghidra.get_func_xrefs(ghidra.get_func_ea_by_name('Common::Lua::LuaState.SetStringField')) func_eas = ghidra.get_func_xrefs(ghidra.get_func_ea_by_name('Common::Lua::LuaState.SetFunctionField')) diff --git a/scripts/ida/form_builder.py b/scripts/ida/form_builder.py index cce031b70e..d89c6ca370 100644 --- a/scripts/ida/form_builder.py +++ b/scripts/ida/form_builder.py @@ -89,7 +89,7 @@ def _on_change(self, fid): return 1 def results(self): - """Return all values as a plain dict keyed by each field's 'key'.""" + """All values keyed by each field's key.""" out = {} for fld in self.fields: t, key = fld["type"], fld.get("key") diff --git a/scripts/ida/ida_wrapper.py b/scripts/ida/ida_wrapper.py index 61e53bf0a4..f7109e2cc3 100644 --- a/scripts/ida/ida_wrapper.py +++ b/scripts/ida/ida_wrapper.py @@ -168,7 +168,7 @@ def apply_tinfo(self, ea: int, tif) -> bool: return ida_typeinf.apply_tinfo(ea, tif, ida_typeinf.TINFO_DEFINITE) def can_guess_func_type(self, ea: int) -> bool: - """Returns False if IDA can not work out the type of the function, even after decompiling it.""" + """Returns False if IDA can not work out the type, even after decompiling.""" tif, funcdata = ida_typeinf.tinfo_t(), ida_typeinf.func_type_data_t() ida_typeinf.guess_tinfo(tif, ea) @@ -180,7 +180,7 @@ def can_guess_func_type(self, ea: int) -> bool: return tif.get_func_details(funcdata) def allow_template_type_names(self): - """clang puts spaces in template instantiation names, a single rejected name makes it drop every top level declaration.""" + """Allow the spaces clang puts in template names, one rejected name drops every declaration.""" ida_idp.process_config_directive( 'TypeNameChars = "~.<>?!/,*&[]-+ "' '"_:$()`\'{}"' @@ -254,9 +254,7 @@ def get_vfunc_ea_from_data(self, class_name: str, offset: int) -> int: return self.get_qword(vtable_ea + offset) def set_func_name(self, ea, name, cmt=None, unique=False): - """ - :param unique: Only name functions IDA has not named yet, a number is appended if the name is taken by another address - """ + """With unique only default names are replaced, a taken name gets a number.""" if unique: if not idc.get_func_name(ea).startswith("sub_"): return @@ -318,7 +316,7 @@ def get_string_from_ptr(self, ea: int) -> str: return val.decode('utf-8') def heads_before(self, ea: int, count: int): - """The instructions in front of ea, closest first, ex. the argument setup of a call.""" + """The instructions before ea, closest first.""" for _ in range(count): ea = idc.prev_head(ea) yield ea @@ -330,11 +328,7 @@ def heads_until_call(self, ea: int, func_name: str): ea = idc.next_head(ea) def get_registers(self, eas, mnemonics=None) -> dict[str, dict[str, object]]: - """ - Collect what the instructions write to each register, keyed by register name. - :param eas: Instruction addresses to look at, later ones win - :param mnemonics: Only look at these instructions, all of them if omitted - """ + """What the instructions write to each register, later ones win.""" regs = {} for ea in eas: if mnemonics is not None and idc.print_insn_mnem(ea) not in mnemonics: @@ -521,7 +515,7 @@ def update_virt_func(self, virt_func: DefinedStructVFunc, struct: DefinedStruct) self.apply_func_type(ea, func_name, virt_func.return_type, virt_func.parameters) def get_static_member_ea(self, static_member: DefinedStructStaticMember) -> int: - """Address the static member's signature resolves to, BADADDR if the signature is not found.""" + """Where the static member's signature resolves to, BADADDR if not found.""" ea = self.search_binary(0, static_member.signature) if ea == idc.BADADDR: return ea @@ -549,7 +543,7 @@ def update_static_member(self, static_member: DefinedStructStaticMember, struct: ida_name.set_name(ea, f"g_{struct.type}_{static_member.name}") def mark_as_vtable(self, vtbl_name: str) -> str | None: - """Returns an error string on failure, None on success.""" + """Returns an error string on failure.""" tif = ida_typeinf.tinfo_t() if not tif.get_named_type(None, vtbl_name): diff --git a/scripts/ida/rename.py b/scripts/ida/rename.py index c75054c4b5..245f603a0d 100644 --- a/scripts/ida/rename.py +++ b/scripts/ida/rename.py @@ -11,7 +11,7 @@ lua_namespace = "Client::Game::Event::Lua" def apply_lua_funcs(): - """Relies on the Common::Lua::LuaState names from data.yml, so apply_data has to run first.""" + """Needs the LuaState names, so apply_data has to run first.""" string_eas = ida.get_func_xrefs(ida.get_func_ea_by_name('Common::Lua::LuaState.SetStringField')) func_eas = ida.get_func_xrefs(ida.get_func_ea_by_name('Common::Lua::LuaState.SetFunctionField')) diff --git a/scripts/shared/class_factory.py b/scripts/shared/class_factory.py index 9edb193763..e441421570 100644 --- a/scripts/shared/class_factory.py +++ b/scripts/shared/class_factory.py @@ -14,9 +14,7 @@ @dataclass(eq=False) class Vtbl: - """ - Object representing a class's vtbl - """ + """A class's vtbl.""" # Address of vtbl ea: int # Name of the base class this vtbl inherits, if it exists @@ -27,19 +25,13 @@ class Vtbl: @classmethod def from_data(cls, vtbl_data: DefinedDataClassVtable, rebase_offset: int = 0) -> Vtbl: - """ - Create a vtbl from its data.yml definition - :param vtbl_data: Vtbl definition as contained in data.yml - :param rebase_offset: Offset to apply to the address if the program has been rebased - """ + """A vtbl from its data.yml definition, rebased by rebase_offset.""" return cls(ea=vtbl_data.ea + rebase_offset, base_name=vtbl_data.base, size=vtbl_data.size) @dataclass(eq=False, repr=False) class FfxivClass: - """ - Object representing a class - """ + """A class of data.yml.""" # Disassembler api, see BaseApi in ffxiv_idarename.py api: Any name: str @@ -58,12 +50,7 @@ class FfxivClass: @classmethod def from_data(cls, api: Any, class_name: str, class_data: DefinedDataClass) -> FfxivClass: - """ - Create a class from its data.yml definition - :param api: Disassembler api - :param class_name: Class name - :param class_data: Class definition as contained in data.yml - """ + """A class from its data.yml definition.""" # Offset the vtbl and funcs if the program has been rebased rebase_offset = api.get_image_base() - STANDARD_IMAGE_BASE @@ -80,12 +67,7 @@ def from_data(cls, api: Any, class_name: str, class_data: DefinedDataClass) -> F @property def main_vtbl_size(self) -> int: - """ - Iterate from the vtbl start until a non-offset or xref is encountered. - This strategy implies that the only xref in a vtbl is the first vfunc. - Can be overriden by explicit size in the vtbl definition. - :return: VTable func count - """ + """The vfunc count, the data.yml size or up to the first non-offset or xref.""" if not self.vtbls: return self._main_vtbl_size @@ -133,10 +115,7 @@ def recurse_tree(current_node: Node, current_class: FfxivClass, all_bases: set[s @property def finalized(self) -> bool: - """ - Has this class and its hierarchy been written out or not - :return: bool yes/no - """ + """If the class and its hierarchy have been written out.""" if self.vtbls: return self._finalized and all( vtbl.resolved_base is not None and vtbl.resolved_base._finalized for vtbl in self.vtbls) @@ -145,10 +124,7 @@ def finalized(self) -> bool: @finalized.setter def finalized(self, value: bool): - """ - Set if finalized or not - :param value: Finalized state - """ + """Set if the class has been written out.""" self._finalized = value # endregion @@ -156,9 +132,7 @@ def finalized(self, value: bool): # region finalize def finalize(self): - """ - Write out this class - """ + """Write out this class.""" self._inherit_func_names_from_main_base() self._comment_vtbls_with_inheritance_tree() @@ -170,10 +144,7 @@ def finalize(self): self.finalized = True def _inherit_func_names_from_main_base(self): - """ - A base is guaranteed to be finalized before the child, - so a base has all the vfunc names of its base already. - """ + """A base is finalized before its child, so it has the vfunc names of its bases.""" if self.vtbls and self.vtbls[0].resolved_base: for idx, base_vfunc_name in self.vtbls[0].resolved_base.vfuncs.items(): if idx in self.vfuncs: @@ -183,12 +154,7 @@ def _inherit_func_names_from_main_base(self): self.vfuncs[idx] = base_vfunc_name def _comment_vtbls_with_inheritance_tree(self): - """ - Adds the inheritance tree as a comment to the start of the vtbl. - grandbase_name - base_name - self_name - """ + """Comment the inheritance tree at the start of the vtbl.""" for vtbl in self.vtbls: comment = self.api.get_comment(vtbl.ea) or "" @@ -199,18 +165,14 @@ def _comment_vtbls_with_inheritance_tree(self): self.api.set_comment(vtbl.ea, comment) def write_vtbl_names(self): - """ - Write out the vtbl name. - """ + """Write out the vtbl names.""" self.api.set_addr_name(self.vtbls[0].ea, self.api.format_class_name_for_vtbl(self.name)) for vtbl in self.vtbls[1:]: self.api.set_addr_name(vtbl.ea, self.api.format_class_name_for_secondary_vtbl(self.name, vtbl.base_name)) def _write_vtbl_functions(self): - """ - Write out the vtbl function names - """ + """Write out the vtbl function names.""" def collect_vtbl_functions(ea: int, size: int, class_name: str, vfunc_names: dict[int, str], base_class_names: list[str]) -> list[tuple[int, str]]: @@ -255,9 +217,7 @@ def collect_vtbl_functions(ea: int, size: int, class_name: str, vfunc_names: dic self.api.set_addr_name(func_ea, func_name) def _write_funcs(self): - """ - Write the names of all non-vtbl funcs - """ + """Write the names of all non-vtbl funcs.""" for func_ea, proposed_func_name in self.funcs.items(): current_func_name = self.api.get_addr_name(func_ea) # type: str @@ -271,9 +231,7 @@ def _write_funcs(self): self.api.set_addr_name(func_ea, func_name) def _write_instances(self): - """ - Write the names of all instances - """ + """Write the names of all instances.""" for instance in self.instances: name = "g_{}_{}".format(self.name, instance.name or "Instance") self.api.set_addr_name(instance.ea, name) @@ -293,11 +251,7 @@ class FfxivClassFactory: _finalize_stack: deque[FfxivClass] = field(default_factory=deque) def register(self, class_name: str, class_data: DefinedDataClass | None = None): - """ - Register a class - :param class_name: Class name - :param class_data: Class definition as contained in data.yml, omit for a placeholder entry - """ + """Register a class, a placeholder if class_data is omitted.""" if class_data is None: class_data = DefinedDataClass() @@ -315,9 +269,7 @@ def register(self, class_name: str, class_data: DefinedDataClass | None = None): self._classes[class_name] = FfxivClass.from_data(self.api, class_name, class_data) def finalize(self): - """ - Perform the class naming - """ + """Name every class.""" self._resolve_base_classes() # We write the vtbl names first for an added check so that @@ -330,10 +282,7 @@ def finalize(self): self._finalize_class(cls) def _resolve_base_classes(self): - """ - Resolve base classes for a class - If missing, warn the user and add a stub entry. - """ + """Resolve the base classes, missing ones get a stub entry and a warning.""" for cls in list(self._classes.values()): for idx, vtbl in enumerate(cls.vtbls): if vtbl.resolved_base is None and vtbl.base_name: @@ -346,10 +295,7 @@ def _resolve_base_classes(self): cls.first_base = self._classes[vtbl.base_name] def _finalize_class(self, cls: FfxivClass): - """ - Perform a single class naming - :param cls: Class object - """ + """Name a single class.""" if cls in self._finalize_stack: names = [c.name for c in self._finalize_stack] + [cls.name] names = "\n".join([" - {0}".format(name) for name in names]) @@ -366,11 +312,7 @@ def _finalize_class(self, cls: FfxivClass): def apply_data(api: Any, data: DefinedData): - """ - Name everything contained in data.yml - :param api: Disassembler api - :param data: Parsed data.yml - """ + """Name everything in data.yml.""" for section in (data.globals, data.functions): for ea, name in (section or {}).items(): if not isinstance(ea, int): diff --git a/scripts/shared/lumina_wrapper.py b/scripts/shared/lumina_wrapper.py index ca4d8ad898..8dd4411957 100644 --- a/scripts/shared/lumina_wrapper.py +++ b/scripts/shared/lumina_wrapper.py @@ -23,7 +23,7 @@ def is_game_path(game_path: str | None) -> bool: - """The game install holds both the game and the boot folder.""" + """If the folder holds both game and boot.""" return bool(game_path) and isdir(join(game_path, "game", "sqpack")) and isdir(join(game_path, "boot")) @@ -73,7 +73,7 @@ def _get_steam_paths() -> list[str]: def _get_steam_game_paths() -> list[str]: - """The game folder in every Steam library, libraries can be on other drives than Steam itself.""" + """The game folder in every Steam library.""" libraries = [] for steam_path in _get_steam_paths(): libraries.append(steam_path) @@ -88,10 +88,7 @@ def _get_steam_game_paths() -> list[str]: def find_game_path() -> str | None: - """ - Where XLCore or XIVLauncher have the game installed, else the default install folder or a Steam library. - :return: None if none of them hold the game, the scripts ask for the folder then - """ + """The game install from it's known locations, None if none of them hold it.""" for get_paths in ( lambda: [_get_xlcore_game_path()], lambda: [_get_xivlauncher_game_path()], @@ -236,11 +233,7 @@ def _save_excel_header_files(file_path: str, excel_header_files: dict[str, tuple def get_excel_header_files( version: str | None, get_game_path: Callable[[], str] ) -> dict[str, tuple[int, list[ExcelColumnDefinition]]]: - """ - The file saved for the version is used if there is one, otherwise Lumina reads the game and the result is saved. - :param version: The version of data.yml, the headers are read from the game every time if None - :param get_game_path: Returns the game install, the folder holding both game and boot, only called when Lumina has to read the game - """ + """The saved headers of the version, read with Lumina if there are none.""" if not version: return _read_excel_header_files(get_game_path()) @@ -256,9 +249,7 @@ def get_excel_header_files( def _read_excel_header_files(game_path: str) -> dict[str, tuple[int, list[ExcelColumnDefinition]]]: - """ - :param game_path: The game install, the folder holding both game and boot, see find_game_path - """ + """The headers Lumina reads from the game install.""" tempdirname = join(getcwd(), "luminatmp") makedirs(tempdirname, exist_ok=True) dotnetload("coreclr") diff --git a/scripts/shared/src_wrapper.py b/scripts/shared/src_wrapper.py index 68aa6e7eb5..faa0769893 100644 --- a/scripts/shared/src_wrapper.py +++ b/scripts/shared/src_wrapper.py @@ -706,7 +706,7 @@ def forward_chunk_key(key: tuple[str, IndentWriter]) -> str: return (str(output), vtable_names) def _collect_template_aliases(self, structs: list[DefinedStruct]) -> dict[str, str]: - """Map the typedef name (see get_template_alias) to the templated type for every function signature.""" + """The typedef name of every templated type in a function signature.""" aliases: dict[str, str] = {} def collect(type_name: str | None): @@ -779,7 +779,7 @@ def build_struct_string( struct_definition_lookup: dict[str, DefinedStruct] | None = None, opaque_pointer_types: set[str] | None = None, ) -> tuple[IndentWriter, IndentWriter]: - """Returns (forward_declarations, struct_definition), the vtable is defined inside the struct as VTable.""" + """The forward declarations and the definition, the vtable inside as VTable.""" struct_string_forward: IndentWriter = IndentWriter() struct_string_define: IndentWriter = IndentWriter() template_struct_lookup = template_struct_lookup or {} @@ -964,7 +964,7 @@ def append_chunk(chunk: IndentWriter | str): return (struct_string_forward, struct_string_define) def build_vtable_lines(self, struct: DefinedStruct) -> list[str]: - """One line per vtable slot, the gaps filled with void* vfN.""" + """One line per vtable slot.""" lines: list[str] = [] previous_offset = 0 vfunc_name_counts: dict[str, int] = {} diff --git a/scripts/shared/structs_schema.py b/scripts/shared/structs_schema.py index ddd1ecefac..3d839f1a8d 100644 --- a/scripts/shared/structs_schema.py +++ b/scripts/shared/structs_schema.py @@ -101,7 +101,7 @@ class DefinedStructExport: def get_template_alias(type_name: str) -> str: - """Typedef name the exported header declares for a templated type, as IDA spells template names differently than the yml.""" + """The typedef name of a templated type, IDA spells template names differently.""" alias = "".join(char if char.isalnum() else "_" for char in type_name.replace(" ", "").replace("*", "_ptr")) return f"__cs_{alias}" From 6543e50dc47318ee64043d3a82f9991fd6b6ce90 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Thu, 15 Oct 2026 05:53:42 +0200 Subject: [PATCH 83/95] fix some struct pack issues --- scripts/ghidra/clang_wrapper.py | 49 ++++- scripts/ghidra/ghidra_wrapper.py | 6 + scripts/ghidra/struct_import.py | 4 +- scripts/ida/ida_wrapper.py | 24 +++ scripts/ida/struct_import.py | 4 +- scripts/shared/src_wrapper.py | 312 +++++++++++++++++++++++++++++-- 6 files changed, 374 insertions(+), 25 deletions(-) diff --git a/scripts/ghidra/clang_wrapper.py b/scripts/ghidra/clang_wrapper.py index f634267b48..1b7af6580e 100644 --- a/scripts/ghidra/clang_wrapper.py +++ b/scripts/ghidra/clang_wrapper.py @@ -5,6 +5,7 @@ os.environ.setdefault("LIBCLANG_DISABLE_CRASH_RECOVERY", "1") from clang.cindex import Config, Cursor, CursorKind, Index, Type, TypeKind +from src_wrapper import layout_probe_prefix from structs_schema import ( DefinedStruct, DefinedStructEnum, @@ -205,15 +206,40 @@ def add_record(self, type: Type, decl: Cursor, name: str): ) offset += base.get_size() - for field in type.get_fields(): - bit_offset = field.get_field_offsetof() - if field.is_bitfield() or bit_offset < 0 or bit_offset % 8 != 0: + union = decl.kind == CursorKind.UNION_DECL + # libclang validates the whole record for every offset asked for, so it is only asked where alignment moved a field + max_field_align = type.get_align() + end = offset + ask_clang = False + for index, field in enumerate(type.get_fields()): + if field.is_bitfield(): + ask_clang = True continue - defined_field = self.get_field(field, bit_offset // 8, follow) + field_type = field.type.get_canonical() + field_size, field_align = field_type.get_size(), field_type.get_align() + if union: + field_offset = 0 + elif ask_clang or index == 0 or field_size < 0 or field_align <= 0: + bit_offset = field.get_field_offsetof() + if bit_offset < 0 or bit_offset % 8 != 0: + continue + field_offset = bit_offset // 8 + else: + field_align = min(field_align, max_field_align) + field_offset = -(-end // field_align) * field_align + if field_offset != end: + bit_offset = field.get_field_offsetof() + if bit_offset != field_offset * 8: + ask_clang = True + if bit_offset < 0 or bit_offset % 8 != 0: + continue + field_offset = bit_offset // 8 + end = field_offset + max(field_size, 0) + defined_field = self.get_field(field, field_offset, follow) if defined_field is not None: fields.append(defined_field) - self.structs[name] = self.new_struct(name, fields, type.get_size(), decl.kind == CursorKind.UNION_DECL) + self.structs[name] = self.new_struct(name, fields, type.get_size(), union) def is_empty(self, type: Type) -> bool: """An empty base takes up no room in the struct inheriting it.""" @@ -294,3 +320,16 @@ def parse_string(self, decl: str) -> tuple[DefinedStructExport, dict[str, str], print(f"Error: clang: {len(errors) - 50} more error(s) not shown") return collector.get_export(), collector.aliases, len(errors) + + def parse_layout_probe(self, source: str) -> dict[str, tuple[int, int]]: + """Size and offset of value in each probe struct of SrcInterface.get_layout_probe.""" + unit = Index.create().parse(main_file, args=self.args, unsaved_files=[(main_file, source)]) + layouts: dict[str, tuple[int, int]] = {} + for child in unit.cursor.get_children(): + if child.kind != CursorKind.STRUCT_DECL or not child.spelling.startswith(layout_probe_prefix): + continue + for field in child.type.get_fields(): + size, bit_offset = field.type.get_size(), field.get_field_offsetof() + if field.spelling == "value" and size > 0 and bit_offset >= 0: + layouts[child.spelling] = (size, bit_offset // 8) + return layouts diff --git a/scripts/ghidra/ghidra_wrapper.py b/scripts/ghidra/ghidra_wrapper.py index 0155444308..91e14ea411 100644 --- a/scripts/ghidra/ghidra_wrapper.py +++ b/scripts/ghidra/ghidra_wrapper.py @@ -295,6 +295,12 @@ def select_parser(self, msvc_path: str | None = None): *includes, ] + def get_probe_layouts(self, source: str) -> dict[str, tuple[int, int]]: + """Size and offset of value in each probe struct of SrcInterface.get_layout_probe.""" + return ClangInterface( + self.parser_args, os.environ.get("LIBCLANG_LIBRARY_FILE") + ).parse_layout_probe(source) + def parse_string(self, decl: str, vtables: list[str]) -> int: """Import the declarations as clang lays them out, returns the error count.""" export, self.template_aliases, error_count = ClangInterface( diff --git a/scripts/ghidra/struct_import.py b/scripts/ghidra/struct_import.py index 488947c8d3..b4a6688a3b 100644 --- a/scripts/ghidra/struct_import.py +++ b/scripts/ghidra/struct_import.py @@ -87,7 +87,7 @@ def ask_game_path() -> str: exd_map = {excel_map[k]: k for k in excel_map} -excel_header, excel_vtables = src.build_export_string(excel_structs, 4) +excel_header, excel_vtables = src.build_export_string(excel_structs, 4, ghidra.get_probe_layouts) print("Parsing structs.yml") @@ -95,7 +95,7 @@ def ask_game_path() -> str: with open(ghidra.get_struct_path()) as f: import_structs = get_yaml(f) -struct_header, struct_vtables = src.build_export_string(import_structs, 1) +struct_header, struct_vtables = src.build_export_string(import_structs, 1, ghidra.get_probe_layouts, excel_header) print("Parsing combined Excel and FFXIV declarations") ghidra.parse_string( diff --git a/scripts/ida/ida_wrapper.py b/scripts/ida/ida_wrapper.py index f7109e2cc3..463c74c0bb 100644 --- a/scripts/ida/ida_wrapper.py +++ b/scripts/ida/ida_wrapper.py @@ -20,6 +20,7 @@ ) from form_builder import DynamicForm from data_schema import get_yaml, DefinedData, DefinedDataClassVtable +from src_wrapper import layout_probe_prefix from platform import system from pathlib import Path @@ -210,6 +211,29 @@ def select_parser(self): f"-x c++ -target x86_64-pc-windows-msvc -fms-compatibility -fms-extensions -U__int64 -std=c++17{includes}", ) + def get_probe_layouts(self, source: str) -> dict[str, tuple[int, int]]: + """Size and offset of value in each probe struct of SrcInterface.get_layout_probe.""" + # a til of its own keeps the probes out of the database + til = ida_typeinf.new_til("cs_layout_probe.til", "FFXIVClientStructs layout probe") + try: + if ida_srclang.parse_decls_with_parser("clang", til, source, False) < 0: + raise RuntimeError("parse_decls_with_parser failed: the clang parser is not available") + + layouts: dict[str, tuple[int, int]] = {} + name = ida_typeinf.first_named_type(til, ida_typeinf.NTF_TYPE) + while name: + if name.startswith(layout_probe_prefix): + tinfo = ida_typeinf.tinfo_t() + udt = ida_typeinf.udt_type_data_t() + if tinfo.get_named_type(til, name) and tinfo.get_udt_details(udt): + for member in udt: + if member.name == "value" and member.size > 0: + layouts[name] = (member.size // 8, member.offset // 8) + name = ida_typeinf.next_named_type(til, name, ida_typeinf.NTF_TYPE) + return layouts + finally: + ida_typeinf.free_til(til) + def parse_string(self, decl: str, vtables: list[str]) -> int: """Returns the number of parser errors reported by IDA.""" error_count = ida_srclang.parse_decls_with_parser("clang", None, decl, False) diff --git a/scripts/ida/struct_import.py b/scripts/ida/struct_import.py index 5fd8a55866..e4a12871b8 100644 --- a/scripts/ida/struct_import.py +++ b/scripts/ida/struct_import.py @@ -106,7 +106,7 @@ def ask_game_path() -> str: exd_map = {excel_map[k]: k for k in excel_map} -excel_header, excel_vtables = src.build_export_string(excel_structs, 4) +excel_header, excel_vtables = src.build_export_string(excel_structs, 4, ida.get_probe_layouts) print("Parsing structs.yml") @@ -114,7 +114,7 @@ def ask_game_path() -> str: with open(ida.get_struct_path()) as f: import_structs = get_yaml(f) -struct_header, struct_vtables = src.build_export_string(import_structs, 1) +struct_header, struct_vtables = src.build_export_string(import_structs, 1, ida.get_probe_layouts, excel_header) print("Parsing combined Excel and FFXIV declarations") ida.parse_string( diff --git a/scripts/shared/src_wrapper.py b/scripts/shared/src_wrapper.py index faa0769893..ae9b640cf3 100644 --- a/scripts/shared/src_wrapper.py +++ b/scripts/shared/src_wrapper.py @@ -7,11 +7,22 @@ DefinedStructVFunc, get_template_alias, ) -from indent_writer import IndentWriter +from typing import Callable + +from indent_writer import IndentWriter, IndentWriterValue + +# a probe struct per type the schema can not size, the offset of value is the alignment of the type +layout_probe_prefix = "__cs_layout_probe_" class SrcInterface(object): full_padding: bool = True + def __init__(self): + self._layout_cache: dict[tuple[str, tuple[str, ...]], tuple[int, bool]] = {} + self._external_layouts: dict[str, tuple[int, int]] = {} + # inline structs of a template use its parameters, ex. the Node of StdLinkedList + self._inherited_template_types: dict[str, tuple[str, ...]] = {} + def get_type_and_type_size_from_gap(self, gap: int, offset: int = 0) -> tuple[str, int]: if offset == 0: offset = gap @@ -182,15 +193,18 @@ def get_size_from_type( return 0 normalized_type = self._normalize_type(type_name) + # the short name lookup of the structs can match an unrelated struct + if normalized_type in enum_lookup: + return enum_lookup[normalized_type] struct_size = self._get_struct_size(normalized_type, struct_lookup) if struct_size is not None: return struct_size - if normalized_type in enum_lookup: - return enum_lookup[normalized_type] primitive_size = self.get_size_from_string(type_name) if primitive_size: return primitive_size + if normalized_type.removeprefix("::") in self._external_layouts: + return self._external_layouts[normalized_type.removeprefix("::")][0] base_type, template_args = self._split_template_type(type_name) if not template_args: @@ -283,11 +297,29 @@ def _get_type_alignment( return 8 normalized_type = self._normalize_type(type_name) + base_type, template_args = self._split_template_type(normalized_type.removeprefix("::")) + if template_args: + struct_definition = (template_struct_lookup or {}).get(base_type) + else: + struct_definition = (struct_definition_lookup or {}).get(base_type) + if base_type in self._external_layouts and not template_args: + return self._external_layouts[base_type][1] + if struct_definition is not None and len(template_args) == len(struct_definition.template_types): + alignment, packed = self._get_struct_layout( + struct_definition, + template_args, + struct_lookup, + enum_lookup, + template_struct_lookup, + struct_definition_lookup, + ) + return 1 if packed else alignment + struct_size = self._get_struct_size(normalized_type, struct_lookup) - if struct_size is not None: - size = struct_size - elif normalized_type in enum_lookup: + if normalized_type in enum_lookup: size = enum_lookup[normalized_type] + elif struct_size is not None: + size = struct_size else: size = self.get_size_from_string(type_name) if size == 0: @@ -316,6 +348,111 @@ def _get_type_alignment( ) return min(size & -size, 8) if size else 1 + def _get_template_parameters(self, struct: DefinedStruct) -> tuple[str, ...]: + """The template parameters of the struct and of the templates it is declared in.""" + return ( + *self._inherited_template_types.get(self._normalize_type(struct.type), ()), + *struct.template_types, + ) + + def _is_template_value_field(self, field: DefinedStructField, template_parameters: tuple[str, ...]) -> bool: + """A template parameter held by value, its size depends on the arguments.""" + return not field.type.strip().endswith("*") and field.type.strip() in template_parameters + + def _has_template_value_field(self, struct: DefinedStruct) -> bool: + """The size of the struct then depends on the arguments.""" + template_parameters = self._get_template_parameters(struct) + return any(self._is_template_value_field(field, template_parameters) for field in struct.fields) + + def _get_pad_alignment(self, prev_size: int, end: int) -> int: + """The alignment of the filler fields get_struct_pad writes from prev_size up to end.""" + alignment = 1 + while end > prev_size: + _, size = self.get_struct_pad(end - prev_size, prev_size) + if self.full_padding: + alignment = max(alignment, size) + prev_size += size + return alignment + + def _get_struct_layout( + self, + struct: DefinedStruct, + template_args: list[str], + struct_lookup: dict[str, int], + enum_lookup: dict[str, int], + template_struct_lookup: dict[str, DefinedStruct] | None = None, + struct_definition_lookup: dict[str, DefinedStruct] | None = None, + ) -> tuple[int, bool]: + """The natural alignment of the struct, and if it has to be packed to keep its field offsets and size.""" + cache_key = (self._normalize_type(struct.type), tuple(template_args)) + if cache_key in self._layout_cache: + return self._layout_cache[cache_key] + + substitutions = dict(zip(struct.template_types, template_args)) + has_base_at_zero = any( + field.base and field.offset == 0 for field in struct.fields + ) + alignment = 8 if struct.virtual_functions and not has_base_at_zero else 1 + packed = False + template_parameters = self._get_template_parameters(struct) + track_padding = not template_parameters and not struct.union + prev_size = 0 + base_types_used = set() + for field in struct.fields: + if field.base and field.offset == prev_size: + base_types_used.add(field.type) + prev_size += self.get_size_from_type( + self._get_field_type(field, template_struct_lookup or {}), + struct_lookup, + enum_lookup, + template_struct_lookup, + template_parameters, + struct_definition_lookup, + ) + if struct.virtual_functions and prev_size == 0: + prev_size = 8 + for field in struct.fields: + if field.type == "__fastcall": + field_alignment = field_size = 8 + else: + field_type = self._get_field_type(field, template_struct_lookup or {}) + field_type = self._substitute_template_types(field_type, substitutions) + field_alignment = self._get_type_alignment( + field_type, + struct_lookup, + enum_lookup, + template_struct_lookup, + struct_definition_lookup, + ) + field_size = 0 + if track_padding and field.type not in base_types_used: + field_size = self.get_size_from_type( + field_type, + struct_lookup, + enum_lookup, + template_struct_lookup, + template_parameters, + struct_definition_lookup, + ) + if isinstance(field, DefinedStructFixedField): + field_size *= field.size + alignment = max(alignment, field_alignment) + if field.offset % field_alignment != 0: + packed = True + if track_padding and field.type not in base_types_used: + if field.offset > prev_size: + alignment = max(alignment, self._get_pad_alignment(prev_size, field.offset)) + prev_size = field.offset + prev_size += field_size + if track_padding and struct.size: + alignment = max(alignment, self._get_pad_alignment(prev_size, struct.size)) + if struct.size and struct.size % alignment != 0: + packed = True + if template_parameters: + packed = False + self._layout_cache[cache_key] = (alignment, packed) + return alignment, packed + @staticmethod def _align_size(size: int, alignment: int) -> int: return (size + alignment - 1) // alignment * alignment @@ -334,7 +471,7 @@ def _get_template_struct_size( f"Template type {struct.type} expects {len(struct.template_types)} " f"arguments, got {len(template_args)}" ) - if struct.size is not None: + if struct.size is not None and not self._has_template_value_field(struct): return struct.size substitutions = dict(zip(struct.template_types, template_args)) @@ -466,9 +603,15 @@ def build_vfunc_signature( def build_export_string( self, export: DefinedStructExport, - full_declspec_align: int = 0 + full_declspec_align: int = 0, + layout_parser: Callable[[str], dict[str, tuple[int, int]]] | None = None, + layout_header: str = "", ): + """layout_parser measures the types the schema can not size, see get_layout_probe, layout_header is what it needs declared.""" output = IndentWriter() + self._layout_cache = {} + self._external_layouts = {} + self._inherited_template_types = {} all_structs: list[DefinedStruct] = [] inline_structs: list[DefinedStruct] = [] @@ -487,6 +630,16 @@ def collect_inline_types(struct: DefinedStruct): all_structs.append(struct) collect_inline_types(struct) + def collect_inherited_template_types(struct: DefinedStruct, template_parameters: tuple[str, ...]): + template_parameters = (*template_parameters, *struct.template_types) + for inline_struct in struct.inline_structs: + if template_parameters: + self._inherited_template_types[self._normalize_type(inline_struct.type)] = template_parameters + collect_inherited_template_types(inline_struct, template_parameters) + + for struct in export.structs: + collect_inherited_template_types(struct, ()) + inline_struct_types = { self._normalize_type(struct.type) for struct in inline_structs } @@ -527,6 +680,22 @@ def collect_inline_type_owners(struct: DefinedStruct, owner_type: str): if struct.template_types: template_struct_lookup[normalized_type] = struct + self._external_layouts = self.resolve_external_layouts( + self._collect_external_types( + [ + struct + for struct in export.structs + if self._normalize_type(struct.type) not in inline_struct_types + ], + struct_lookup, + enum_lookup, + template_struct_lookup, + struct_definition_lookup, + ), + layout_parser, + layout_header, + ) + top_level_structs = [ struct for struct in export.structs @@ -705,6 +874,88 @@ def forward_chunk_key(key: tuple[str, IndentWriter]) -> str: return (str(output), vtable_names) + def _collect_external_types( + self, + structs: list[DefinedStruct], + struct_lookup: dict[str, int], + enum_lookup: dict[str, int], + template_struct_lookup: dict[str, DefinedStruct], + struct_definition_lookup: dict[str, DefinedStruct], + ) -> list[str]: + """The types held by value that are not in the schema, ex. GUID or HANDLE from the windows headers.""" + type_names: set[str] = set() + + def collect(type_name: str, template_parameters: tuple[str, ...]): + type_name = type_name.strip() + if type_name.endswith("*") or type_name == "__fastcall": + return + _, template_args = self._split_template_type(type_name) + if template_args: + for template_arg in template_args: + collect(template_arg, template_parameters) + return + type_name = self._normalize_type(type_name).removeprefix("::") + if type_name in template_parameters or self.is_base_type(type_name): + return + try: + size = self.get_size_from_type( + type_name, + struct_lookup, + enum_lookup, + template_struct_lookup, + template_parameters, + struct_definition_lookup, + ) + except ValueError: + return + if size == 0: + type_names.add(type_name) + + def collect_struct(struct: DefinedStruct, template_parameters: tuple[str, ...]): + template_parameters = (*template_parameters, *struct.template_types) + for field in struct.fields: + collect(self._get_field_type(field, template_struct_lookup), template_parameters) + for inline_struct in struct.inline_structs: + collect_struct(inline_struct, template_parameters) + + for struct in structs: + collect_struct(struct, ()) + return sorted(type_names) + + def get_layout_probe(self, type_names: list[str], header: str = "") -> str: + """A probe struct per type, a layout parser returns {probe name: (size of value, offset of value)} for it.""" + probes = "\n".join( + f"struct {layout_probe_prefix}{index} {{ char pad; ::{type_name} value; }};" + for index, type_name in enumerate(type_names) + ) + return f"{self.get_predef_header()}\n{header}\n{probes}\n" + + def resolve_external_layouts( + self, + type_names: list[str], + layout_parser: Callable[[str], dict[str, tuple[int, int]]] | None, + layout_header: str = "", + ) -> dict[str, tuple[int, int]]: + """Size and alignment of the types, measured by the parser of the disassembler.""" + if not type_names: + return {} + if layout_parser is None: + print(f"Warning: no layout parser, {len(type_names)} type(s) can not be sized: {', '.join(type_names)}") + return {} + + probe_layouts = layout_parser(self.get_layout_probe(type_names, layout_header)) + layouts: dict[str, tuple[int, int]] = {} + missing: list[str] = [] + for index, type_name in enumerate(type_names): + probe_layout = probe_layouts.get(f"{layout_probe_prefix}{index}") + if probe_layout is None or probe_layout[0] <= 0 or probe_layout[1] <= 0: + missing.append(type_name) + else: + layouts[type_name] = probe_layout + if missing: + print(f"Warning: the parser could not size {len(missing)} type(s): {', '.join(missing)}") + return layouts + def _collect_template_aliases(self, structs: list[DefinedStruct]) -> dict[str, str]: """The typedef name of every templated type in a function signature.""" aliases: dict[str, str] = {} @@ -791,19 +1042,22 @@ def build_struct_string( ) short_name = self.get_short_name(struct.type) + template_parameters = self._get_template_parameters(struct) - if declspec_align > 0: + # pragma pack can not lower a declspec align, even align(1) + if declspec_align > 1: declspec = f"__declspec(align({declspec_align})) " else: declspec = "" if template_declaration: struct_string_forward += template_declaration - struct_string_forward += f"struct {struct.name};" + record_keyword = "union" if struct.union else "struct" + struct_string_forward += f"{record_keyword} {struct.name};" if template_declaration: struct_string_define += template_declaration - struct_define_line = f"struct {declspec}{short_name}" + struct_define_line = f"{record_keyword} {declspec}{short_name}" struct_extend_lines = [] base_fields = [field for field in struct.fields if field.base] @@ -819,7 +1073,7 @@ def build_struct_string( struct_lookup, enum_lookup, template_struct_lookup, - tuple(struct.template_types), + template_parameters, struct_definition_lookup, ) @@ -895,14 +1149,28 @@ def append_chunk(chunk: IndentWriter | str): emitted_field_names.add(field_name) offset = 0 + union_size = 0 + size_known = True + packed = not template_parameters and self._get_struct_layout( + struct, + [], + struct_lookup, + enum_lookup, + template_struct_lookup, + struct_definition_lookup, + )[1] for field_index, field in enumerate(struct.fields): if field.type in base_fields_used: continue + if struct.union: + union_size = max(union_size, prev_size) + prev_size = 0 + offset = field.offset - if not struct.is_sequential: + if (not struct.is_sequential or packed) and size_known: while offset > prev_size: fill_size = offset - prev_size line, size = self.get_struct_pad(fill_size, prev_size) @@ -918,7 +1186,7 @@ def append_chunk(chunk: IndentWriter | str): struct_lookup, enum_lookup, template_struct_lookup, - struct.template_types, + list(template_parameters), struct_definition_lookup, field_name, opaque_pointer_types, @@ -943,15 +1211,23 @@ def append_chunk(chunk: IndentWriter | str): struct_lookup, enum_lookup, template_struct_lookup, - struct.template_types, + list(template_parameters), struct_definition_lookup, field_name, opaque_pointer_types, ) struct_string_define += field_define prev_size += field_define_size + if self._is_template_value_field(field, template_parameters): + size_known = False + + if struct.union: + prev_size = max(union_size, prev_size) + if struct.size and struct.size > prev_size: + struct_string_define += f"uint8_t field_0[{struct.size}];" + prev_size = struct.size - if struct.size is not None and struct.size != 0: + if struct.size is not None and struct.size != 0 and not self._has_template_value_field(struct): while struct.size > prev_size: fill_size = struct.size - prev_size line, size = self.get_struct_pad(fill_size, prev_size) @@ -961,6 +1237,10 @@ def append_chunk(chunk: IndentWriter | str): struct_string_define.unindent() struct_string_define += "};" + if packed: + struct_string_define.lines.insert(0, IndentWriterValue(0, "#pragma pack(push, 1)")) + struct_string_define += "#pragma pack(pop)" + return (struct_string_forward, struct_string_define) def build_vtable_lines(self, struct: DefinedStruct) -> list[str]: From 040cb4a9af31782def2aa5701c33da999ff46165 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Thu, 15 Oct 2026 22:52:24 +0200 Subject: [PATCH 84/95] fix path.join not working due to python execute --- scripts/ida/full_run.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/ida/full_run.py b/scripts/ida/full_run.py index 57de045963..c3293e5496 100644 --- a/scripts/ida/full_run.py +++ b/scripts/ida/full_run.py @@ -1,15 +1,15 @@ -from os import path +import os import idautils import ida_auto import idc -working_directory = path.dirname(path.realpath(__file__)) +working_directory = os.path.dirname(os.path.realpath(__file__)) print("Loading rename.py") -exec(open(path.join(working_directory, "rename.py")).read()) +exec(open(os.path.join(working_directory, "rename.py")).read()) print("Loading struct_import.py") -exec(open(path.join(working_directory, "struct_import.py")).read()) +exec(open(os.path.join(working_directory, "struct_import.py")).read()) for s in idautils.Segments(): start = idc.get_segm_start(s) From 62ae4ec29b4393ea3a4f15218c776473b07a4422 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Thu, 15 Oct 2026 22:54:38 +0200 Subject: [PATCH 85/95] fix path.join in ghidra for full run --- scripts/ghidra/full_run.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/ghidra/full_run.py b/scripts/ghidra/full_run.py index 16a97189bf..5b4949d33c 100644 --- a/scripts/ghidra/full_run.py +++ b/scripts/ghidra/full_run.py @@ -3,17 +3,17 @@ # @menupath Tools.Scripts.ffxiv_full_run # @runtime PyGhidra -from os import path +import os try: - working_directory = path.dirname(path.realpath(__file__)) + working_directory = os.path.dirname(os.path.realpath(__file__)) except NameError: - working_directory = path.dirname(str(sourceFile)) # noqa + working_directory = os.path.dirname(str(sourceFile)) # noqa print("Loading rename.py") -exec(open(path.join(working_directory, "rename.py")).read()) +exec(open(os.path.join(working_directory, "rename.py")).read()) print("Loading struct_import.py") -exec(open(path.join(working_directory, "struct_import.py")).read()) +exec(open(os.path.join(working_directory, "struct_import.py")).read()) analyzeChanges(currentProgram) # noqa From 9be0bf88a11952aad4782a497aae72bb4f78e786 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Fri, 16 Oct 2026 00:02:27 +0200 Subject: [PATCH 86/95] Ghidra inheritance needs a bit of a different flow than IDA --- scripts/ghidra/clang_wrapper.py | 50 +++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/scripts/ghidra/clang_wrapper.py b/scripts/ghidra/clang_wrapper.py index 1b7af6580e..53ae143024 100644 --- a/scripts/ghidra/clang_wrapper.py +++ b/scripts/ghidra/clang_wrapper.py @@ -1,5 +1,6 @@ import os import re +from dataclasses import replace # the crash recovery of libclang takes the SIGSEGV the JVM raises on purpose and kills Ghidra, read when the index is created os.environ.setdefault("LIBCLANG_DISABLE_CRASH_RECOVERY", "1") @@ -109,6 +110,7 @@ def _collect_declarations(self, cursor: Cursor): self.aliases[child.spelling] = type_string def get_export(self) -> DefinedStructExport: + self.flatten_primary_bases() opaque = [ self.new_struct(name, [], None, union) for name, union in self.opaque.items() @@ -116,6 +118,54 @@ def get_export(self) -> DefinedStructExport: ] return DefinedStructExport(list(self.enums.values()), [*opaque, *self.structs.values()]) + def get_primary_base(self, struct: DefinedStruct) -> DefinedStruct | None: + """The base at offset 0, it shares its vtable pointer with the struct inheriting it.""" + if struct.union: + return None + base_field = next((field for field in struct.fields if field.base and field.offset == 0), None) + if base_field is None: + return None + base = self.structs.get(base_field.type) + return None if base is None or base.union else base + + def flatten_primary_bases(self): + """Ghidra can not retype the vtable pointer inside an embedded base, so a struct with a vtable of its own + gets the members of its offset 0 bases with __vtable pointing to its own VTable instead.""" + + def get_primary_fields(struct: DefinedStruct) -> list: + base = self.get_primary_base(struct) + if base is None: + return list(struct.fields) + fields = [field for field in struct.fields if not (field.base and field.offset == 0)] + return [*get_primary_fields(base), *fields] + + flattened: dict[str, list] = {} + for name, struct in self.structs.items(): + vtable_name = f"{name}::VTable" + if vtable_name not in self.structs or self.get_primary_base(struct) is None: + continue + primary_fields = get_primary_fields(struct) + vtable_field = next((field for field in primary_fields if field.offset == 0 and field.name == "__vtable"), None) + if vtable_field is None: + # the root base has no vtable pointer to replace + continue + + fields = [DefinedStructField("__vtable", f"{vtable_name}*", 0, False, [])] + names = {"__vtable"} + # the most derived first, a member hides the one of a base with the same name + for field in reversed(primary_fields): + if field is vtable_field: + continue + if field.name in names: + field = replace(field, name=f"{field.name}_{field.offset:X}") + names.add(field.name) + fields.append(field) + flattened[name] = sorted(fields, key=lambda field: field.offset) + + # replaced after the loop, so every struct is flattened from its bases as clang laid them out + for name, fields in flattened.items(): + self.structs[name] = replace(self.structs[name], fields=fields) + def new_struct(self, type_name: str, fields: list, size: int | None, union: bool) -> DefinedStruct: parts = split_type_path(type_name) return DefinedStruct( From dbc4f81811c063c5c43dafc8d15019ad03c1d79a Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Fri, 16 Oct 2026 00:05:03 +0200 Subject: [PATCH 87/95] clean up some comments --- scripts/ghidra/ghidra_wrapper.py | 3 +-- scripts/ghidra/struct_import.py | 1 - scripts/ida/ida_wrapper.py | 4 +--- scripts/shared/class_factory.py | 4 +--- scripts/shared/lumina_wrapper.py | 1 - 5 files changed, 3 insertions(+), 10 deletions(-) diff --git a/scripts/ghidra/ghidra_wrapper.py b/scripts/ghidra/ghidra_wrapper.py index 91e14ea411..00f008d1dd 100644 --- a/scripts/ghidra/ghidra_wrapper.py +++ b/scripts/ghidra/ghidra_wrapper.py @@ -499,7 +499,6 @@ def update_virt_func(self, virt_func: DefinedStructVFunc, struct: DefinedStruct) ea = self.get_func_ea_by_name(func_name) if ea == self.BADADDR: if self.get_vft_from_data(struct.type) is None: - # no vtable address in data.yml, so there is no slot to look up return ea = self.get_vfunc_ea_from_data(struct.type, virt_func.offset) if ea in (0, self.BADADDR): @@ -687,7 +686,7 @@ def format_class_name(self, class_name: str) -> str: return class_name def format_vfunc_name(self, ea: int, current_func_name: str, proposed_func_name: str, class_name: str, base_class_names: list[str]) -> str | None: - current_func_name = current_func_name.removeprefix("thunk_") # jump + current_func_name = current_func_name.removeprefix("thunk_") # Previously renamed as a vfunc if current_func_name.startswith(class_name): diff --git a/scripts/ghidra/struct_import.py b/scripts/ghidra/struct_import.py index b4a6688a3b..3f455df626 100644 --- a/scripts/ghidra/struct_import.py +++ b/scripts/ghidra/struct_import.py @@ -117,7 +117,6 @@ def do_pattern(pattern, suffix, sheetSearchPattern, sheetSearchOffset): ) sheetIdx = ghidra.get_dword(sheetEa) - # don't rename any funcs that are already named if ghidra.has_default_name(ea): if exd_map.get(sheetIdx) == None: print( diff --git a/scripts/ida/ida_wrapper.py b/scripts/ida/ida_wrapper.py index 463c74c0bb..cf2e735dd2 100644 --- a/scripts/ida/ida_wrapper.py +++ b/scripts/ida/ida_wrapper.py @@ -306,8 +306,7 @@ def get_qword(self, ea: int) -> int: def get_addr_name(self, ea: int) -> str: name = idc.get_name(ea) - # if name is None or '' and segment name is .text then create the function and get the name of it. - # this is done for parts of the code that has no xrefs to it due to how IDA detects function boundaries + # code with no xrefs to it is not made a function due to how IDA detects function boundaries if (name is None or name == '') and idc.get_segm_name(ea) == '.text': ida_funcs.add_func(ea) name = idc.get_name(ea) @@ -508,7 +507,6 @@ def update_virt_func(self, virt_func: DefinedStructVFunc, struct: DefinedStruct) ea = self.get_func_ea_by_name(func_name) if ea == idc.BADADDR: if self.get_vft_from_data(struct.type) is None: - # no vtable address in data.yml, so there is no slot to look up return ea = self.get_vfunc_ea_from_data(struct.type, virt_func.offset) if ea in (0, idc.BADADDR): diff --git a/scripts/shared/class_factory.py b/scripts/shared/class_factory.py index e441421570..df6eeaab06 100644 --- a/scripts/shared/class_factory.py +++ b/scripts/shared/class_factory.py @@ -51,7 +51,6 @@ class FfxivClass: @classmethod def from_data(cls, api: Any, class_name: str, class_data: DefinedDataClass) -> FfxivClass: """A class from its data.yml definition.""" - # Offset the vtbl and funcs if the program has been rebased rebase_offset = api.get_image_base() - STANDARD_IMAGE_BASE return cls( @@ -76,7 +75,7 @@ def main_vtbl_size(self) -> int: return self._main_vtbl_size if self._main_vtbl_size == 0: - self._main_vtbl_size = 1 # Set to 1, skip the first entry + self._main_vtbl_size = 1 # skip the first entry for ea in itertools.count(self.vtbls[0].ea + 8, 8): if self.api.get_addr_name(ea) != '': break @@ -177,7 +176,6 @@ def _write_vtbl_functions(self): def collect_vtbl_functions(ea: int, size: int, class_name: str, vfunc_names: dict[int, str], base_class_names: list[str]) -> list[tuple[int, str]]: vtbl_builder = [] - # Iterate through each offset for func_idx in range(0, size): vtbl_vfunc_ea = ea + func_idx * 8 vfunc_ea = self.api.get_qword(vtbl_vfunc_ea) # type: int diff --git a/scripts/shared/lumina_wrapper.py b/scripts/shared/lumina_wrapper.py index 8dd4411957..498e9e616f 100644 --- a/scripts/shared/lumina_wrapper.py +++ b/scripts/shared/lumina_wrapper.py @@ -220,7 +220,6 @@ def _load_excel_header_files(file_path: str) -> dict[str, tuple[int, list[ExcelC def _save_excel_header_files(file_path: str, excel_header_files: dict[str, tuple[int, list[ExcelColumnDefinition]]]): makedirs(excel_header_folder, exist_ok=True) with open(file_path, "w", encoding="utf-8") as f: - # Sheet: [id, [[type, offset], ...]] yamldump( { name: [id, [[int(column.type), column.offset] for column in columns]] From 0b6e8cf27cc56e68a94f67d42e24cbea8602e676 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Sun, 18 Oct 2026 19:52:35 +0200 Subject: [PATCH 88/95] change exd naming for ghidra to use correct naming for function call IDA converts the first naming of _ to . for functions while ghidra doesn't --- scripts/ghidra/struct_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ghidra/struct_import.py b/scripts/ghidra/struct_import.py index 3f455df626..308cd90927 100644 --- a/scripts/ghidra/struct_import.py +++ b/scripts/ghidra/struct_import.py @@ -129,7 +129,7 @@ def do_pattern(pattern, suffix, sheetSearchPattern, sheetSearchOffset): if suffix == None: suffix = "" - fnName = "Component::Exd::ExdModule_Get%s%s" % ( + fnName = "Component::Exd::ExdModule.Get%s%s" % ( exd_map[sheetIdx], suffix, ) From f5dac723660f531afe8a09610286080134976e01 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Mon, 19 Oct 2026 00:32:18 +0200 Subject: [PATCH 89/95] fix union field types being discarded --- scripts/shared/src_wrapper.py | 131 +++++++++++++++++++++++-------- scripts/shared/structs_schema.py | 8 ++ 2 files changed, 105 insertions(+), 34 deletions(-) diff --git a/scripts/shared/src_wrapper.py b/scripts/shared/src_wrapper.py index ae9b640cf3..0b20c1cdd7 100644 --- a/scripts/shared/src_wrapper.py +++ b/scripts/shared/src_wrapper.py @@ -348,6 +348,17 @@ def _get_type_alignment( ) return min(size & -size, 8) if size else 1 + def _get_union_members( + self, field: DefinedStructField | DefinedStructFixedField + ) -> list[DefinedStructField]: + """The members of a union field as fields at its offset, empty when the field is not a union.""" + if isinstance(field, DefinedStructFixedField) or field.type == "__fastcall": + return [] + return [ + DefinedStructField(union_value.name, union_value.type, field.offset, False, []) + for union_value in field.union_values + ] + def _get_template_parameters(self, struct: DefinedStruct) -> tuple[str, ...]: """The template parameters of the struct and of the templates it is declared in.""" return ( @@ -415,25 +426,37 @@ def _get_struct_layout( if field.type == "__fastcall": field_alignment = field_size = 8 else: - field_type = self._get_field_type(field, template_struct_lookup or {}) - field_type = self._substitute_template_types(field_type, substitutions) - field_alignment = self._get_type_alignment( - field_type, - struct_lookup, - enum_lookup, - template_struct_lookup, - struct_definition_lookup, - ) - field_size = 0 - if track_padding and field.type not in base_types_used: - field_size = self.get_size_from_type( + # a union is as large and as aligned as its largest and most aligned member + field_types = [ + self._substitute_template_types( + self._get_field_type(member, template_struct_lookup or {}), + substitutions, + ) + for member in self._get_union_members(field) or [field] + ] + field_alignment = max( + self._get_type_alignment( field_type, struct_lookup, enum_lookup, template_struct_lookup, - template_parameters, struct_definition_lookup, ) + for field_type in field_types + ) + field_size = 0 + if track_padding and field.type not in base_types_used: + field_size = max( + self.get_size_from_type( + field_type, + struct_lookup, + enum_lookup, + template_struct_lookup, + template_parameters, + struct_definition_lookup, + ) + for field_type in field_types + ) if isinstance(field, DefinedStructFixedField): field_size *= field.size alignment = max(alignment, field_alignment) @@ -486,26 +509,33 @@ def _get_template_struct_size( field_size = 8 field_alignment = 8 else: - field_type = self._get_field_type(field, template_struct_lookup) - field_type = self._substitute_template_types(field_type, substitutions) - field_size = self.get_size_from_type( - field_type, - struct_lookup, - enum_lookup, - template_struct_lookup, - struct_definition_lookup=struct_definition_lookup, - ) - if field_size == 0: - raise ValueError( - f"Cannot determine size of {field_type} in template type {struct.type}" + field_size = 0 + field_alignment = 1 + for member in self._get_union_members(field) or [field]: + field_type = self._get_field_type(member, template_struct_lookup) + field_type = self._substitute_template_types(field_type, substitutions) + member_size = self.get_size_from_type( + field_type, + struct_lookup, + enum_lookup, + template_struct_lookup, + struct_definition_lookup=struct_definition_lookup, + ) + if member_size == 0: + raise ValueError( + f"Cannot determine size of {field_type} in template type {struct.type}" + ) + field_size = max(field_size, member_size) + field_alignment = max( + field_alignment, + self._get_type_alignment( + field_type, + struct_lookup, + enum_lookup, + template_struct_lookup, + struct_definition_lookup, + ), ) - field_alignment = self._get_type_alignment( - field_type, - struct_lookup, - enum_lookup, - template_struct_lookup, - struct_definition_lookup, - ) if isinstance(field, DefinedStructFixedField): field_size *= field.size @@ -719,7 +749,11 @@ def iter_struct_tree(struct: DefinedStruct): struct_type = self._normalize_type(struct.type) opaque_pointer_types = set() for nested_struct in iter_struct_tree(struct): - for field in nested_struct.fields: + for field in [ + member + for nested_field in nested_struct.fields + for member in [nested_field, *self._get_union_members(nested_field)] + ]: field_type = self._normalize_type(field.type) pointer_base = field_type has_pointer = pointer_base.endswith("*") @@ -747,6 +781,11 @@ def iter_struct_tree(struct: DefinedStruct): for struct in all_structs: identifiers.add(self.get_short_name(struct.type)) identifiers.update(field.name for field in struct.fields) + identifiers.update( + union_value.name + for field in struct.fields + for union_value in field.union_values + ) identifiers.update( virtual_function.name for virtual_function in struct.virtual_functions or [] @@ -914,7 +953,8 @@ def collect(type_name: str, template_parameters: tuple[str, ...]): def collect_struct(struct: DefinedStruct, template_parameters: tuple[str, ...]): template_parameters = (*template_parameters, *struct.template_types) for field in struct.fields: - collect(self._get_field_type(field, template_struct_lookup), template_parameters) + for member in self._get_union_members(field) or [field]: + collect(self._get_field_type(member, template_struct_lookup), template_parameters) for inline_struct in struct.inline_structs: collect_struct(inline_struct, template_parameters) @@ -1204,6 +1244,29 @@ def append_chunk(chunk: IndentWriter | str): fastcall_field_define = fastcall_field_define[:-2] struct_string_define += fastcall_field_define + ");" prev_size += 8 + elif union_members := self._get_union_members(field): + struct_string_define += "union {" + struct_string_define.indent() + union_define_size = 0 + for member in union_members: + member_define, member_define_size = self.get_string_define_and_size( + member, + struct.namespace, + struct_lookup, + enum_lookup, + template_struct_lookup, + list(template_parameters), + struct_definition_lookup, + None, + opaque_pointer_types, + ) + struct_string_define += member_define + union_define_size = max(union_define_size, member_define_size) + if self._is_template_value_field(member, template_parameters): + size_known = False + struct_string_define.unindent() + struct_string_define += f"}} {field_name};" + prev_size += union_define_size else: field_define, field_define_size = self.get_string_define_and_size( field, diff --git a/scripts/shared/structs_schema.py b/scripts/shared/structs_schema.py index 3d839f1a8d..4d7f695c44 100644 --- a/scripts/shared/structs_schema.py +++ b/scripts/shared/structs_schema.py @@ -52,6 +52,7 @@ class DefinedStructField(DefinedStructFuncParam): offset: int base: bool template_args: list[str] + union_values: list[DefinedStructFuncParam] @dataclass @@ -137,9 +138,15 @@ def parse_struct(struct: YamlStructExport) -> DefinedStruct: for field in struct["fields"]: base = field["base"] if "base" in field else False template_args = [] + union_values = [] if "template_args" in field: for template_arg in field["template_args"]: template_args.append(template_arg) + if "union_values" in field: + for union_value in field["union_values"]: + union_values.append( + DefinedStructFuncParam(union_value["name"], union_value["type"]) + ) if "size" in field: fields.append( DefinedStructFixedField( @@ -177,6 +184,7 @@ def parse_struct(struct: YamlStructExport) -> DefinedStruct: field["offset"], base, template_args, + union_values, ) ) if "virtual_functions" in struct: From bb4955f6520e2d969f5a2bf90c50da4a2bda3b39 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Mon, 19 Oct 2026 00:52:18 +0200 Subject: [PATCH 90/95] fix some calls being wrong due to union field type fix --- scripts/ghidra/clang_wrapper.py | 10 +++++----- scripts/shared/exdschema_wrapper.py | 8 ++++++-- scripts/shared/src_wrapper.py | 2 +- scripts/shared/structs_schema.py | 2 ++ 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/scripts/ghidra/clang_wrapper.py b/scripts/ghidra/clang_wrapper.py index 53ae143024..d00fb07289 100644 --- a/scripts/ghidra/clang_wrapper.py +++ b/scripts/ghidra/clang_wrapper.py @@ -150,7 +150,7 @@ def get_primary_fields(struct: DefinedStruct) -> list: # the root base has no vtable pointer to replace continue - fields = [DefinedStructField("__vtable", f"{vtable_name}*", 0, False, [])] + fields = [DefinedStructField("__vtable", f"{vtable_name}*", 0, False, [], [])] names = {"__vtable"} # the most derived first, a member hides the one of a base with the same name for field in reversed(primary_fields): @@ -252,7 +252,7 @@ def add_record(self, type: Type, decl: Cursor, name: str): continue offset = -(-offset // base.get_align()) * base.get_align() fields.append( - DefinedStructField(f"baseclass_{offset:X}", self.get_type_string(base, follow, True), offset, True, []) + DefinedStructField(f"baseclass_{offset:X}", self.get_type_string(base, follow, True), offset, True, [], []) ) offset += base.get_size() @@ -323,7 +323,7 @@ def get_field(self, field: Cursor, offset: int, follow: bool): for index, argument in enumerate(function.argument_types()) ] return_type = self.get_type_string(function.get_result(), follow, False) or "__int64" - return DefinedStructFuncField(name, "__fastcall", offset, False, [], return_type, parameters) + return DefinedStructFuncField(name, "__fastcall", offset, False, [], [], return_type, parameters) type_string = self.get_type_string(type, follow, True) if type_string is None: @@ -333,8 +333,8 @@ def get_field(self, field: Cursor, offset: int, follow: bool): return None if count is not None: - return DefinedStructFixedField(name, type_string, offset, False, [], count, False) - return DefinedStructField(name, type_string, offset, False, []) + return DefinedStructFixedField(name, type_string, offset, False, [], [], count, False) + return DefinedStructField(name, type_string, offset, False, [], []) class ClangInterface(object): diff --git a/scripts/shared/exdschema_wrapper.py b/scripts/shared/exdschema_wrapper.py index 869c16437a..f4e099e69a 100644 --- a/scripts/shared/exdschema_wrapper.py +++ b/scripts/shared/exdschema_wrapper.py @@ -148,6 +148,7 @@ def map_header_and_schema( definition_offset, False, [], + [], field.count, False, ) @@ -160,6 +161,7 @@ def map_header_and_schema( definition_offset, False, [], + [], field.count, False, ) @@ -196,7 +198,8 @@ def map_header_and_schema( enum_type, definition_offset, False, - [] + [], + [], ) ) else: @@ -206,7 +209,8 @@ def map_header_and_schema( definition.get_base_type_string(), definition_offset, False, - [] + [], + [], ) ) diff --git a/scripts/shared/src_wrapper.py b/scripts/shared/src_wrapper.py index 0b20c1cdd7..8035fff40f 100644 --- a/scripts/shared/src_wrapper.py +++ b/scripts/shared/src_wrapper.py @@ -355,7 +355,7 @@ def _get_union_members( if isinstance(field, DefinedStructFixedField) or field.type == "__fastcall": return [] return [ - DefinedStructField(union_value.name, union_value.type, field.offset, False, []) + DefinedStructField(union_value.name, union_value.type, field.offset, False, [], []) for union_value in field.union_values ] diff --git a/scripts/shared/structs_schema.py b/scripts/shared/structs_schema.py index 4d7f695c44..9ca9cd459f 100644 --- a/scripts/shared/structs_schema.py +++ b/scripts/shared/structs_schema.py @@ -155,6 +155,7 @@ def parse_struct(struct: YamlStructExport) -> DefinedStruct: field["offset"], base, template_args, + union_values, field["size"], field["is_string"], ) @@ -172,6 +173,7 @@ def parse_struct(struct: YamlStructExport) -> DefinedStruct: field["offset"], base, template_args, + union_values, field["return_type"], parameters, ) From 7a14a1ca49153d7c128060f4d56755845363eead Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Mon, 19 Oct 2026 00:55:41 +0200 Subject: [PATCH 91/95] fix some dependency ordering requirements --- CExporter/Exporter.TypeProcessing.cs | 10 ++++++++-- CExporter/Exporter.cs | 4 ++++ CExporter/ProcessedData.cs | 12 ++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/CExporter/Exporter.TypeProcessing.cs b/CExporter/Exporter.TypeProcessing.cs index d5c0678e9c..a296ea001d 100644 --- a/CExporter/Exporter.TypeProcessing.cs +++ b/CExporter/Exporter.TypeProcessing.cs @@ -626,13 +626,19 @@ .. memberFunctionParameters.Select(ProcessParameter).ToArray()] var unionValues = unionFields.Select(unionField => { (string? typeOverride, Type[] typeOverrideTemplates) = (null, []); - if (unionField.FieldType is { Namespace: ExporterStatics.StdNamespacePrefix }) + if (unionField.FieldType is { Namespace: ExporterStatics.StdNamespacePrefix }) { (typeOverride, typeOverrideTemplates) = ProcessStdType(unionField.FieldType); + foreach (var requiredType in typeOverrideTemplates) { + _processType.Add(requiredType); + } + } else + _processType.Add(unionField.FieldType); return (new ProcessedFieldUnion { FieldName = unionField.Name, FieldType = unionField.FieldType, - FieldTypeOverride = typeOverride + FieldTypeOverride = typeOverride, + FieldTypeOverrideTemplates = typeOverrideTemplates }, typeOverrideTemplates); }).ToArray(); diff --git a/CExporter/Exporter.cs b/CExporter/Exporter.cs index 2b1fd23bd5..5a4c4612db 100644 --- a/CExporter/Exporter.cs +++ b/CExporter/Exporter.cs @@ -418,6 +418,10 @@ static IEnumerable fieldReferences(ProcessedField field) { : field.FieldTypeOverrideTemplates ?? []; foreach (var templateArgument in templateArguments) yield return templateArgument.FixTypeName(); + foreach (var unionValue in field.UnionValues) { + if (unionValue.FieldType.IsFunctionPointer || unionValue.FieldType.IsUnmanagedFunctionPointer) continue; + yield return unionValue.FieldTypeOverride ?? unionValue.FieldType.FixTypeName(); + } } } diff --git a/CExporter/ProcessedData.cs b/CExporter/ProcessedData.cs index 02c0a55325..242d67beb6 100644 --- a/CExporter/ProcessedData.cs +++ b/CExporter/ProcessedData.cs @@ -21,6 +21,7 @@ public class ProcessedFieldUnion { public required Type FieldType; public required string FieldName; public required string? FieldTypeOverride; + public Type[]? FieldTypeOverrideTemplates; } public class ProcessedField { @@ -82,7 +83,18 @@ public class ProcessedStruct { public string[] DependencyNames { get { if (_dependencyNames.Length == 0) { + var unionMembers = Fields.SelectMany(unionField => unionField.UnionValues.Select(unionValue => new ProcessedField { + FieldType = unionValue.FieldType, + FieldTypeOverride = unionValue.FieldTypeOverride, + FieldTypeOverrideTemplates = unionValue.FieldTypeOverrideTemplates, + FieldName = unionValue.FieldName, + FieldOffset = unionField.FieldOffset, + Bits = [], + IsUnion = false, + UnionValues = [] + })); _dependencyNames = Fields + .Concat(unionMembers) .Where(t => { if (t.FieldTypeOverride == null) { if (t.FieldType.IsEnum) { From 4f34634d17a329ce3709fea11ad2426529570562 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Mon, 19 Oct 2026 04:15:21 +0200 Subject: [PATCH 92/95] fix vtables with ida not using its proper inherited one --- scripts/ida/struct_import.py | 1 + scripts/shared/src_wrapper.py | 26 +++++++++++++++++++++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/scripts/ida/struct_import.py b/scripts/ida/struct_import.py index e4a12871b8..c1b06a0781 100644 --- a/scripts/ida/struct_import.py +++ b/scripts/ida/struct_import.py @@ -16,6 +16,7 @@ from structs_schema import get_yaml src = SrcInterface() +src.ida_vtables = True ida = IdaInterface() print("Loading data.yml") ida.load_data_yml() diff --git a/scripts/shared/src_wrapper.py b/scripts/shared/src_wrapper.py index 8035fff40f..d746f92b89 100644 --- a/scripts/shared/src_wrapper.py +++ b/scripts/shared/src_wrapper.py @@ -16,6 +16,7 @@ class SrcInterface(object): full_padding: bool = True + ida_vtables: bool = False def __init__(self): self._layout_cache: dict[tuple[str, tuple[str, ...]], tuple[int, bool]] = {} @@ -873,7 +874,7 @@ def collect_inline_enum_definitions(struct: DefinedStruct): def collect_vtable_names(struct: DefinedStruct): # a template vtable only exists in IDA as its instantiations if struct.virtual_functions and not struct.template_types: - vtable_name = f"{struct.type}::VTable" + vtable_name = f"{struct.type}_vtbl" if self.ida_vtables else f"{struct.type}::VTable" if vtable_name not in seen_vtable_names: seen_vtable_names.add(vtable_name) vtable_names.append(vtable_name) @@ -1070,7 +1071,7 @@ def build_struct_string( struct_definition_lookup: dict[str, DefinedStruct] | None = None, opaque_pointer_types: set[str] | None = None, ) -> tuple[IndentWriter, IndentWriter]: - """The forward declarations and the definition, the vtable inside as VTable.""" + """The forward declarations and the definition, the vtable inside as VTable or beside as Name_vtbl for IDA.""" struct_string_forward: IndentWriter = IndentWriter() struct_string_define: IndentWriter = IndentWriter() template_struct_lookup = template_struct_lookup or {} @@ -1155,7 +1156,16 @@ def append_chunk(chunk: IndentWriter | str): for _, inline_define in inline_struct_chunks: append_chunk(inline_define) - if struct.virtual_functions: + ida_vtable = bool(struct.virtual_functions) and self.ida_vtables + if ida_vtable: + if template_declaration: + struct_string_forward += template_declaration + struct_string_forward += f"struct {short_name}_vtbl;" + if prev_size == 0: + template_suffix = f"<{', '.join(struct.template_types)}>" if struct.template_types else "" + struct_string_define += f"{short_name}_vtbl{template_suffix}* __vftable;" + prev_size = 8 + elif struct.virtual_functions: struct_string_define += "struct VTable {" struct_string_define.indent() for vfunc_line in self.build_vtable_lines(struct): @@ -1300,6 +1310,16 @@ def append_chunk(chunk: IndentWriter | str): struct_string_define.unindent() struct_string_define += "};" + if ida_vtable: + if template_declaration: + struct_string_define += template_declaration + struct_string_define += f"struct {short_name}_vtbl {{" + struct_string_define.indent() + for vfunc_line in self.build_vtable_lines(struct): + struct_string_define += vfunc_line + struct_string_define.unindent() + struct_string_define += "};" + if packed: struct_string_define.lines.insert(0, IndentWriterValue(0, "#pragma pack(push, 1)")) struct_string_define += "#pragma pack(pop)" From 222c49489c2bfc422d2f4872297a74f8c238c1c4 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Tue, 20 Oct 2026 10:19:28 +0200 Subject: [PATCH 93/95] don't flatten structs in Ghidra. --- scripts/ghidra/clang_wrapper.py | 71 ++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 33 deletions(-) diff --git a/scripts/ghidra/clang_wrapper.py b/scripts/ghidra/clang_wrapper.py index d00fb07289..e91d4900e4 100644 --- a/scripts/ghidra/clang_wrapper.py +++ b/scripts/ghidra/clang_wrapper.py @@ -110,7 +110,7 @@ def _collect_declarations(self, cursor: Cursor): self.aliases[child.spelling] = type_string def get_export(self) -> DefinedStructExport: - self.flatten_primary_bases() + self.wrap_primary_bases() opaque = [ self.new_struct(name, [], None, union) for name, union in self.opaque.items() @@ -128,43 +128,48 @@ def get_primary_base(self, struct: DefinedStruct) -> DefinedStruct | None: base = self.structs.get(base_field.type) return None if base is None or base.union else base - def flatten_primary_bases(self): - """Ghidra can not retype the vtable pointer inside an embedded base, so a struct with a vtable of its own - gets the members of its offset 0 bases with __vtable pointing to its own VTable instead.""" - - def get_primary_fields(struct: DefinedStruct) -> list: - base = self.get_primary_base(struct) - if base is None: - return list(struct.fields) - fields = [field for field in struct.fields if not (field.base and field.offset == 0)] - return [*get_primary_fields(base), *fields] + def has_vtable_pointer(self, struct: DefinedStruct) -> bool: + """The struct or a base at offset 0 starts with __vtable.""" + if any(field.offset == 0 and field.name == "__vtable" for field in struct.fields): + return True + base = self.get_primary_base(struct) + return base is not None and self.has_vtable_pointer(base) - flattened: dict[str, list] = {} + def wrap_primary_bases(self): + """Ghidra can not retype the vtable pointer inside an embedded base, so a struct with a vtable of its own + holds its offset 0 base in a union with __vtable pointing to its own VTable, the base keeps its layout.""" + wrappers: dict[str, DefinedStruct] = {} for name, struct in self.structs.items(): vtable_name = f"{name}::VTable" - if vtable_name not in self.structs or self.get_primary_base(struct) is None: - continue - primary_fields = get_primary_fields(struct) - vtable_field = next((field for field in primary_fields if field.offset == 0 and field.name == "__vtable"), None) - if vtable_field is None: - # the root base has no vtable pointer to replace + base = self.get_primary_base(struct) + if vtable_name not in self.structs or base is None or not self.has_vtable_pointer(base): continue + base_field = next(field for field in struct.fields if field.base and field.offset == 0) + # ghidra settles a tie between union members on the first, so a vtable call goes through __vtable + wrapper = self.new_struct( + f"{name}::__primary_base", + [ + DefinedStructField("__vtable", f"{vtable_name}*", 0, False, [], []), + replace(base_field, name=re.sub(r"\W", "_", split_type_path(base.type)[-1]), base=False), + ], + base.size, + True, + ) + wrappers[name] = wrapper - fields = [DefinedStructField("__vtable", f"{vtable_name}*", 0, False, [], [])] - names = {"__vtable"} - # the most derived first, a member hides the one of a base with the same name - for field in reversed(primary_fields): - if field is vtable_field: - continue - if field.name in names: - field = replace(field, name=f"{field.name}_{field.offset:X}") - names.add(field.name) - fields.append(field) - flattened[name] = sorted(fields, key=lambda field: field.offset) - - # replaced after the loop, so every struct is flattened from its bases as clang laid them out - for name, fields in flattened.items(): - self.structs[name] = replace(self.structs[name], fields=fields) + # replaced after the loop, so every base is looked at as clang laid it out + # every union comes right before the struct holding it, after the base and vtable it is made of + structs: dict[str, DefinedStruct] = {} + for name, struct in self.structs.items(): + if name in wrappers: + wrapper = wrappers[name] + structs[wrapper.type] = wrapper + struct = replace(struct, fields=[ + replace(field, type=wrapper.type) if field.base and field.offset == 0 else field + for field in struct.fields + ]) + structs[name] = struct + self.structs = structs def new_struct(self, type_name: str, fields: list, size: int | None, union: bool) -> DefinedStruct: parts = split_type_path(type_name) From 5bf15b988e26866adb0183e404e6a8fe36d58042 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Fri, 23 Oct 2026 04:30:10 +0200 Subject: [PATCH 94/95] add some ghidra logging --- scripts/ghidra/full_run.py | 9 +++ scripts/ghidra/ghidra_logging.py | 110 +++++++++++++++++++++++++++++++ scripts/ghidra/rename.py | 5 ++ scripts/ghidra/struct_import.py | 5 ++ 4 files changed, 129 insertions(+) create mode 100644 scripts/ghidra/ghidra_logging.py diff --git a/scripts/ghidra/full_run.py b/scripts/ghidra/full_run.py index 5b4949d33c..50404ce24a 100644 --- a/scripts/ghidra/full_run.py +++ b/scripts/ghidra/full_run.py @@ -4,12 +4,20 @@ # @runtime PyGhidra import os +import sys try: working_directory = os.path.dirname(os.path.realpath(__file__)) except NameError: working_directory = os.path.dirname(str(sourceFile)) # noqa +if working_directory not in sys.path: + sys.path.append(working_directory) + +from ghidra_logging import start_logging, stop_logging + +start_logging("full_run.py", monitor, isRunningHeadless()) # noqa + print("Loading rename.py") exec(open(os.path.join(working_directory, "rename.py")).read()) @@ -17,3 +25,4 @@ exec(open(os.path.join(working_directory, "struct_import.py")).read()) analyzeChanges(currentProgram) # noqa +stop_logging() diff --git a/scripts/ghidra/ghidra_logging.py b/scripts/ghidra/ghidra_logging.py new file mode 100644 index 0000000000..76abc4a367 --- /dev/null +++ b/scripts/ghidra/ghidra_logging.py @@ -0,0 +1,110 @@ +"""Forwards stdout and stderr of a script run to Ghidra's log, which saves it to application.log.""" + +import sys + +from ghidra.app.script import GhidraScript +from ghidra.util import Msg + +try: + # what GhidraScript.println wraps its lines in, lets the log filter on script messages + from ghidra.app.script import ScriptMessage +except ImportError: + ScriptMessage = str + +ORIGINATOR = GhidraScript.class_ + + +class GhidraLogStream(object): + """Text stream that passes writes on to the stream it replaces and logs each full line.""" + + def __init__(self, target, script_name: str, is_stderr: bool, echo: bool): + self.target = target + self.script_name = script_name + self.is_stderr = is_stderr + self.echo = echo + self.buffer = "" + + def write(self, text: str) -> int: + if self.echo: + self.target.write(text) + self.buffer += text + *lines, self.buffer = self.buffer.split("\n") + for line in lines: + self.log(line) + return len(text) + + def flush(self): + if self.buffer: + self.log(self.buffer) + self.buffer = "" + if self.echo: + self.target.flush() + + def log(self, line: str): + line = line.rstrip("\r") + if not line.strip(): + return + message = ScriptMessage(f"{self.script_name}> {line}") + stripped = line.lstrip() + if self.is_stderr or stripped.startswith("Error"): + Msg.error(ORIGINATOR, message) + elif stripped.startswith("Warning"): + Msg.warn(ORIGINATOR, message) + else: + Msg.info(ORIGINATOR, message) + + def isatty(self) -> bool: + return False + + def __getattr__(self, name): + return getattr(self.target, name) + + +class _Redirect(object): + def __init__(self, monitor, stdout, stderr): + self.monitor = monitor + self.stdout = stdout + self.stderr = stderr + self.depth = 1 + + +_active: _Redirect | None = None + + +def start_logging(script_name: str, monitor, headless: bool): + """Sends stdout and stderr to Ghidra's log until stop_logging. + + Scripts exec'd by another script share its monitor and nest into its redirect. + """ + global _active + if _active is not None: + if _active.monitor is monitor: + _active.depth += 1 + return + _restore() + + stdout = GhidraLogStream(sys.stdout, script_name, False, not headless) + stderr = GhidraLogStream(sys.stderr, script_name, True, not headless) + _active = _Redirect(monitor, stdout, stderr) + sys.stdout = stdout + sys.stderr = stderr + + +def stop_logging(): + if _active is None: + return + _active.depth -= 1 + if _active.depth <= 0: + _restore() + + +def _restore(): + global _active + for stream in (_active.stdout, _active.stderr): + stream.flush() + # the script runner may have swapped the streams back already + if sys.stdout is _active.stdout: + sys.stdout = _active.stdout.target + if sys.stderr is _active.stderr: + sys.stderr = _active.stderr.target + _active = None diff --git a/scripts/ghidra/rename.py b/scripts/ghidra/rename.py index 22729ed757..80d49f99dd 100644 --- a/scripts/ghidra/rename.py +++ b/scripts/ghidra/rename.py @@ -16,6 +16,7 @@ sys.path.append(import_path) from ghidra_wrapper import GhidraInterface +from ghidra_logging import start_logging, stop_logging from class_factory import apply_data lua_namespace = "Client::Game::Event::Lua" @@ -58,6 +59,8 @@ def apply_lua_funcs(): func_name = regs['r8'] ghidra.set_func_name(regs['r9']['val'], f"{name}.{func_name}", unique=True) +start_logging("rename.py", monitor, isRunningHeadless()) # noqa + ghidra = GhidraInterface(currentProgram, monitor) # noqa print("Loading data.yml") ghidra.load_data_yml() @@ -67,3 +70,5 @@ def apply_lua_funcs(): print("Naming lua functions") apply_lua_funcs() print("Done") + +stop_logging() diff --git a/scripts/ghidra/struct_import.py b/scripts/ghidra/struct_import.py index 308cd90927..14c1e19a31 100644 --- a/scripts/ghidra/struct_import.py +++ b/scripts/ghidra/struct_import.py @@ -27,8 +27,11 @@ ) from src_wrapper import SrcInterface from ghidra_wrapper import GhidraInterface +from ghidra_logging import start_logging, stop_logging from structs_schema import get_yaml, DefinedStructFuncParam +start_logging("struct_import.py", monitor, isRunningHeadless()) # noqa + src = SrcInterface() # one filler field per gap, clang gets slow on a struct with a field for every unknown qword src.full_padding = False @@ -213,3 +216,5 @@ def do_pattern(pattern, suffix, sheetSearchPattern, sheetSearchOffset): print(f"Updating static members for {struct.type}") for member in struct.static_members: ghidra.update_static_member(member, struct) + +stop_logging() From 42a9674f981258afbe1b3160efd26b7800713f50 Mon Sep 17 00:00:00 2001 From: wolfcomp <4028289+wolfcomp@users.noreply.github.com> Date: Fri, 30 Oct 2026 08:59:22 +0100 Subject: [PATCH 95/95] remove old file --- ida/structs_schema.py | 116 ------------------------------------------ 1 file changed, 116 deletions(-) delete mode 100644 ida/structs_schema.py diff --git a/ida/structs_schema.py b/ida/structs_schema.py deleted file mode 100644 index 828f1f3103..0000000000 --- a/ida/structs_schema.py +++ /dev/null @@ -1,116 +0,0 @@ -class DefinedStructBase: - def __init__(self, name, type, namespace): - # type: (str, str, str) -> None - self.name = name - self.type = type - self.namespace = namespace - - -class DefinedStructEnum(DefinedStructBase, object): - def __init__(self, name, type, underlying, namespace, flags, values): - # type: (str, str, str, str, bool, dict[str, int]) -> None - super(DefinedStructEnum, self).__init__(name, type, namespace) - self.name = name - self.type = type - self.values = values - self.flags = flags - self.underlying = underlying - - -class DefinedStructFuncParam: - def __init__(self, name, type): - # type: (str, str) -> None - self.name = name - if type == "__fastcall": - self.type = "__int64" - else: - self.type = type - - -class DefinedStructVFunc: - def __init__(self, name, return_type, offset, parameters): - # type: (str, str, int, list[DefinedStructFuncParam]) -> None - self.name = name - self.return_type = return_type - self.offset = offset - self.parameters = parameters - self.inherited_from_preprocess = False - - -class DefinedStructMemFunc: - def __init__(self, signature, return_type, parameters, name): - # type: (str, str, list[DefinedStructFuncParam], str) -> None - self.signature = signature - self.return_type = return_type - self.parameters = parameters - self.name = name - - -class DefinedStructField(DefinedStructFuncParam, object): - def __init__(self, name, type, offset, base): - # type: (str, str, int, bool) -> None - super(DefinedStructField, self).__init__(name, type) - self.offset = offset - self.base = base - self.srclang_field_name = name - self.srclang_is_baseclass = False - - -class DefinedStructFuncField(DefinedStructField, object): - def __init__(self, name, type, offset, base, return_type, params): - # type: (str, str, int, bool, str | None, list[DefinedStructFuncParam] | None) -> None - super(DefinedStructFuncField, self).__init__(name, type, offset, base) - self.return_type = return_type - self.parameters = params - - -class DefinedStructStaticMember: - def __init__(self, name, signature, relative_offsets, return_type, is_pointer): - # type: (str, str, list[int], str, bool) -> None - self.name = name - self.signature = signature - self.relative_offsets = relative_offsets - self.return_type = return_type - self.is_pointer = is_pointer - - -class DefinedStructFixedField(DefinedStructField, object): - def __init__(self, name, type, offset, base, size, is_string): - # type: (str, str, int, bool, str | None, bool) -> None - super(DefinedStructFixedField, self).__init__(name, type, offset, base) - self.size = size - self.is_string = is_string - -class DefinedStruct(DefinedStructBase, object): - def __init__( - self, - name, - type, - namespace, - fields, - size, - vtable_size, - virtual_functions, - member_functions, - union, - static_member_functions, - static_members, - ): - # type: (str, str, str, list[DefinedStructField], int | None, int | None, list[DefinedStructVFunc] | None, list[DefinedStructMemFunc], str, list[DefinedStructMemFunc] | None, list[DefinedStructStaticMember] | None) -> None - super(DefinedStruct, self).__init__(name, type, namespace) - self.fields = fields - self.size = size - self.vtable_size = vtable_size - self.virtual_functions = virtual_functions - self.member_functions = member_functions - self.union = bool(union) - self.static_member_functions = static_member_functions - self.static_members = static_members - self.srclang_needs_vfptr = len(fields) == 0 or fields[0].offset != 0 - - -class DefinedStructExport: - def __init__(self, enums, structs): - # type: (list[DefinedStructEnum], list[DefinedStruct]) -> None - self.enums = enums - self.structs = structs