diff --git a/.gitattributes b/.gitattributes index c2f642f443..cb1ea3aa99 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,4 +3,5 @@ *.png binary ida/ffxiv_client_structs*.h linguist-generated=true -ida/ffxiv_structs.yml linguist-generated=true +scripts/ffxiv_structs.yml linguist-generated=true +scripts/structs.yml linguist-generated=true diff --git a/.github/workflows/pr-change-check-patch-day.yaml b/.github/workflows/pr-change-check-patch-day.yaml index 2a6cdc3593..ed61d79873 100644 --- a/.github/workflows/pr-change-check-patch-day.yaml +++ b/.github/workflows/pr-change-check-patch-day.yaml @@ -29,7 +29,7 @@ jobs: - name: Check error txt file for content id: check_error run: | - if [ -s ./ida/errors.txt ]; then + if [ -s ./scripts/errors.txt ]; then echo "error=true" >> $GITHUB_OUTPUT else echo "error=false" >> $GITHUB_OUTPUT @@ -40,7 +40,7 @@ jobs: with: script: | const fs = require('fs'); - const content = fs.readFileSync('./ida/errors.txt', 'utf8'); + const content = fs.readFileSync('./scripts/errors.txt', 'utf8'); const comments = (await github.rest.issues.listComments({ issue_number: context.payload.pull_request.number, owner: context.repo.owner, @@ -108,5 +108,5 @@ jobs: - uses: actions/upload-artifact@v4 with: name: structs.yml - path: ./ida/ffxiv_structs.yml + path: ./scripts/structs.yml retention-days: 7 diff --git a/.github/workflows/pr-change-check.yaml b/.github/workflows/pr-change-check.yaml index 94a8390916..c42921aef6 100644 --- a/.github/workflows/pr-change-check.yaml +++ b/.github/workflows/pr-change-check.yaml @@ -30,7 +30,7 @@ jobs: - name: Check error txt file for content id: check_error run: | - if [ -s ./ida/errors.txt ]; then + if [ -s ./scripts/errors.txt ]; then echo "error=true" >> $GITHUB_OUTPUT else echo "error=false" >> $GITHUB_OUTPUT @@ -41,7 +41,7 @@ jobs: with: script: | const fs = require('fs'); - const content = fs.readFileSync('./ida/errors.txt', 'utf8'); + const content = fs.readFileSync('./scripts/errors.txt', 'utf8'); const comments = (await github.rest.issues.listComments({ issue_number: context.payload.pull_request.number, owner: context.repo.owner, @@ -109,7 +109,7 @@ jobs: - uses: actions/upload-artifact@v4 with: name: structs.yml - path: ./ida/ffxiv_structs.yml + path: ./scripts/structs.yml retention-days: 7 build-pr: @@ -179,7 +179,7 @@ jobs: run: | dotnet tool install -g Microsoft.DotNet.ApiCompat.Tool dotnet run --project CompatChecker/CompatChecker.csproj -c Release -- ida/cs-main/FFXIVClientStructs.dll ida/cs-pr/FFXIVClientStructs.dll ida/errors.txt - if [ -s ./ida/errors.txt ]; then + if [ -s ./scripts/errors.txt ]; then echo "breaking=true" >> $GITHUB_OUTPUT else echo "breaking=false" >> $GITHUB_OUTPUT @@ -190,7 +190,7 @@ jobs: with: script: | const fs = require('fs'); - const content = fs.readFileSync('./ida/errors.txt', 'utf8'); + const content = fs.readFileSync('./scripts/errors.txt', 'utf8'); github.rest.issues.addLabels({ issue_number: context.payload.pull_request.number, owner: context.repo.owner, diff --git a/.github/workflows/push-build.yaml b/.github/workflows/push-build.yaml index b6617c8a11..5fa5008bb6 100644 --- a/.github/workflows/push-build.yaml +++ b/.github/workflows/push-build.yaml @@ -24,7 +24,7 @@ jobs: - name: Check error txt file for content id: check_error run: | - if [ -s ./ida/errors.txt ]; then + if [ -s ./scripts/errors.txt ]; then echo "error=true" >> $GITHUB_OUTPUT else echo "error=false" >> $GITHUB_OUTPUT @@ -33,4 +33,4 @@ jobs: if: steps.check_error.outputs.error == 'true' uses: peter-evans/commit-comment@v4 with: - body-path: './ida/errors.txt' + body-path: './scripts/errors.txt' diff --git a/.github/workflows/validate-data.yaml b/.github/workflows/validate-data.yaml index bd9bc63c47..a6b60eaba3 100644 --- a/.github/workflows/validate-data.yaml +++ b/.github/workflows/validate-data.yaml @@ -7,10 +7,10 @@ on: - synchronize - reopened paths: - - 'ida/data.yml' + - 'scripts/data.yml' push: paths: - - 'ida/data.yml' + - 'scripts/data.yml' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -28,4 +28,4 @@ jobs: run: | npm install -g js-yaml - name: Validate data.yml - run: NODE_PATH="$(npm root -g):$NODE_PATH" node ./ida/data-validator.js \ No newline at end of file + run: NODE_PATH="$(npm root -g):$NODE_PATH" node ./scripts/data-validator.js \ No newline at end of file diff --git a/.gitignore b/.gitignore index a0cda0c0db..84273833bb 100644 --- a/.gitignore +++ b/.gitignore @@ -4,9 +4,6 @@ ida/uld_names.txt .vscode/ .idea/ -# CompatChecker test files -tests/ - ## Ignore Visual Studio temporary files, build results, and ## files generated by popular Visual Studio add-ons. ## @@ -435,3 +432,10 @@ FodyWeavers.xsd *.msix *.msm *.msp + + +# test folders +scripts/luminatmp/ +scripts/excel_headers/ +tests/ +luminatmp/ \ No newline at end of file diff --git a/CExporter/Exporter.TypeProcessing.cs b/CExporter/Exporter.TypeProcessing.cs new file mode 100644 index 0000000000..a296ea001d --- /dev/null +++ b/CExporter/Exporter.TypeProcessing.cs @@ -0,0 +1,692 @@ +using System; +using System.Collections.Frozen; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using FFXIVClientStructs.Attributes; +using InteropGenerator.Runtime.Attributes; + +namespace CExporter; + +public partial class Exporter { + private static void ProcessType(ProcessingType type, bool quiet) { + var ret = PreProcessType(type, quiet); + switch (ret) { + case null: + return; + case ProcessedStruct s: + var existingIndex = _structs.FindIndex(existing => + existing.StructTypeName == s.StructTypeName); + if (existingIndex < 0) { + _structs.Add(s); + } else if (s.TemplateTypes.Length > 0 && _structs[existingIndex].TemplateTypes.Length == 0) { + var nonGeneric = _structs[existingIndex]; + s.Fields = [.. s.Fields.Where(field => + !field.IsBase || field.FieldType != nonGeneric.StructType)]; + s.VirtualFunctions ??= nonGeneric.VirtualFunctions; + s.VirtualFunctionSize = Math.Max(s.VirtualFunctionSize, nonGeneric.VirtualFunctionSize); + s.MemberFunctions = [.. nonGeneric.MemberFunctions, .. s.MemberFunctions]; + var genericSelfType = $"{nonGeneric.StructTypeName}<{string.Join(", ", s.TemplateTypes)}>*"; + foreach (var virtualFunction in s.VirtualFunctions ?? []) { + foreach (var parameter in virtualFunction.VirtualFunctionParameters ?? []) { + if (parameter.FieldName == "self" + && parameter.FieldType.IsPointer() + && parameter.FieldType.GetPointerType() == nonGeneric.StructType) + parameter.FieldTypeOverride = genericSelfType; + } + if ((virtualFunction.VirtualFunctionReturnType?.IsPointer() ?? false) && virtualFunction.VirtualFunctionReturnType?.GetPointerType() == nonGeneric.StructType) + virtualFunction.VirtualFunctionReturnTypeOverride = genericSelfType; + } + foreach (var memberFunction in s.MemberFunctions) { + foreach (var parameter in memberFunction.MemberFunctionParameters) { + if (parameter.FieldName == "self" + && parameter.FieldType.IsPointer() + && parameter.FieldType.GetPointerType() == nonGeneric.StructType) + parameter.FieldTypeOverride = genericSelfType; + } + if (memberFunction.MemberFunctionReturnType.Type.IsPointer() && memberFunction.MemberFunctionReturnType.Type.GetPointerType() == nonGeneric.StructType) + memberFunction.MemberFunctionReturnType.OverrideType = genericSelfType; + } + _structs[existingIndex] = s; + } + break; + case ProcessedEnum e: + _enums.Add(e); + break; + } + } + + private static (string typeOverride, Type[] typeOverrideTemplates) ProcessStdType(Type fieldType) { + const string basicString = "BasicString"; + string[] pointerValuesType = ["Vector", "Span"]; + var pointerCount = 0; + while (fieldType.IsPointer()) { + fieldType = fieldType.GetPointerType(); + pointerCount++; + } + var readOnlyNameSpan = fieldType.Name.AsSpan()[3..]; + Span stdNameSpan = stackalloc char[readOnlyNameSpan.Length + 5]; + stdNameSpan.Clear(); + ExporterStatics.StdCppNamespace.AsSpan().CopyTo(stdNameSpan); + if (readOnlyNameSpan[^2] == '`') { + for (var i = 0; i < readOnlyNameSpan.Length; i++) { + if (readOnlyNameSpan[i] == '`') break; + stdNameSpan[5 + i] = char.ToLower(readOnlyNameSpan[i]); + } + } else { + if (readOnlyNameSpan.CompareTo(basicString.AsSpan(), StringComparison.InvariantCultureIgnoreCase) != 0) { + for (var i = 0; i < readOnlyNameSpan.Length; i++) { + if (readOnlyNameSpan[i] == '`') break; + stdNameSpan[5 + i] = char.ToLower(readOnlyNameSpan[i]); + } + } else { + throw new NotImplementedException("Yell at me to fix this if it happens."); + } + } + var generics = fieldType.GenericTypeArguments; + var shouldBePointer = false; + foreach (var pointerType in pointerValuesType) { + if (!readOnlyNameSpan.StartsWith(pointerType)) continue; + shouldBePointer = true; + break; + } + while (stdNameSpan[^1] == '\0') stdNameSpan = stdNameSpan[..^1]; + if (stdNameSpan is "std::string") return ($"{stdNameSpan}", []); + if (stdNameSpan is "std::unorderedmap") stdNameSpan = "std::unordered_map".ToCharArray(); + var genericsDefine = generics.Select(t => { + var pointerCount = 0; + while (t.IsPointer()) { + t = t.GetPointerType(); + pointerCount++; + } + + if (t is { Namespace: ExporterStatics.StdNamespacePrefix }) { + var (typeOverride, requiredTypes) = ProcessStdType(t); + return (typeOverride + new string('*', pointerCount), requiredTypes); + } + + while (pointerCount > 0) { + t = t.MakePointerType(); + pointerCount--; + } + + return (t.FixTypeName(), [shouldBePointer ? t.MakePointerType() : t]); + }).Cast<(string typeOverride, Type[] typeOverrideTemplates)>().ToArray(); + return ($"{stdNameSpan}<{string.Join(',', genericsDefine.Select(t => t.typeOverride))}>{new string('*', pointerCount)}", genericsDefine.SelectMany(t => t.typeOverrideTemplates).ToArray()); + } + + private static ProcessedField ProcessField(FieldInfo field, int offset) { + if (field.FieldType is { Namespace: ExporterStatics.StdNamespacePrefix } && !field.FieldType.IsGenericTypeParameterOrPointer()) { + if (!field.FieldType.Name.StartsWith("StdLinkedList") && !(field.DeclaringType?.FullName!.Contains("StdLinkedList") ?? false)){ + var (typeOverride, typeOverrideTemplates) = ProcessStdType(field.FieldType); + return new ProcessedField { + FieldType = field.FieldType, + FieldOffset = field.GetFieldOffset() - offset, + FieldName = field.Name, + IsBase = field.IsDirectBase(), + FieldTypeOverride = typeOverride, + FieldTypeOverrideTemplates = typeOverrideTemplates, + Bits = [], + IsUnion = false, + UnionValues = [] + }; + } + if(!(field.DeclaringType?.FullName!.Contains("StdLinkedList") ?? false)) { + _processType.Add(field.FieldType); + return new ProcessedField { + FieldType = field.FieldType, + FieldOffset = field.GetFieldOffset() - offset, + FieldName = field.Name, + IsBase = field.IsDirectBase(), + FieldTypeOverride = "StdLinkedList", + FieldTypeOverrideTemplates = field.FieldType.GetGenericArguments(), + Bits = [], + IsUnion = false, + UnionValues = [] + }; + } + _processType.Add(field.FieldType); + return new ProcessedField { + FieldType = field.FieldType, + FieldOffset = field.GetFieldOffset() - offset, + FieldName = field.Name, + IsBase = field.IsDirectBase(), + FieldTypeOverride = null, + FieldTypeOverrideTemplates = null, + Bits = [], + IsUnion = false, + UnionValues = [] + }; + } + if (field.FieldType is { Namespace: "FFXIVClientStructs.STD.Helper", Name: "Node*" }) { + var fieldTypeOverride = $"std::map<{string.Join(',', field.FieldType.GetGenericArguments()[0].GetGenericArguments().Select(t => t.FixTypeName()))}>::iterator"; + return new ProcessedField { + FieldType = field.FieldType, + FieldOffset = field.GetFieldOffset() - offset, + FieldName = field.Name, + IsBase = field.IsDirectBase(), + FieldTypeOverride = fieldTypeOverride, + Bits = [], + IsUnion = false, + UnionValues = [] + }; + } + if (field.FieldType.IsFunctionPointer || field.FieldType.IsUnmanagedFunctionPointer) { + _processType.Add(field.FieldType.GetFunctionPointerReturnType()); + return new ProcessedFunctionField { + FieldType = field.FieldType, + FieldOffset = field.GetFieldOffset() - offset, + FieldName = field.Name, + FunctionReturnType = field.FieldType.GetFunctionPointerReturnType(), + FunctionParameters = field.FieldType.GetFunctionPointerParameterTypes().Select((p, i) => { + _processType.Add(p); + return new ProcessedField { + FieldType = p, + FieldOffset = -1, + FieldName = 'a' + (i + 1).ToString(), + Bits = [], + IsUnion = false, + UnionValues = [] + }; + }).ToArray(), + Bits = [], + IsUnion = false, + UnionValues = [] + }; + } + if (field.FieldType.IsFixedBuffer()) { + var arr = field.GetCustomAttributes().Where(t => t.GetType().Name.Contains("FixedSizeArrayAttribute")).ToArray(); + var size = field.FieldType.StructLayoutAttribute!.Size; + if (arr.Length != 0) { + var type = arr[0].GetType(); + size = (int)type.GetProperty("Count")!.GetValue(arr[0])!; + var elementType = type.GenericTypeArguments[0]; + _processType.Add(elementType); + return new ProcessedFixedField { + FieldType = elementType, + FieldOffset = field.GetFieldOffset() - offset, + FieldName = field.Name, + FixedSize = size, + Bits = [], + IsUnion = false, + UnionValues = [] + }; + } + var fixedType = field.FieldType.GetFields()[0].FieldType; + _processType.Add(fixedType); + if (fixedType.IsBaseType()) + size /= fixedType.SizeOf(); + var fieldOverrideType = fixedType == typeof(nint) ? field.GetCustomAttribute()?.SheetName ?? null : null; + if (fieldOverrideType != null) + fieldOverrideType = $"Component::Exd::Sheets::{fieldOverrideType}*"; + return new ProcessedFixedField { + FieldType = fixedType, + FieldOffset = field.GetFieldOffset() - offset, + FieldName = field.Name, + FixedSize = size, + FieldTypeOverride = fieldOverrideType, + Bits = [], + IsUnion = false, + UnionValues = [] + }; + } + if (field.FieldType.GetCustomAttribute() != null) { + var arrLength = field.FieldType.GetCustomAttribute()!.Length; + var elementType = field.FieldType.GetGenericArguments()[0]; + var isString = field.GetCustomAttribute()?.IsString ?? false; + var fieldOverrideType = elementType == typeof(nint) ? field.GetCustomAttribute()?.SheetName ?? null : null; + Type[] requiredOverrideTypes = []; + if (fieldOverrideType != null) + fieldOverrideType = $"Component::Exd::Sheets::{fieldOverrideType}*"; + if (isString && elementType == typeof(byte)) + fieldOverrideType = "char"; + if (elementType.IsPointer() && elementType.GetPointerType() is { Namespace: ExporterStatics.StdNamespacePrefix } || elementType is { Namespace: ExporterStatics.StdNamespacePrefix }) { + (fieldOverrideType, requiredOverrideTypes) = ProcessStdType(elementType); + foreach (var requiredType in requiredOverrideTypes) { + _processType.Add(requiredType); + } + } else + _processType.Add(elementType); + return new ProcessedFixedField { + FieldType = elementType, + FieldOffset = field.GetFieldOffset() - offset, + FieldName = field.Name[1].ToString().ToUpper() + field.Name[2..], + FixedSize = arrLength, + FixedString = isString, + FieldTypeOverride = fieldOverrideType, + Bits = [], + IsUnion = false, + UnionValues = [], + FieldTypeOverrideTemplates = requiredOverrideTypes + }; + } + if (field.GetCustomAttribute() != null) { + var sheetName = field.GetCustomAttribute()!.SheetName; + return new ProcessedField { + FieldType = field.FieldType, + FieldOffset = field.GetFieldOffset() - offset, + FieldName = $"{sheetName}Sheet", + FieldTypeOverride = $"Component::Exd::Sheets::{sheetName}", + Bits = [], + IsUnion = false, + UnionValues = [] + }; + } + if (field.GetCustomAttribute() != null) { + var sheetName = field.GetCustomAttribute()!.SheetName; + return new ProcessedField { + FieldType = field.FieldType, + FieldOffset = field.GetFieldOffset() - offset, + FieldName = field.Name, + FieldTypeOverride = $"Component::Exd::Sheets::{sheetName}*", + Bits = [], + IsUnion = false, + UnionValues = [] + }; + } + if (field.GetCustomAttribute() is { } typeForceAttribute) { + return new ProcessedField { + FieldType = field.FieldType, + FieldOffset = field.GetFieldOffset() - offset, + FieldName = field.Name, + FieldTypeOverride = typeForceAttribute.TypeName, + FieldTypeOverrideCheck = typeForceAttribute.DontCheck, + Bits = [], + IsUnion = false, + UnionValues = [] + }; + } + if (field.IsBitArray()) { + var bitsDefinition = field.GetCustomAttributes().Where(t => t.GetType().Name.Contains("BitFieldAttribute`1")).Select(t => { + var attrType = t.GetType(); + var name = (string)attrType.GetProperty("Name")!.GetValue(t)!; + var offset = (int)attrType.GetProperty("Index")!.GetValue(t)!; + var size = (int)attrType.GetProperty("Length")!.GetValue(t)!; + var type = attrType.GenericTypeArguments[0]; + _processType.Add(type); + return new ProcessedBitField { + Name = name, + Offset = offset, + Size = size, + Type = ExporterStatics.GetBestMatchFromSize(size) + }; + }).ToArray(); + var bits = new List(); + foreach (var bit in bitsDefinition) { + if (bits.Count == 0 && bit.Offset != 0) + bits.Add(new ProcessedBitField { + Name = "UnkBits0", + Offset = 0, + Size = bit.Offset, + Type = ExporterStatics.GetBestMatchFromSize(bit.Offset) + }); + else if (bit.Offset == 0) { + bits.Add(bit); + continue; + } + var last = bits.Last(); + if (last.Offset + last.Size != bit.Offset) { + var newOffset = last.Offset + last.Size; + var size = bit.Offset - newOffset; + + bits.Add(new ProcessedBitField { + Name = $"UnkBits{newOffset}", + Offset = newOffset, + Size = size, + Type = ExporterStatics.GetBestMatchFromSize(size) + }); + } + bits.Add(bit); + } + return new ProcessedField { + FieldType = field.FieldType, + FieldOffset = field.GetFieldOffset() - offset, + FieldName = field.Name, + IsBase = field.IsDirectBase(), + Bits = bits.ToArray(), + IsUnion = false, + UnionValues = [] + }; + } + _processType.Add(field.FieldType); + return new ProcessedField { + FieldType = field.FieldType, + FieldOffset = field.GetFieldOffset() - offset, + FieldName = field.Name, + IsBase = field.IsDirectBase(), + // template_args only covers generic structs held by value + FieldTypeOverride = field.FieldType.IsPointer() ? GetGenericTypeOverride(field.FieldType) : null, + Bits = [], + IsUnion = false, + UnionValues = [] + }; + } + + private static ProcessedField ProcessParameter(ParameterInfo parameter) { + return ProcessParameter( + parameter.ParameterType, + parameter.Name!, + parameter.GetCustomAttribute(), + parameter.GetCustomAttribute()); + } + + private static ProcessedField ProcessVirtualParameter(Type parameter, int i, ParameterInfo[]? parameters) { + var parameterInfo = i == 0 ? null : parameters?[i - 1]; + var name = i == 0 ? "self" : parameterInfo?.Name ?? $"a{i + 1}"; + return ProcessParameter( + parameter, + name, + parameter.GetCustomAttribute(), + parameterInfo?.GetCustomAttribute()); + } + + /// + /// Builds the type name with its generic arguments for constructed non std generic structs (ex. hkArray<hkQsTransformf>*). + /// drops the arguments as fields export them separately through template_args. + /// + private static string? GetGenericTypeOverride(Type type) { + if (type.IsFunctionPointer || type.IsUnmanagedFunctionPointer) return null; + var pointerCount = 0; + while (type.IsPointer()) { + if (type.IsPointer) type = type.GetElementType()!; + else type = type.GenericTypeArguments[0]; + if (type.IsFunctionPointer || type.IsUnmanagedFunctionPointer) return null; + pointerCount++; + } + if (!type.IsConstructedGenericType || type.IsStdDefiniton() || type.IsFixedBuffer()) return null; + var arguments = type.GenericTypeArguments.Select(t => GetGenericTypeOverride(t) ?? t.FixTypeName()); + return $"{type.FixTypeName()}<{string.Join(',', arguments)}>{new string('*', pointerCount)}"; + } + + private static ProcessedField ProcessParameter( + Type parameterType, + string parameterName, + CExporterExcelAttribute? excelAttribute, + CExporterTypeForceAttribute? typeForceAttribute) { + string? typeOverride = null; + Type[]? typeOverrideTemplates = null; + + if (parameterType is { Namespace: ExporterStatics.StdNamespacePrefix }) { + (typeOverride, typeOverrideTemplates) = ProcessStdType(parameterType); + } else if (parameterType is { Namespace: "FFXIVClientStructs.STD.Helper", Name: "Node*" }) { + typeOverride = $"std::map<{string.Join(',', parameterType.GetGenericArguments()[0].GetGenericArguments().Select(t => t.FixTypeName()))}>::iterator"; + } else if (excelAttribute != null) { + typeOverride = $"Component::Exd::Sheets::{excelAttribute.SheetName}*"; + } else if (typeForceAttribute != null) { + typeOverride = typeForceAttribute.TypeName; + } else { + typeOverride = GetGenericTypeOverride(parameterType); + _processType.Add(parameterType); + } + + return new ProcessedField { + FieldType = parameterType, + FieldOffset = -1, + FieldName = parameterName, + FieldTypeOverride = typeOverride, + FieldTypeOverrideTemplates = typeOverrideTemplates, + Bits = [], + IsUnion = false, + UnionValues = [] + }; + } + + private static object? PreProcessType(ProcessingType processingType, bool quiet) { + var (type, overrideType) = processingType; + if (type.IsFixedBuffer() || type.IsBaseType() || type.IsGenericTypeParameterOrPointer()) return null; + while (type.IsPointer) type = type.GetElementType()!; + while (type.IsGenericPointer()) type = type.GenericTypeArguments[0]; + + if (type.IsEnum) { + if (_enums.Any(t => t.EnumType == type)) return null; + var processedEnum = new ProcessedEnum { + EnumType = type, + EnumName = type.Name, + EnumNamespace = type.GetNamespace(), + IsFlags = type.GetCustomAttribute() != null, + EnumValues = type.GetFields().Where(t => t.GetCustomAttribute() == null && t.FieldType == type).ToDictionary(f => f.Name, f => f.GetRawConstantValue()!.ToString()!) + }; + + if (!quiet) Console.WriteLine($"Processed {processedEnum.EnumNamespace.Replace(".", "::")}::{processedEnum.EnumName} with {processedEnum.EnumValues.Count} fields"); + return processedEnum; + } else if (type.IsStruct()) { + while (type.IsPointer) type = type.GetElementType()!; + while (type.IsGenericPointer()) type = type.GenericTypeArguments[0]; + if (type.IsFunctionPointer || type.IsUnmanagedFunctionPointer) { + foreach (var functionPointerParameterType in type.GetFunctionPointerParameterTypes()) { + ProcessType(functionPointerParameterType, quiet); + type = type.GetFunctionPointerReturnType(); + } + } + if (!type.IsStruct() || type.IsEnum) return null; + var size = type.SizeOf(); + if (type.Name.Contains('`')) { + type = type.GetGenericTypeDefinition(); + } + var typeName = type.FixTypeName(); + var existingIndex = _structs.FindIndex(existing => + existing.StructTypeName == typeName); + var templateTypes = type.IsGenericTypeDefinition + && !type.GetGenericArguments().GenericSequenceEqual(type.DeclaringType?.GetGenericArguments() ?? []) + ? type.GetGenericArguments().Select(t => t.Name).ToArray() + : []; + if (existingIndex >= 0 && (templateTypes.Length == 0 || _structs[existingIndex].TemplateTypes.Length > 0)) + return null; + if (overrideType?.IsInStructList(_structs) ?? false) return null; + var vtable = type.GetField("VirtualTable", ExporterStatics.BindingFlags)?.FieldType; + var vtableSize = 0; + ProcessedVirtualFunction[]? virtualFunctions = null; + if (vtable != null) { + vtable = vtable.GetElementType()!; + var memberFunctions = type.GetMethods(ExporterStatics.BindingFlags).Select(t => Tuple.Create(t.Name, t.GetParameters(), t.ReturnType)).ToArray(); + virtualFunctions = vtable.GetFields(ExporterStatics.BindingFlags).Where(t => t.GetCustomAttribute() == null && t.GetCustomAttribute() == null).Select(f => { + var parameterTypes = f.FieldType.GetFunctionPointerParameterTypes(); + var memberFunction = memberFunctions.FirstOrDefault(t => t.Item1 == f.Name && t.Item2.Length == parameterTypes.Length - 1); + var returnType = f.FieldType.GetFunctionPointerReturnType(); + string? returnTypeOverride = null; + Type[] requiredTypes = []; + if (returnType is { Namespace: ExporterStatics.StdNamespacePrefix }) + (returnTypeOverride, requiredTypes) = ProcessStdType(returnType); + if (memberFunction?.Item3 != returnType) memberFunction = null; + if (returnTypeOverride != null) { + foreach (var requiredType in requiredTypes) { + _processType.Add(requiredType); + } + } else + _processType.Add(f.FieldType.GetFunctionPointerReturnType()); + return new ProcessedVirtualFunction { + VirtualFunctionName = f.Name.GetCorrectedName(), + Offset = f.GetFieldOffset(), + VirtualFunctionReturnType = f.FieldType.GetFunctionPointerReturnType(), + VirtualFunctionParameters = parameterTypes.Select((p, i) => ProcessVirtualParameter(p, i, memberFunction?.Item2)).ToArray(), + VirtualFunctionReturnTypeOverride = returnTypeOverride + }; + }).ToArray(); + vtableSize = vtable.StructLayoutAttribute?.Size ?? vtableSize; + } + + var memberFunctionClass = type.GetMember("MemberFunctionPointers", ExporterStatics.BindingFlags).FirstOrDefault()?.DeclaringType; + ProcessedMemberFunction[] memberFunctionsArray = []; + if (memberFunctionClass != null) { + var memberFunctions = memberFunctionClass.GetMethods(ExporterStatics.BindingFlags).Where(t => t.GetCustomAttribute() == null && t.GetCustomAttribute() == null).ToArray(); + foreach (var memberFunction in memberFunctions) { + var memberFunctionAddress = memberFunction.GetCustomAttribute(); + if (memberFunctionAddress == null) continue; + var memberFunctionParameters = memberFunction.GetParameters(); + string? memberFunctionOverrideType = null; + if (memberFunction.ReturnType.IsFunctionPointer) { + memberFunctionOverrideType = "__int64"; + } + var returnType = memberFunction.ReturnType; + var pointerCount = 0; + while (returnType.IsPointer()) { + pointerCount++; + returnType = returnType.GetPointerType(); + } + if (returnType is { Namespace: ExporterStatics.StdNamespacePrefix }) { + (memberFunctionOverrideType, var requiredTypes) = ProcessStdType(returnType); + foreach (var requiredType in requiredTypes) { + _processType.Add(requiredType); + } + memberFunctionOverrideType += new string('*', pointerCount); + } else { + memberFunctionOverrideType ??= GetGenericTypeOverride(memberFunction.ReturnType); + } + var memberFunctionReturnType = new ProcessedMemberFunctionReturn(memberFunction.ReturnType, memberFunctionOverrideType); + if (memberFunction.GetCustomAttribute() is not { } excelAttribute) + _processType.Add(memberFunctionReturnType.Type); + else + memberFunctionReturnType.OverrideType = $"Component::Exd::Sheets::{excelAttribute.SheetName}*"; + memberFunctionsArray = + [ + .. memberFunctionsArray, + new ProcessedMemberFunction { + MemberFunctionSignature = memberFunctionAddress!.Signature, + MemberFunctionName = memberFunction.Name.GetCorrectedName(), + MemberFunctionReturnType = memberFunctionReturnType, + MemberFunctionParameters = [ + new ProcessedField { + FieldTypeOverride = memberFunctionClass.FixTypeName() + "*", + FieldType = memberFunctionClass, + FieldOffset = -1, + FieldName = "self", + Bits = [], + IsUnion = false, + UnionValues = [] + }, + .. memberFunctionParameters.Select(ProcessParameter).ToArray()] + }, + ]; + } + } + + var fields = type.GetFields(ExporterStatics.BindingFlags).Where(t => !type.IsInheritance(t)).ToArray(); + var unionInfoFields = fields.Where(t => t.GetCustomAttribute() == null && t.GetCustomAttribute() == null && t.GetCustomAttribute() != null) + .GroupBy(t => t.GetCustomAttribute()!.Union) + .ToFrozenDictionary(t => t.Key, t => t.GroupBy(k => k.GetCustomAttribute()!.Struct).OrderBy(k => k.Key).ToFrozenDictionary(k => k.Key, k => k.ToArray())); + + var nestedTypes = ((TypeInfo)type).DeclaredNestedTypes.Where(t => !ExporterStatics.IgnoredTypeNames.Contains(t.Name) && !t.Name.EndsWith("VirtualTable", StringComparison.Ordinal) && t.GetCustomAttribute() == null && t.GetCustomAttribute() == null).ToArray(); + + var processedStruct = new ProcessedStruct { + StructType = type, + IsUnion = type.GetCustomAttribute() != null, + StructName = type.Name.IndexOf('`') != -1 ? type.Name[..type.Name.IndexOf('`')] : type.Name, + StructNamespace = type.GetNamespace(), + StructSize = size, + VirtualFunctionSize = vtableSize, + Fields = ProcessFields(fields), + VirtualFunctions = virtualFunctions, + MemberFunctions = memberFunctionsArray, + StructTypeOverride = overrideType, + TemplateTypes = templateTypes, + InlineStructs = nestedTypes.Where(t => t.IsStruct()).Select(t => (ProcessedStruct?)PreProcessType(t, quiet)).Where(t => t != null).ToArray()!, + InlineEnums = nestedTypes.Where(t => t.IsEnum).Select(t => (ProcessedEnum?)PreProcessType(t, quiet)).Where(t => t != null).ToArray()!, + IsSequential = type.StructLayoutAttribute?.Value == LayoutKind.Sequential, + StaticMemberFunctions = null, + StaticMembers = null, + }; + + foreach (var (fieldName, unionInfos) in unionInfoFields) { + var processedUnionStructFields = new List(); + FieldInfo[] unionFields; + if (unionInfos.Count > 1) { + foreach (var unionStructName in unionInfos.Keys.Where(t => t != "")) { + unionFields = unionInfos[unionStructName]; + var offsetStart = unionFields[0].GetCustomAttribute()!.Value; + var unionStructType = $"{type.GetNamespace().Replace(".", "::")}::{type.Name}_union_{unionStructName}"; + _structs.Add(new ProcessedStruct { + MemberFunctions = [], + StructName = $"{type.Name}_union_{unionStructName}", + StructNamespace = type.GetNamespace(), + StructSize = unionInfos[""][0].FieldType.SizeOf(), + StructType = type, + TemplateTypes = [], + VirtualFunctionSize = 0, + StructTypeOverride = unionStructType, + Fields = unionFields.Select(t => ProcessField(t, offsetStart)).ToArray(), + IsUnion = false, + InlineStructs = null, + InlineEnums = null, + IsSequential = false, + StaticMemberFunctions = null, + StaticMembers = null, + VirtualFunctions = null, + }); + processedUnionStructFields.Add(new ProcessedFieldUnion { + FieldType = unionFields[0].FieldType, + FieldName = unionStructName, + FieldTypeOverride = unionStructType + }); + } + } + + unionFields = unionInfos[""]; + + var unionValues = unionFields.Select(unionField => { + (string? typeOverride, Type[] typeOverrideTemplates) = (null, []); + if (unionField.FieldType is { Namespace: ExporterStatics.StdNamespacePrefix }) { + (typeOverride, typeOverrideTemplates) = ProcessStdType(unionField.FieldType); + foreach (var requiredType in typeOverrideTemplates) { + _processType.Add(requiredType); + } + } else + _processType.Add(unionField.FieldType); + + return (new ProcessedFieldUnion { + FieldName = unionField.Name, + FieldType = unionField.FieldType, + FieldTypeOverride = typeOverride, + FieldTypeOverrideTemplates = typeOverrideTemplates + }, typeOverrideTemplates); + }).ToArray(); + + processedStruct.Fields = [..processedStruct.Fields, new ProcessedField { + Bits = [], + FieldName = fieldName, + FieldOffset = unionFields[0].GetCustomAttribute()!.Value, + FieldType = unionFields[0].FieldType, + IsUnion = true, + UnionValues = unionValues.Select(t => t.Item1).Concat(processedUnionStructFields).ToArray(), + FieldTypeOverrideTemplates = unionValues.SelectMany(t => t.typeOverrideTemplates).ToArray() + }]; + } + + if (!quiet) Console.WriteLine($"Processed {processedStruct.StructTypeName} with {processedStruct.Fields.Length + (processedStruct.VirtualFunctions?.Length ?? 0) + processedStruct.MemberFunctions.Length} fields and methods"); + return processedStruct; + } + return null; + } + + public static ProcessedField[] ProcessFields(FieldInfo[] fields) { + FieldInfo[] fieldsToProcess = fields.Where(t => !ExporterStatics.IgnoredTypeNames.Contains(t.Name) && t.GetCustomAttribute() == null && t.GetCustomAttribute() == null && t.GetCustomAttribute() == null).ToArray(); + int[][] fieldsToUse = []; + bool isExcel = false; + int currentField = 0; + while (fieldsToProcess is [{ Name: "WithOps" }] or [{ Name: "Tree" }]) { + fieldsToProcess = fieldsToProcess[0].FieldType.GetFields(ExporterStatics.BindingFlags).Where(t => !ExporterStatics.IgnoredTypeNames.Contains(t.Name) && t.GetCustomAttribute() == null && t.GetCustomAttribute() == null && t.GetCustomAttribute() == null).ToArray(); + } + for (var i = 0; i < fieldsToProcess.Length; i++) { + var field = fieldsToProcess[i]; + if (field.FieldType.ContainsGenericParameters && !field.FieldType.IsGenericTypeParameterOrPointer()) continue; + if (field.GetCustomAttribute() != null) + _processType.Add(field.FieldType); + if (field.GetCustomAttribute() != null) { + isExcel = true; + fieldsToUse = [.. fieldsToUse, [currentField, i + 1]]; + continue; + } + if (field.GetCustomAttribute() != null) { + if (isExcel) + currentField = i + 1; + + isExcel = false; + continue; + } + } + if (fieldsToUse.Length == 0) return fieldsToProcess.Select(f => ProcessField(f, 0)).ToArray(); + fieldsToUse = [.. fieldsToUse, [currentField, fieldsToProcess.Length]]; + return fieldsToUse.SelectMany(t => fieldsToProcess[t[0]..t[1]].Select(f => ProcessField(f, 0))).ToArray(); + } +} diff --git a/CExporter/Exporter.cs b/CExporter/Exporter.cs index 248fbc8c3d..5a4c4612db 100644 --- a/CExporter/Exporter.cs +++ b/CExporter/Exporter.cs @@ -5,19 +5,16 @@ using System.IO; using System.Linq; using System.Reflection; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; +using System.Text.RegularExpressions; using FFXIVClientStructs.Attributes; using InteropGenerator.Runtime.Attributes; using Pastel; -using YamlDotNet.Core; -using YamlDotNet.Core.Events; using YamlDotNet.Serialization; using YamlDotNet.Serialization.NamingConventions; namespace CExporter; -public class Exporter { +public partial class Exporter { private static List _enums = []; private static List _structs = []; private static HashSet _processType = []; @@ -86,7 +83,7 @@ public static void ProcessTypes(bool quiet) { while (_processType.Count > 0) { if (!quiet) Console.WriteLine($"{PassString(count)} with {_processType.Count} structs and enum types"); var tmp = _processType - .Where(t => t.Type is { IsUnmanagedFunctionPointer: false, IsFunctionPointer: false }) + .Where(t => t.Type is { IsUnmanagedFunctionPointer: false, IsFunctionPointer: false } and not { Namespace: ExporterStatics.StdNamespacePrefix or ExporterStatics.StdHelperNamespacePrefix } || t.Type.Name.StartsWith("StdLinkedList")) .ToArray(); _processType.Clear(); foreach (var @struct in tmp) { @@ -107,56 +104,96 @@ public static void ProcessTypes(bool quiet) { } - public static void ProcessStaticFunctions(bool quiet) { - Type[] types = [.. ExporterStatics.GetHavokTypes(), .. ExporterStatics.GetXIVTypes()]; - types = types.Where(t => t.IsStruct()).Where(type => type.GetMethods(ExporterStatics.StaticBindingFlags).Length != 0).ToArray(); - if (!quiet) Console.WriteLine("::group::Processed Struct Static Members"); - var typeAndMembers = types.Select(t => (t, t.GetMethods(ExporterStatics.StaticBindingFlags))).ToArray(); - foreach (var (type, methods) in typeAndMembers) { - if (!quiet) Console.WriteLine($"Processing {type} with {methods.Length} methods"); - var sanitizedName = type.FixTypeName(); - var currentStructIndex = _structs.FindIndex(s => s.StructTypeName == sanitizedName); - if (currentStructIndex == -1) { - if (!quiet) Console.WriteLine($"Error in struct {type} please fix"); - continue; - } - var currentStruct = _structs[currentStructIndex]; - foreach (var methodInfo in methods) { - var staticAddress = methodInfo.GetCustomAttribute(); - if (staticAddress != null) { - var name = methodInfo.Name; - if (name.StartsWith("Get")) { - name = name[3..]; - } - currentStruct.StaticMembers ??= []; - currentStruct.StaticMembers = [.. currentStruct.StaticMembers, - new ProcessedStaticMembers { - Name = name, - Signature = staticAddress.Signature, - RelativeFollowOffsets = staticAddress.RelativeFollowOffsets, - IsPointer = staticAddress.IsPointer, - ReturnType = methodInfo.ReturnType.GetPointerType() - }]; + public static (ProcessedStruct s, int c) ProcessStaticFunction(ProcessedStruct currentStruct) + { + var methods = currentStruct.StructType.GetMethods(ExporterStatics.StaticBindingFlags); + foreach (var methodInfo in methods) { + var staticAddress = methodInfo.GetCustomAttribute(); + if (staticAddress != null) { + var name = methodInfo.Name; + if (name.StartsWith("Get")) { + name = name[3..]; } - var memberFunction = methodInfo.GetCustomAttribute(); - if (memberFunction != null || staticAddress != null) - _processType.Add(methodInfo.ReturnType); - if (memberFunction == null) continue; - var memberFunctionReturnType = new ProcessedMemberFunctionReturn(methodInfo.ReturnType, null); - if (methodInfo.GetCustomAttribute() is not { } excelAttribute) - _processType.Add(memberFunctionReturnType.Type); - else - memberFunctionReturnType.OverrideType = $"Component::Exd::Sheets::{excelAttribute.SheetName}*"; - currentStruct.StaticMemberFunctions ??= []; - currentStruct.StaticMemberFunctions = [.. currentStruct.StaticMemberFunctions, - new ProcessedMemberFunction { - MemberFunctionSignature = memberFunction.Signature, - MemberFunctionName = methodInfo.Name, - MemberFunctionReturnType = memberFunctionReturnType, - MemberFunctionParameters = methodInfo.GetParameters().Select(ProcessParameter).ToArray() + currentStruct.StaticMembers ??= []; + currentStruct.StaticMembers = [.. currentStruct.StaticMembers, + new ProcessedStaticMembers { + Name = name, + Signature = staticAddress.Signature, + RelativeFollowOffsets = staticAddress.RelativeFollowOffsets, + IsPointer = staticAddress.IsPointer, + ReturnType = methodInfo.ReturnType.GetPointerType() }]; } - _structs[currentStructIndex] = currentStruct; + var memberFunction = methodInfo.GetCustomAttribute(); + if (memberFunction != null || staticAddress != null) + _processType.Add(methodInfo.ReturnType); + if (memberFunction == null) continue; + var returnType = methodInfo.ReturnType; + var pointerCount = 0; + while (returnType.IsPointer()) { + pointerCount++; + returnType = returnType.GetPointerType(); + } + string? overrideType = null; + if (returnType is { Namespace: ExporterStatics.StdNamespacePrefix }) { + (overrideType, var requiredTypes) = ProcessStdType(returnType); + foreach (var requiredType in requiredTypes) { + _processType.Add(requiredType); + } + overrideType += new string('*', pointerCount); + } else { + overrideType = GetGenericTypeOverride(methodInfo.ReturnType); + } + var memberFunctionReturnType = new ProcessedMemberFunctionReturn(methodInfo.ReturnType, overrideType); + if (methodInfo.GetCustomAttribute() is not { } excelAttribute) + _processType.Add(memberFunctionReturnType.Type); + else + memberFunctionReturnType.OverrideType = $"Component::Exd::Sheets::{excelAttribute.SheetName}*"; + currentStruct.StaticMemberFunctions ??= []; + currentStruct.StaticMemberFunctions = [.. currentStruct.StaticMemberFunctions, + new ProcessedMemberFunction { + MemberFunctionSignature = memberFunction.Signature, + MemberFunctionName = methodInfo.Name, + MemberFunctionReturnType = memberFunctionReturnType, + MemberFunctionParameters = methodInfo.GetParameters().Select(ProcessParameter).ToArray() + }]; + } + return (currentStruct, methods.Length); + } + + public static (ProcessedStruct[] inlineStruct, (uint s, uint c)) ProcessStaticFunctionsInline(ProcessedStruct[] inlineStruct) + { + var membersCount = 0u; + var structCount = 0u; + for (var currentStructIndex = 0; currentStructIndex < inlineStruct.Length; currentStructIndex++) { + var (s, c) = ProcessStaticFunction(inlineStruct[currentStructIndex]); + inlineStruct[currentStructIndex] = s; + structCount++; + membersCount += (uint)c; + if(inlineStruct[currentStructIndex].InlineStructs != null) { + var (inline, (inlineCount, memberCount)) = ProcessStaticFunctionsInline(_structs[currentStructIndex].InlineStructs!); + _structs[currentStructIndex].InlineStructs = inline; + structCount += inlineCount; + membersCount += memberCount; + } + } + return (inlineStruct, (structCount, membersCount)); + } + + public static void ProcessStaticFunctions(bool quiet) { + if (!quiet) Console.WriteLine("::group::Processed Struct Static Members"); + uint structCount = 0, members = 0; + for (var currentStructIndex = 0; currentStructIndex < _structs.Count; currentStructIndex++) { + var (s, c) = ProcessStaticFunction(_structs[currentStructIndex]); + _structs[currentStructIndex] = s; + structCount++; + members += (uint)c; + if(_structs[currentStructIndex].InlineStructs != null) { + var (inline, (inlineCount, memberCount)) = ProcessStaticFunctionsInline(_structs[currentStructIndex].InlineStructs!); + _structs[currentStructIndex].InlineStructs = inline; + structCount += inlineCount; + members += memberCount; + } } if (!quiet) { @@ -186,7 +223,7 @@ public static void ProcessStaticFunctions(bool quiet) { Console.WriteLine("::endgroup::"); Console.WriteLine(); Console.WriteLine($"Processed {_enums.Count - enumsCount} enums and {_structs.Count - structsCount} structs"); - Console.WriteLine($"Processed {typeAndMembers.Length} structs with {typeAndMembers.Sum(t => t.Item2.Length)} members"); + Console.WriteLine($"Processed {structCount} structs with {members} members"); Console.WriteLine(); Console.WriteLine($"Processed total {_enums.Count} enums and {_structs.Count} structs"); } @@ -200,8 +237,6 @@ public static void VerifyNoOverlap() { var checks = sizes.Where(t => t != size && t.StartOffset <= size.StartOffset).ToArray(); if (checks.Any(t => t.EndOffset > size.StartOffset && t.StartOffset != size.StartOffset)) ExporterStatics.ErrorList.Add($"Field overlap detected in {processedStruct.StructType.FixTypeName().Pastel(Color.MediumSlateBlue)} with field {size.Field.Pastel(Color.BlueViolet)}"); - if (checks.Any(t => t.StartOffset == size.StartOffset)) - ExporterStatics.ErrorList.Add($"Union field detected but not defined in {processedStruct.StructType.FixTypeName().Pastel(Color.MediumSlateBlue)} with field {size.Field.Pastel(Color.BlueViolet)}"); if (size.StartOffset >= processedStruct.StructSize) ExporterStatics.ErrorList.Add($"Field offset exceeds struct size in {processedStruct.StructType.FixTypeName().Pastel(Color.MediumSlateBlue)} with field {size.Field.Pastel(Color.BlueViolet)}"); if (size.EndOffset > processedStruct.StructSize) @@ -233,905 +268,199 @@ public static void ProcessDefinedVTables(DataDefinition def) { foreach (var (index, name) in c.Value.vfuncs) { var offset = (int)index * 8; var vf = s.VirtualFunctions.FirstOrDefault(vf => vf.Offset == offset); + ProcessedField[]? vparams = null; + Type? vreturnType = null; + if (name == "dtor") { + vreturnType = s.StructType.MakePointerType(); + vparams = [ + new ProcessedField{ + FieldType = vreturnType, + FieldName = "self", + Bits = [], + IsUnion = false, + UnionValues = [], + FieldOffset = -1 + }, + new ProcessedField{ + FieldType = typeof(byte), + FieldName = "freeFlags", + Bits = [], + IsUnion = false, + UnionValues = [], + FieldOffset = -1 + } + ]; + } if (vf == null) - s.VirtualFunctions = [.. s.VirtualFunctions, new() { VirtualFunctionName = name, Offset = offset }]; + s.VirtualFunctions = [.. s.VirtualFunctions, new() { VirtualFunctionName = name, Offset = offset, VirtualFunctionParameters = vparams, VirtualFunctionReturnType = vreturnType, VirtualFunctionReturnTypeOverride = null }]; else if (vf.VirtualFunctionName != name) ExporterStatics.WarningList.Add($"Virtual function name mismatch: {c.Key} vf#{index}, '{vf.VirtualFunctionName}' in CS, '{name}' in data.yml"); } } } - public static void Write(DirectoryInfo dir) { - // make sure we have all the dependencies for each struct before we write them - var structs = _structs.ToFrozenDictionary(x => x.StructTypeName, x => x); + public static void VerifyDependencies() => ResolveFullDependencies(); + + private static Dictionary> ResolveFullDependencies() { + var structs = _structs.SelectMany(t => t.ChildrenAndSelf).DistinctBy(x => x.StructTypeName).ToFrozenDictionary(x => x.StructTypeName, x => x); var structToFullDeps = new Dictionary>(); - _structs.Sort((a, b) => { - var cmp = ResolveFullDependencies(a).Count.CompareTo(ResolveFullDependencies(b).Count); - return cmp != 0 ? cmp : string.Compare(a.StructTypeName, b.StructTypeName, StringComparison.Ordinal); - }); + var resolving = new List(); -#if DEBUG - for (var i = 0; i < _structs.Count; i++) { - foreach (var dep in _structs[i].DependencyNames) { - if (_structs.FindIndex(0, i, x => x.StructTypeName == dep) == -1) - throw new InvalidOperationException(); - } - } -#endif + foreach (var processedStruct in structs.Values) + resolve(processedStruct); + + return structToFullDeps; - HashSet ResolveFullDependencies(ProcessedStruct s) { + HashSet resolve(ProcessedStruct s) { if (structToFullDeps.TryGetValue(s.StructTypeName, out var deps)) return deps; - var fullDeps = s.DependencyNames.ToHashSet(); - foreach (var dn in s.DependencyNames) - fullDeps.UnionWith(ResolveFullDependencies(structs[dn])); - return structToFullDeps[s.StructTypeName] = fullDeps; - } - - var serializer = new SerializerBuilder() - .WithNamingConvention(UnderscoredNamingConvention.Instance) - .WithTypeConverter(ProcessedStructConverter.Instance) - .WithTypeConverter(ProcessedEnumConverter.Instance) - .WithTypeConverter(ProcessedFieldConverter.Instance) - .WithTypeConverter(ProcessedVirtualFunctionConverter.Instance) - .WithTypeConverter(ProcessedMemberFunctionConverter.Instance) - .Build(); - var yaml = serializer.Serialize(new YamlExport { - Enums = [.. _enums], - Structs = _structs.Select(t => t.FixOrder()).ToArray() - }); - - new FileInfo(Path.Join(dir.FullName, "ffxiv_structs.yml")).WriteFile(yaml); - } - - private static void ProcessType(ProcessingType type, bool quiet) { - var ret = PreProcessType(type, quiet); - switch (ret) { - case null: - return; - case ProcessedStruct s: - _structs.Add(s); - break; - case ProcessedEnum e: - _enums.Add(e); - break; - } - } - - private static ProcessedField ProcessField(FieldInfo field, int offset) { - if (field.FieldType is { Namespace: "FFXIVClientStructs.STD.Helper", Name: "Node*" }) { - var fieldTypeOverride = $"RedBlackTree::Node<{field.FieldType.GetGenericArguments()[0].FixTypeName()}>*"; - _processType.Add(new ProcessingType(field.FieldType, fieldTypeOverride[..^1])); - return new ProcessedField { - FieldType = field.FieldType, - FieldOffset = field.GetFieldOffset() - offset, - FieldName = field.Name, - IsBase = field.IsDirectBase(), - FieldTypeOverride = fieldTypeOverride, - Bits = [] - }; - } - if (field.FieldType.IsFunctionPointer || field.FieldType.IsUnmanagedFunctionPointer) { - _processType.Add(field.FieldType.GetFunctionPointerReturnType()); - return new ProcessedFunctionField { - FieldType = field.FieldType, - FieldOffset = field.GetFieldOffset() - offset, - FieldName = field.Name, - FunctionReturnType = field.FieldType.GetFunctionPointerReturnType(), - FunctionParameters = field.FieldType.GetFunctionPointerParameterTypes().Select((p, i) => { - _processType.Add(p); - return new ProcessedField { - FieldType = p, - FieldOffset = -1, - FieldName = 'a' + (i + 1).ToString(), - Bits = [] - }; - }).ToArray(), - Bits = [] - }; - } - if (field.FieldType.IsFixedBuffer()) { - var arr = field.GetCustomAttributes().Where(t => t.GetType().Name.Contains("FixedSizeArrayAttribute")).ToArray(); - var size = field.FieldType.StructLayoutAttribute!.Size; - if (arr.Length != 0) { - var type = arr[0].GetType(); - size = (int)type.GetProperty("Count")!.GetValue(arr[0])!; - var elementType = type.GenericTypeArguments[0]; - _processType.Add(elementType); - return new ProcessedFixedField { - FieldType = elementType, - FieldOffset = field.GetFieldOffset() - offset, - FieldName = field.Name, - FixedSize = size, - Bits = [] - }; + var cycleStart = resolving.IndexOf(s.StructTypeName); + if (cycleStart >= 0) { + var cycle = resolving.Skip(cycleStart).Append(s.StructTypeName).ToArray(); + var error = $"Cyclic type dependency detected: {string.Join(" -> ", cycle)}"; + if (!ExporterStatics.ErrorList.Contains(error)) ExporterStatics.ErrorList.Add(error); + return []; } - var fixedType = field.FieldType.GetFields()[0].FieldType; - _processType.Add(fixedType); - if (fixedType.IsBaseType()) - size /= fixedType.SizeOf(); - var fieldOverrideType = fixedType == typeof(nint) ? field.GetCustomAttribute()?.SheetName ?? null : null; - if (fieldOverrideType != null) - fieldOverrideType = $"Component::Exd::Sheets::{fieldOverrideType}*"; - return new ProcessedFixedField { - FieldType = fixedType, - FieldOffset = field.GetFieldOffset() - offset, - FieldName = field.Name, - FixedSize = size, - FieldTypeOverride = fieldOverrideType, - Bits = [] - }; - } - if (field.FieldType.GetCustomAttribute() != null) { - var arrLength = field.FieldType.GetCustomAttribute()!.Length; - var elementType = field.FieldType.GetGenericArguments()[0]; - var isString = field.GetCustomAttribute()?.IsString ?? false; - var fieldOverrideType = elementType == typeof(nint) ? field.GetCustomAttribute()?.SheetName ?? null : null; - if (fieldOverrideType != null) - fieldOverrideType = $"Component::Exd::Sheets::{fieldOverrideType}*"; - if (isString && elementType == typeof(byte)) - fieldOverrideType = "char"; - _processType.Add(elementType); - return new ProcessedFixedField { - FieldType = elementType, - FieldOffset = field.GetFieldOffset() - offset, - FieldName = field.Name[1].ToString().ToUpper() + field.Name[2..], - FixedSize = arrLength, - FixedString = isString, - FieldTypeOverride = fieldOverrideType, - Bits = [] - }; - } - if (field.GetCustomAttribute() != null) { - var sheetName = field.GetCustomAttribute()!.SheetName; - return new ProcessedField { - FieldType = field.FieldType, - FieldOffset = field.GetFieldOffset() - offset, - FieldName = $"{sheetName}Sheet", - FieldTypeOverride = $"Component::Exd::Sheets::{sheetName}", - Bits = [] - }; - } - if (field.GetCustomAttribute() != null) { - var sheetName = field.GetCustomAttribute()!.SheetName; - return new ProcessedField { - FieldType = field.FieldType, - FieldOffset = field.GetFieldOffset() - offset, - FieldName = field.Name, - FieldTypeOverride = $"Component::Exd::Sheets::{sheetName}*", - Bits = [] - }; - } - if (field.GetCustomAttribute() is { } typeForceAttribute) { - return new ProcessedField { - FieldType = field.FieldType, - FieldOffset = field.GetFieldOffset() - offset, - FieldName = field.Name, - FieldTypeOverride = typeForceAttribute.TypeName, - FieldTypeOverrideCheck = typeForceAttribute.DontCheck, - Bits = [] - }; - } - if (field.IsBitArray()) { - var bitsDefinition = field.GetCustomAttributes().Where(t => t.GetType().Name.Contains("BitFieldAttribute`1")).Select(t => { - var attrType = t.GetType(); - var name = (string)attrType.GetProperty("Name")!.GetValue(t)!; - var offset = (int)attrType.GetProperty("Index")!.GetValue(t)!; - var size = (int)attrType.GetProperty("Length")!.GetValue(t)!; - var type = attrType.GenericTypeArguments[0]; - _processType.Add(type); - return new ProcessedBitField { - Name = name, - Offset = offset, - Size = size, - Type = ExporterStatics.GetBestMatchFromSize(size) - }; - }).ToArray(); - var bits = new List(); - foreach (var bit in bitsDefinition) { - if (bits.Count == 0 && bit.Offset != 0) - bits.Add(new ProcessedBitField { - Name = "UnkBits0", - Offset = 0, - Size = bit.Offset, - Type = ExporterStatics.GetBestMatchFromSize(bit.Offset) - }); - else if (bit.Offset == 0) { - bits.Add(bit); - continue; - } - var last = bits.Last(); - if (last.Offset + last.Size != bit.Offset) { - var newOffset = last.Offset + last.Size; - var size = bit.Offset - newOffset; - - bits.Add(new ProcessedBitField { - Name = $"UnkBits{newOffset}", - Offset = newOffset, - Size = size, - Type = ExporterStatics.GetBestMatchFromSize(size) - }); - } - bits.Add(bit); - } - return new ProcessedField { - FieldType = field.FieldType, - FieldOffset = field.GetFieldOffset() - offset, - FieldName = field.Name, - IsBase = field.IsDirectBase(), - Bits = bits.ToArray() - }; - } - _processType.Add(field.FieldType); - return new ProcessedField { - FieldType = field.FieldType, - FieldOffset = field.GetFieldOffset() - offset, - FieldName = field.Name, - IsBase = field.IsDirectBase(), - Bits = [] - }; - } - - private static ProcessedField ProcessParameter(ParameterInfo parameter) { - if (parameter.ParameterType is { Namespace: "FFXIVClientStructs.STD.Helper", Name: "Node*" }) { - var fieldTypeOverride = $"RedBlackTree::Node<{parameter.ParameterType.GetGenericArguments()[0].FixTypeName()}>*"; - _processType.Add(new ProcessingType(parameter.ParameterType, fieldTypeOverride[..^1])); - return new ProcessedField { - FieldType = parameter.ParameterType, - FieldOffset = -1, - FieldName = parameter.Name!, - FieldTypeOverride = fieldTypeOverride, - Bits = [] - }; - } - if (parameter.GetCustomAttribute() != null) { - var sheetName = parameter.GetCustomAttribute()!.SheetName; - return new ProcessedField { - FieldType = parameter.ParameterType, - FieldOffset = -1, - FieldName = parameter.Name!, - FieldTypeOverride = $"Component::Exd::Sheets::{sheetName}*", - Bits = [] - }; - } - if (parameter.GetCustomAttribute() != null) { - return new ProcessedField { - FieldType = parameter.ParameterType, - FieldOffset = -1, - FieldName = parameter.Name!, - FieldTypeOverride = parameter.GetCustomAttribute()!.TypeName, - Bits = [] - }; - } - _processType.Add(parameter.ParameterType); - return new ProcessedField { - FieldType = parameter.ParameterType, - FieldOffset = -1, - FieldName = parameter.Name!, - Bits = [] - }; - } - - private static ProcessedField ProcessVirtualParameter(Type parameter, int i, ParameterInfo[]? parameters) { - if (parameter is { Namespace: "FFXIVClientStructs.STD.Helper", Name: "Node*" }) { - var fieldTypeOverride = $"RedBlackTree::Node<{parameter.GetGenericArguments()[0].FixTypeName()}>*"; - _processType.Add(new ProcessingType(parameter, fieldTypeOverride[..^1])); - return new ProcessedField { - FieldType = parameter, - FieldOffset = -1, - FieldName = i == 0 ? "this" : parameters?[i - 1].Name ?? $"a{i + 1}", - FieldTypeOverride = fieldTypeOverride, - Bits = [] - }; - } - if (parameter.GetCustomAttribute() != null) { - var sheetName = parameter.GetCustomAttribute()!.SheetName; - return new ProcessedField { - FieldType = parameter, - FieldOffset = -1, - FieldName = i == 0 ? "this" : parameters?[i - 1].Name ?? $"a{i + 1}", - FieldTypeOverride = $"Component::Exd::Sheets::{sheetName}*", - Bits = [] - }; - } - if (i != 0 && parameters?[i - 1] is not null && parameters[i - 1].GetCustomAttribute() != null) { - return new ProcessedField { - FieldType = parameter, - FieldOffset = -1, - FieldName = parameters[i - 1].Name ?? $"a{i + 1}", - FieldTypeOverride = parameters[i - 1].GetCustomAttribute()!.TypeName, - Bits = [] - }; - } - _processType.Add(parameter); - return new ProcessedField { - FieldType = parameter, - FieldOffset = -1, - FieldName = i == 0 ? "this" : parameters?[i - 1].Name ?? $"a{i + 1}", - Bits = [] - }; - } - private static object? PreProcessType(ProcessingType processingType, bool quiet) { - var (type, overrideType) = processingType; - if (type.IsFixedBuffer() || type.IsBaseType()) return null; - while (type.IsPointer) type = type.GetElementType()!; - while (type.IsGenericPointer()) type = type.GenericTypeArguments[0]; - - if (type.IsEnum) { - if (_enums.Any(t => t.EnumType == type)) return null; - var processedEnum = new ProcessedEnum { - EnumType = type, - EnumName = type.Name, - EnumNamespace = type.GetNamespace(), - IsFlags = type.GetCustomAttribute() != null, - EnumValues = type.GetFields().Where(t => t.GetCustomAttribute() == null && t.FieldType == type).ToDictionary(f => f.Name, f => f.GetRawConstantValue()!.ToString()!) - }; - - if (!quiet) Console.WriteLine($"Processed {processedEnum.EnumNamespace}::{processedEnum.EnumName} with {processedEnum.EnumValues.Count} fields"); - _enums.Add(processedEnum); - } else if (type.IsStruct()) { - while (type.IsPointer) type = type.GetElementType()!; - while (type.IsGenericPointer()) type = type.GenericTypeArguments[0]; - if (type.IsFunctionPointer || type.IsUnmanagedFunctionPointer) { - foreach (var functionPointerParameterType in type.GetFunctionPointerParameterTypes()) { - ProcessType(functionPointerParameterType, quiet); - type = type.GetFunctionPointerReturnType(); + resolving.Add(s.StructTypeName); + var fullDeps = s.DependencyNames.ToHashSet(); + foreach (var dependencyName in s.DependencyNames) { + if (!structs.TryGetValue(dependencyName, out var dependency)) { + var error = $"Unable to resolve type dependency {dependencyName} referenced by {s.StructTypeName}"; + if (!ExporterStatics.ErrorList.Contains(error)) ExporterStatics.ErrorList.Add(error); + continue; } - } - if (!type.IsStruct() || type.IsEnum) return null; - if (type.IsInStructList(_structs) || (overrideType?.IsInStructList(_structs) ?? false)) return null; - var vtable = type.GetField("VirtualTable", ExporterStatics.BindingFlags)?.FieldType; - var vtableSize = 0; - ProcessedVirtualFunction[]? virtualFunctions = null; - if (vtable != null) { - vtable = vtable.GetElementType()!; - var memberFunctions = type.GetMethods(ExporterStatics.BindingFlags).Select(t => Tuple.Create(t.Name, t.GetParameters(), t.ReturnType)).ToArray(); - virtualFunctions = vtable.GetFields(ExporterStatics.BindingFlags).Where(t => t.GetCustomAttribute() == null && t.GetCustomAttribute() == null).Select(f => { - var parameterTypes = f.FieldType.GetFunctionPointerParameterTypes(); - var memberFunction = memberFunctions.FirstOrDefault(t => t.Item1 == f.Name && t.Item2.Length == parameterTypes.Length - 1); - var returnType = f.FieldType.GetFunctionPointerReturnType(); - if (memberFunction?.Item3 != returnType) memberFunction = null; - _processType.Add(f.FieldType.GetFunctionPointerReturnType()); - var vfName = f.Name; - if (vfName.StartsWith("Ctor") || vfName.StartsWith("Dtor")) - vfName = char.ToLowerInvariant(vfName[0]) + vfName[1..]; // lowercase ctor/dtor - return new ProcessedVirtualFunction { - VirtualFunctionName = vfName, - Offset = f.GetFieldOffset(), - VirtualFunctionReturnType = f.FieldType.GetFunctionPointerReturnType(), - VirtualFunctionParameters = parameterTypes.Select((p, i) => ProcessVirtualParameter(p, i, memberFunction?.Item2)).ToArray() - }; - }).ToArray(); - vtableSize = vtable.StructLayoutAttribute?.Size ?? vtableSize; - } - var memberFunctionClass = type.GetMember("MemberFunctionPointers", ExporterStatics.BindingFlags).FirstOrDefault()?.DeclaringType; - ProcessedMemberFunction[] memberFunctionsArray = []; - //if (type == typeof(lua_State)) Debugger.Break(); - if (memberFunctionClass != null) { - var memberFunctions = memberFunctionClass.GetMethods(ExporterStatics.BindingFlags).Where(t => t.GetCustomAttribute() == null && t.GetCustomAttribute() == null).ToArray(); - foreach (var memberFunction in memberFunctions) { - var memberFunctionAddress = memberFunction.GetCustomAttribute(); - if (memberFunctionAddress == null) continue; - var memberFunctionParameters = memberFunction.GetParameters(); - string? memberFunctionOverrideType = null; - if (memberFunction.ReturnType.IsFunctionPointer) { - memberFunctionOverrideType = "__int64"; + fullDeps.UnionWith(resolve(dependency)); + if (dependency.StructType.DeclaringType is not null) { + var owner = dependency.StructType; + while (owner.DeclaringType is not null) owner = owner.DeclaringType; + var ownerName = owner.FixTypeName(); + var sourceOwner = s.StructType; + while (sourceOwner.DeclaringType is not null) sourceOwner = sourceOwner.DeclaringType; + if (ownerName != sourceOwner.FixTypeName() && structs.TryGetValue(ownerName, out var ownerStruct)) { + fullDeps.Add(ownerName); + fullDeps.UnionWith(resolve(ownerStruct)); } - var memberFunctionReturnType = new ProcessedMemberFunctionReturn(memberFunction.ReturnType, memberFunctionOverrideType); - if (memberFunction.GetCustomAttribute() is not { } excelAttribute) - _processType.Add(memberFunctionReturnType.Type); - else - memberFunctionReturnType.OverrideType = $"Component::Exd::Sheets::{excelAttribute.SheetName}*"; - var funcName = memberFunction.Name; - if (funcName.StartsWith("Ctor") || funcName.StartsWith("Dtor")) - funcName = char.ToLowerInvariant(funcName[0]) + funcName[1..]; // lowercase ctor/dtor - memberFunctionsArray = - [ - .. memberFunctionsArray, - new ProcessedMemberFunction { - MemberFunctionSignature = memberFunctionAddress!.Signature, - MemberFunctionName = funcName, - MemberFunctionReturnType = memberFunctionReturnType, - MemberFunctionParameters = [ - new ProcessedField { - FieldTypeOverride = memberFunctionClass.FixTypeName() + "*", - FieldType = memberFunctionClass, - FieldOffset = -1, - FieldName = "this", - Bits = [] - }, - .. memberFunctionParameters.Select(ProcessParameter).ToArray()] - }, - ]; } } - - var fields = type.GetFields(ExporterStatics.BindingFlags).Where(t => !type.IsInheritance(t)).ToArray(); - var unionFields = fields.Where(t => t.GetCustomAttribute() == null && t.GetCustomAttribute() == null && t.GetCustomAttribute() != null).ToArray(); - - var unionOffsets = new Dictionary(new CExporterUnionCompare()); - - if (unionFields.Any()) { - var unions = new List(); - foreach (var unionField in unionFields) { - var attr = unionField.GetCustomAttribute()!; - if (!unionOffsets.TryGetValue(attr, out var unionStartField)) { - unionOffsets[attr] = unionField; - unionStartField = unionField; - } - var index = attr.IsStruct ? unions.FindIndex(t => t.StructName == attr.Struct) : unions.FindIndex(t => t.StructName == attr.Union); - if (index == -1) { - unions.Add(new ProcessedStruct { - StructType = type, - IsUnion = !attr.IsStruct, - StructName = attr.IsStruct ? attr.Struct : attr.Union, - StructNamespace = type.FixTypeName() + (attr.IsStruct ? $"{ExporterStatics.Separator}{attr.Union}" : ""), - StructTypeName = type.FixTypeName() + $"{ExporterStatics.Separator}{attr.Union}{(attr.IsStruct ? $"{ExporterStatics.Separator}{attr.Struct}" : "")}", - StructSize = 0, - VirtualFunctionSize = 0, - Fields = [ - ProcessField(unionField, unionStartField.GetFieldOffset()) - ], - MemberFunctions = [], - StructTypeOverride = type.FixTypeName() + $"{ExporterStatics.Separator}{attr.Union}{(attr.IsStruct ? $"{ExporterStatics.Separator}{attr.Struct}" : "")}" - }); - } else - unions[index].Fields = [ - .. unions[index].Fields, - ProcessField(unionField, unionStartField.GetFieldOffset()) - ]; - _processType.Add(unionField.FieldType); - } - foreach (var t1 in unions) { - t1.StructSize = t1.Fields.Max(t => t.FieldOffset + t.FieldType.SizeOf()); - } - var subUnions = unions.Where(t => !t.IsUnion).ToArray(); - foreach (var subStruct in subUnions) { - var unionName = subStruct.StructNamespace.Split(ExporterStatics.Separator)[^1]; - var unionNamespace = subStruct.StructNamespace[..subStruct.StructNamespace.LastIndexOf(ExporterStatics.Separator, StringComparison.Ordinal)]; - var unionStructIndex = unions.FindIndex(t => t.StructNamespace == unionNamespace && t.StructName == unionName); - var unionStruct = unions[unionStructIndex]; - unionStruct.Fields = [ - .. unionStruct.Fields, - new ProcessedField { - FieldTypeOverride = subStruct.StructTypeOverride, - FieldType = type, - FieldOffset = 0, - FieldName = subStruct.StructName, - Bits = [] - } - ]; - if (!quiet) Console.WriteLine($"Processed {unionStruct.StructTypeOverride} with {unionStruct.Fields.Length} fields and methods"); - unions[unionStructIndex] = unionStruct; - } - _structs.AddRange(unions); - } - - var processedStruct = new ProcessedStruct { - StructType = type, - IsUnion = type.GetCustomAttribute() != null, - StructName = type.Name, - StructNamespace = type.GetNamespace(), - StructTypeName = overrideType ?? type.FixTypeName(), - StructSize = type.SizeOf(), - VirtualFunctionSize = vtableSize, - Fields = ProcessFields(fields), - VirtualFunctions = virtualFunctions, - MemberFunctions = memberFunctionsArray, - StructTypeOverride = overrideType - }; - - foreach (var (unionAttr, fieldInfo) in unionOffsets.Where(t => !t.Key.IsStruct)) { - processedStruct.Fields = [ - .. processedStruct.Fields, - new ProcessedField { - FieldType = fieldInfo.FieldType, - FieldOffset = fieldInfo.GetFieldOffset(), - FieldName = unionAttr.Union, - FieldTypeOverride = type.FixTypeName() + $"{ExporterStatics.Separator}{unionAttr.Union}", - Bits = [] - } - ]; - processedStruct.Fields = [.. processedStruct.Fields.OrderBy(t => t.FieldOffset)]; - } - - if (!quiet) Console.WriteLine($"Processed {processedStruct.StructTypeName} with {processedStruct.Fields.Length + (processedStruct.VirtualFunctions?.Length ?? 0) + processedStruct.MemberFunctions.Length} fields and methods"); - _structs.Add(processedStruct); + resolving.RemoveAt(resolving.Count - 1); + return structToFullDeps[s.StructTypeName] = fullDeps; } - return null; } - public static ProcessedField[] ProcessFields(FieldInfo[] fields) { - FieldInfo[] fieldsToProcess = fields.Where(t => !ExporterStatics.IgnoredTypeNames.Contains(t.Name) && t.GetCustomAttribute() == null && t.GetCustomAttribute() == null && t.GetCustomAttribute() == null).ToArray(); - int[][] fieldsToUse = []; - bool isExcel = false; - int currentField = 0; - while (fieldsToProcess is [{ Name: "WithOps" }] or [{ Name: "Tree" }]) { - fieldsToProcess = fieldsToProcess[0].FieldType.GetFields(ExporterStatics.BindingFlags).Where(t => !ExporterStatics.IgnoredTypeNames.Contains(t.Name) && t.GetCustomAttribute() == null && t.GetCustomAttribute() == null && t.GetCustomAttribute() == null).ToArray(); - } - for (var i = 0; i < fieldsToProcess.Length; i++) { - var field = fieldsToProcess[i]; - if (field.GetCustomAttribute() != null) - _processType.Add(field.FieldType); - if (field.GetCustomAttribute() != null) { - isExcel = true; - fieldsToUse = [.. fieldsToUse, [currentField, i + 1]]; - continue; - } - if (field.GetCustomAttribute() != null) { - if (isExcel) - currentField = i + 1; - - isExcel = false; - continue; + [GeneratedRegex(@"[A-Za-z_]\w*(::[A-Za-z_]\w*)*")] + private static partial Regex TypeNameRegex(); + + // A nested type can't be forward declared in C++, so any reference to one (even through a pointer or a + // function signature) needs the struct that declares it to be emitted first. + private static void OrderByNestedTypeReferences(Dictionary> structToFullDeps) { + var inlineOwners = new Dictionary(); + var inlineTypes = _structs.SelectMany(t => t.ChildrenAndSelf.Skip(1)).Select(t => t.StructTypeName).ToHashSet(); + var topLevel = _structs.Where(t => !inlineTypes.Contains(t.StructTypeName)).ToArray(); + foreach (var owner in topLevel) + foreach (var inline in owner.ChildrenAndSelf.Skip(1)) + inlineOwners.TryAdd(inline.StructTypeName, owner.StructTypeName); + + var topLevelNames = topLevel.Select(t => t.StructTypeName).ToHashSet(); + var dependencies = new Dictionary>(); + foreach (var s in topLevel) { + var deps = new HashSet(); + foreach (var dependency in structToFullDeps[s.StructTypeName]) + addDependency(dependency); + foreach (var reference in s.ChildrenAndSelf.SelectMany(typeReferences)) + foreach (Match match in TypeNameRegex().Matches(reference)) + if (inlineOwners.ContainsKey(match.Value)) addDependency(match.Value); + dependencies[s.StructTypeName] = deps; + + void addDependency(string name) { + if (inlineOwners.TryGetValue(name, out var owner)) name = owner; + if (name != s.StructTypeName && topLevelNames.Contains(name)) deps.Add(name); } } - if (fieldsToUse.Length == 0) return fieldsToProcess.Select(f => ProcessField(f, 0)).ToArray(); - fieldsToUse = [.. fieldsToUse, [currentField, fieldsToProcess.Length]]; - return fieldsToUse.SelectMany(t => fieldsToProcess[t[0]..t[1]].Select(f => ProcessField(f, 0))).ToArray(); - } -} - -public class YamlExport { - public required ProcessedEnum[] Enums; - public required ProcessedStruct[] Structs; -} - -public class ProcessedBitField { - public required int Size; - public required int Offset; - public required Type Type; - public required string Name; -} - -public class ProcessedField { - public string? FieldTypeOverride; - public bool? FieldTypeOverrideCheck; - public required Type FieldType; - public required string FieldName; - public required int FieldOffset; - public bool IsBase; - public required ProcessedBitField[] Bits; - public virtual int FieldSize => FieldType.SizeOf(); -} - -public class ProcessedFixedField : ProcessedField { - public required int FixedSize; - public bool FixedString; - - public override int FieldSize => FixedSize * FieldType.SizeOf(); -} - -public class ProcessedFunctionField : ProcessedField { - public required Type FunctionReturnType; - public required ProcessedField[] FunctionParameters; -} -public class ProcessedStaticMembers { - public required string Name; - public required string Signature; - public required ushort[] RelativeFollowOffsets; - public bool IsPointer; - public required Type ReturnType; -} - -public class ProcessedStruct { - public string? StructTypeOverride; - public bool IsUnion; - public required Type StructType; - public required string StructName; - public required string StructNamespace; - public required int StructSize; - public required ProcessedField[] Fields; - public required int VirtualFunctionSize; - public ProcessedVirtualFunction[]? VirtualFunctions; // null if there are no virtual functions, empty if there's a vtable with unknown contents - public required ProcessedMemberFunction[] MemberFunctions; - public ProcessedMemberFunction[]? StaticMemberFunctions; - public ProcessedStaticMembers[]? StaticMembers; - [YamlIgnore] - public required string StructTypeName; - [YamlIgnore] - private string[] _dependencyNames = []; - [YamlIgnore] - public string[] DependencyNames { - get { - if (_dependencyNames.Length == 0) { - _dependencyNames = Fields - .Where(t => { - if (t.FieldTypeOverride != null) { - if (t.FieldTypeOverride.EndsWith('*') || t.FieldTypeOverride.StartsWith("Component::Exd::Sheets::") || ExporterStatics.BaseTypeNames.Contains(t.FieldTypeOverride)) - return false; - return !t.FieldTypeOverrideCheck.HasValue || !t.FieldTypeOverrideCheck.Value; - } - return (t.GetType() == typeof(ProcessedField) || t.GetType() == typeof(ProcessedFixedField)) && (!(t.FieldType.IsPointer() || t.FieldType.IsPrimitive || t.FieldType.IsFixedBuffer() || t.FieldType.IsEnum || t.FieldType.IsBaseType())); - }) - .Select(t => t.FieldTypeOverride ?? t.FieldType.FixTypeName()).Distinct().ToArray(); + // keeps the existing order wherever it already satisfies the dependencies + var remaining = topLevel.ToList(); + var ordered = new List(_structs.Count); + var emitted = new HashSet(); + while (remaining.Count > 0) { + var index = remaining.FindIndex(t => dependencies[t.StructTypeName].IsSubsetOf(emitted)); + if (index < 0) index = 0; // cyclic, the importer falls back to an opaque pointer for these + ordered.Add(remaining[index]); + emitted.Add(remaining[index].StructTypeName); + remaining.RemoveAt(index); + } + ordered.AddRange(_structs.Where(t => inlineTypes.Contains(t.StructTypeName))); + _structs = ordered; + + static IEnumerable typeReferences(ProcessedStruct s) { + foreach (var field in s.Fields) { + foreach (var reference in fieldReferences(field)) + yield return reference; + if (field is not ProcessedFunctionField functionField) continue; + yield return functionField.FunctionReturnType.FixTypeName(); + foreach (var reference in functionField.FunctionParameters.SelectMany(fieldReferences)) + yield return reference; } - return _dependencyNames; - } - } - public ProcessedStruct FixOrder() { - Fields = [.. Fields.OrderBy(t => t.FieldOffset)]; - return this; - } -} - -public class ProcessedEnum { - public required Type EnumType; - public required string EnumName; - public required string EnumNamespace; - public required bool IsFlags; - public required Dictionary EnumValues; -} - -public class ProcessedVirtualFunction { - public required string VirtualFunctionName; - public required int Offset; - public Type? VirtualFunctionReturnType; - public ProcessedField[]? VirtualFunctionParameters; -} - -public class ProcessedMemberFunction { - public required string MemberFunctionSignature; - public required string MemberFunctionName; - public required ProcessedMemberFunctionReturn MemberFunctionReturnType; - public required ProcessedField[] MemberFunctionParameters; -} - -public class ProcessedMemberFunctionReturn(Type type, string? overrideType) { - public Type Type = type; - public string? OverrideType = overrideType; - public static implicit operator ProcessedMemberFunctionReturn((Type Type, string OverrideType) value) => new(value.Type, value.OverrideType); - public static implicit operator (Type Type, string? OverrideType)(ProcessedMemberFunctionReturn value) => (value.Type, value.OverrideType); - public static implicit operator ProcessedMemberFunctionReturn(Type type) => new(type, null); - public static implicit operator Type(ProcessedMemberFunctionReturn value) => value.Type; - public static implicit operator string(ProcessedMemberFunctionReturn value) => value.OverrideType ?? value.Type.FixTypeName(); -} - -public class ProcessedEnumConverter : IYamlTypeConverter { - public bool Accepts(Type type) => type == typeof(ProcessedEnum); - public object? ReadYaml(IParser parser, Type type, ObjectDeserializer rootDeserializer) => throw new NotImplementedException(); - public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializer serializer) { - if (value is not ProcessedEnum e) return; - emitter.Emit(new MappingStart()); - emitter.Emit(new Scalar("type")); - emitter.Emit(new Scalar(e.EnumType.FixTypeName())); - emitter.Emit(new Scalar("name")); - emitter.Emit(new Scalar(e.EnumName)); - emitter.Emit(new Scalar("underlying")); - emitter.Emit(new Scalar(e.EnumType.GetEnumUnderlyingType().FixTypeName())); - emitter.Emit(new Scalar("namespace")); - emitter.Emit(new Scalar(e.EnumNamespace)); - emitter.Emit(new Scalar("flags")); - emitter.Emit(e.IsFlags ? new Scalar("True") : new Scalar("False")); - emitter.Emit(new Scalar("values")); - emitter.Emit(new MappingStart()); - foreach (var (key, val) in e.EnumValues) { - emitter.Emit(new Scalar(AnchorName.Empty, TagName.Empty, key, ScalarStyle.DoubleQuoted, true, false)); - emitter.Emit(new Scalar(val)); - } - emitter.Emit(new MappingEnd()); - emitter.Emit(new MappingEnd()); - } - - public static readonly IYamlTypeConverter Instance = new ProcessedEnumConverter(); -} - -public class ProcessedFieldConverter : IYamlTypeConverter { - public bool Accepts(Type type) => type == typeof(ProcessedField); - public object? ReadYaml(IParser parser, Type type, ObjectDeserializer rootDeserializer) => throw new NotImplementedException(); - public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializer serializer) { - if (value is not ProcessedField f) return; - emitter.Emit(new MappingStart()); - emitter.Emit(new Scalar("type")); - if (f.FieldType.IsFunctionPointer || f.FieldType.IsUnmanagedFunctionPointer) { - emitter.Emit(new Scalar("__fastcall")); - } else { - emitter.Emit(new Scalar(f.FieldTypeOverride ?? f.FieldType.FixTypeName())); - } - emitter.Emit(new Scalar("name")); - emitter.Emit(new Scalar(f.FieldName)); - if (f.FieldOffset >= 0) { - emitter.Emit(new Scalar("offset")); - emitter.Emit(new Scalar(f.FieldOffset.ToString())); - } - if (f.Bits.Length > 0) { - emitter.Emit(new Scalar("bits")); - emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); - foreach (var bit in f.Bits) { - emitter.Emit(new MappingStart()); - emitter.Emit(new Scalar("type")); - emitter.Emit(new Scalar(bit.Type.FixTypeName())); - emitter.Emit(new Scalar("offset")); - emitter.Emit(new Scalar(bit.Offset.ToString())); - emitter.Emit(new Scalar("size")); - emitter.Emit(new Scalar(bit.Size.ToString())); - emitter.Emit(new Scalar("name")); - emitter.Emit(new Scalar(bit.Name)); - emitter.Emit(new MappingEnd()); + foreach (var virtualFunction in s.VirtualFunctions ?? []) { + if (virtualFunction.VirtualFunctionReturnType != null) + yield return virtualFunction.VirtualFunctionReturnTypeOverride ?? virtualFunction.VirtualFunctionReturnType.FixTypeName(); + foreach (var reference in (virtualFunction.VirtualFunctionParameters ?? []).SelectMany(fieldReferences)) + yield return reference; } - emitter.Emit(new SequenceEnd()); } - if (f.IsBase) { - emitter.Emit(new Scalar("base")); - emitter.Emit(new Scalar("true")); - } - switch (f) { - case ProcessedFunctionField func: { - emitter.Emit(new Scalar("return_type")); - emitter.Emit(new Scalar(func.FunctionReturnType.FixTypeName())); - emitter.Emit(new Scalar("parameters")); - emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); - foreach (var p in func.FunctionParameters) { - emitter.Emit(new MappingStart()); - emitter.Emit(new Scalar("type")); - emitter.Emit(new Scalar(p.FieldTypeOverride ?? p.FieldType.FixTypeName())); - emitter.Emit(new Scalar("name")); - emitter.Emit(new Scalar(p.FieldName)); - emitter.Emit(new MappingEnd()); - } - emitter.Emit(new SequenceEnd()); - break; - } - case ProcessedFixedField fix: - emitter.Emit(new Scalar("size")); - emitter.Emit(new Scalar(fix.FixedSize.ToString())); - emitter.Emit(new Scalar("is_string")); - emitter.Emit(new Scalar(fix.FixedString.ToString())); - break; - } - emitter.Emit(new MappingEnd()); - } - - public static readonly IYamlTypeConverter Instance = new ProcessedFieldConverter(); -} -public class ProcessedStructConverter : IYamlTypeConverter { - public bool Accepts(Type type) => type == typeof(ProcessedStruct); - public object? ReadYaml(IParser parser, Type type, ObjectDeserializer rootDeserializer) => throw new NotImplementedException(); - public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializer serializer) { - if (value is not ProcessedStruct s) return; - emitter.Emit(new MappingStart()); - emitter.Emit(new Scalar("type")); - emitter.Emit(new Scalar(!string.IsNullOrWhiteSpace(s.StructTypeOverride) ? s.StructTypeOverride : s.StructType.FixTypeName())); - emitter.Emit(new Scalar("name")); - emitter.Emit(new Scalar(s.StructName)); - emitter.Emit(new Scalar("namespace")); - emitter.Emit(new Scalar(s.StructNamespace)); - emitter.Emit(new Scalar("union")); - emitter.Emit(new Scalar(s.IsUnion.ToString())); - if (s.StructType.StructLayoutAttribute?.Value == LayoutKind.Explicit && s.StructSize != 0) { - emitter.Emit(new Scalar("size")); - emitter.Emit(new Scalar(s.StructSize.ToString())); - } - emitter.Emit(new Scalar("fields")); - emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); - foreach (var field in s.Fields) { - ProcessedFieldConverter.Instance.WriteYaml(emitter, field, field.GetType(), serializer); - } - emitter.Emit(new SequenceEnd()); - if (s.VirtualFunctionSize != 0) { - emitter.Emit(new Scalar("vtable_size")); - emitter.Emit(new Scalar(s.VirtualFunctionSize.ToString())); - } - if (s.VirtualFunctions != null) { - emitter.Emit(new Scalar("virtual_functions")); - emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); - foreach (var virtualFunction in s.VirtualFunctions) { - ProcessedVirtualFunctionConverter.Instance.WriteYaml(emitter, virtualFunction, virtualFunction.GetType(), serializer); + static IEnumerable fieldReferences(ProcessedField field) { + if (field.FieldType.IsFunctionPointer || field.FieldType.IsUnmanagedFunctionPointer) yield break; + yield return field.FieldTypeOverride ?? field.FieldType.FixTypeName(); + var templateArguments = field.FieldType.IsConstructedGenericType && !field.FieldType.IsStdDefiniton() + ? field.FieldType.GenericTypeArguments + : field.FieldTypeOverrideTemplates ?? []; + foreach (var templateArgument in templateArguments) + yield return templateArgument.FixTypeName(); + foreach (var unionValue in field.UnionValues) { + if (unionValue.FieldType.IsFunctionPointer || unionValue.FieldType.IsUnmanagedFunctionPointer) continue; + yield return unionValue.FieldTypeOverride ?? unionValue.FieldType.FixTypeName(); } - emitter.Emit(new SequenceEnd()); - } - emitter.Emit(new Scalar("member_functions")); - emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); - foreach (var memberFunction in s.MemberFunctions) { - ProcessedMemberFunctionConverter.Instance.WriteYaml(emitter, memberFunction, memberFunction.GetType(), serializer); } - emitter.Emit(new SequenceEnd()); - if (s.StaticMembers != null) { - emitter.Emit(new Scalar("static_members")); - emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); - foreach (var staticMember in s.StaticMembers) { - ProcessedStaticMembersConverter.Instance.WriteYaml(emitter, staticMember, staticMember.GetType(), serializer); - } - emitter.Emit(new SequenceEnd()); - } - if (s.StaticMemberFunctions != null) { - emitter.Emit(new Scalar("static_member_functions")); - emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); - foreach (var staticMemberFunction in s.StaticMemberFunctions) { - ProcessedMemberFunctionConverter.Instance.WriteYaml(emitter, staticMemberFunction, staticMemberFunction.GetType(), serializer); - } - emitter.Emit(new SequenceEnd()); - } - emitter.Emit(new MappingEnd()); } - public static readonly IYamlTypeConverter Instance = new ProcessedStructConverter(); -} - -public class ProcessedMemberFunctionConverter : IYamlTypeConverter { - public bool Accepts(Type type) => type == typeof(ProcessedMemberFunction); - public object? ReadYaml(IParser parser, Type type, ObjectDeserializer rootDeserializer) => throw new NotImplementedException(); - public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializer serializer) { - if (value is not ProcessedMemberFunction m) return; - emitter.Emit(new MappingStart()); - emitter.Emit(new Scalar("signature")); - emitter.Emit(new Scalar(m.MemberFunctionSignature)); - emitter.Emit(new Scalar("return_type")); - emitter.Emit(new Scalar(m.MemberFunctionReturnType)); - emitter.Emit(new Scalar("name")); - emitter.Emit(new Scalar(m.MemberFunctionName)); - emitter.Emit(new Scalar("parameters")); - emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); - foreach (var parameter in m.MemberFunctionParameters) { - ProcessedFieldConverter.Instance.WriteYaml(emitter, parameter, parameter.GetType(), serializer); - } - emitter.Emit(new SequenceEnd()); - emitter.Emit(new MappingEnd()); - } - - public static readonly IYamlTypeConverter Instance = new ProcessedMemberFunctionConverter(); -} + public static void Write(DirectoryInfo dir) { + // make sure we have all the dependencies for each struct before we write them + var structToFullDeps = ResolveFullDependencies(); + _structs.Sort((a, b) => { + var cmp = structToFullDeps[a.StructTypeName].Count.CompareTo(structToFullDeps[b.StructTypeName].Count); + return cmp != 0 ? cmp : string.Compare(a.StructTypeName, b.StructTypeName, StringComparison.Ordinal); + }); -public class ProcessedVirtualFunctionConverter : IYamlTypeConverter { + OrderByNestedTypeReferences(structToFullDeps); - public bool Accepts(Type type) => type == typeof(ProcessedVirtualFunction); - public object? ReadYaml(IParser parser, Type type, ObjectDeserializer rootDeserializer) => throw new NotImplementedException(); - public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializer serializer) { - if (value is not ProcessedVirtualFunction v) return; - emitter.Emit(new MappingStart()); - emitter.Emit(new Scalar("name")); - emitter.Emit(new Scalar(v.VirtualFunctionName)); - emitter.Emit(new Scalar("offset")); - emitter.Emit(new Scalar(v.Offset.ToString())); - if (v.VirtualFunctionReturnType != null) { - emitter.Emit(new Scalar("return_type")); - emitter.Emit(new Scalar(v.VirtualFunctionReturnType.FixTypeName())); + void orderInlineStructs(ProcessedStruct parent) { + if (parent.InlineStructs == null) return; + parent.InlineStructs = [.. parent.InlineStructs + .OrderBy(t => structToFullDeps[t.StructTypeName].Count) + .ThenBy(t => t.StructTypeName, StringComparer.Ordinal)]; + foreach (var inlineStruct in parent.InlineStructs) + orderInlineStructs(inlineStruct); } - if (v.VirtualFunctionParameters != null) { - emitter.Emit(new Scalar("parameters")); - emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); - foreach (var parameter in v.VirtualFunctionParameters) { - ProcessedFieldConverter.Instance.WriteYaml(emitter, parameter, parameter.GetType(), serializer); - } - emitter.Emit(new SequenceEnd()); - } - emitter.Emit(new MappingEnd()); - } - public static readonly IYamlTypeConverter Instance = new ProcessedVirtualFunctionConverter(); -} + foreach (var s in _structs) + orderInlineStructs(s); -public class ProcessedStaticMembersConverter : IYamlTypeConverter { - public bool Accepts(Type type) => type == typeof(ProcessedStaticMembers); - public object? ReadYaml(IParser parser, Type type, ObjectDeserializer rootDeserializer) => throw new NotImplementedException(); - public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializer serializer) { - if (value is not ProcessedStaticMembers s) return; - emitter.Emit(new MappingStart()); - emitter.Emit(new Scalar("name")); - emitter.Emit(new Scalar(s.Name)); - emitter.Emit(new Scalar("signature")); - emitter.Emit(new Scalar(s.Signature)); - emitter.Emit(new Scalar("relative_follow_offsets")); - emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); - foreach (var offset in s.RelativeFollowOffsets) { - emitter.Emit(new Scalar(offset.ToString())); - } - emitter.Emit(new SequenceEnd()); - emitter.Emit(new Scalar("is_pointer")); - emitter.Emit(new Scalar(s.IsPointer.ToString())); - emitter.Emit(new Scalar("return_type")); - emitter.Emit(new Scalar(s.ReturnType.FixTypeName())); - emitter.Emit(new MappingEnd()); - } + var serializer = new SerializerBuilder() + .WithNamingConvention(UnderscoredNamingConvention.Instance) + .WithTypeConverter(ProcessedStructConverter.Instance) + .WithTypeConverter(ProcessedEnumConverter.Instance) + .WithTypeConverter(ProcessedFieldConverter.Instance) + .WithTypeConverter(ProcessedVirtualFunctionConverter.Instance) + .WithTypeConverter(ProcessedMemberFunctionConverter.Instance) + .Build(); + var yaml = serializer.Serialize(new YamlExport { + Enums = [.. _enums], + Structs = _structs.Select(t => t.FixOrder()).ToArray() + }); - public static readonly IYamlTypeConverter Instance = new ProcessedStaticMembersConverter(); -} + new FileInfo(Path.Join(dir.FullName, "structs.yml")).WriteFile(yaml); + } -public record ProcessingType(Type Type, string? OverrideTypeName) { - public static implicit operator Type(ProcessingType t) => t.Type; - public static implicit operator ProcessingType(Type t) => new(t, null); } diff --git a/CExporter/ExporterStatics.cs b/CExporter/ExporterStatics.cs index 05782d0ec4..03d2bd2fc4 100644 --- a/CExporter/ExporterStatics.cs +++ b/CExporter/ExporterStatics.cs @@ -9,45 +9,38 @@ namespace CExporter; public static class ExporterStatics { #pragma warning disable CA2211 // ReSharper disable once InconsistentNaming - public static readonly string FFXIVNamespacePrefix = string.Join(".", nameof(FFXIVClientStructs), nameof(FFXIVClientStructs.FFXIV)); - public static readonly string StdNamespacePrefix = string.Join(".", nameof(FFXIVClientStructs), nameof(FFXIVClientStructs.STD)); - public static readonly string InteropNamespacePrefix = string.Join(".", nameof(FFXIVClientStructs), nameof(FFXIVClientStructs.Interop)); - public static readonly string HavokNamespacePrefix = string.Join(".", nameof(FFXIVClientStructs), nameof(FFXIVClientStructs.Havok)); + public const string FFXIVNamespacePrefix = $"{nameof(FFXIVClientStructs)}.{nameof(FFXIVClientStructs.FFXIV)}"; + public const string StdNamespacePrefix = $"{nameof(FFXIVClientStructs)}.{nameof(FFXIVClientStructs.STD)}"; + public const string StdHelperNamespacePrefix = $"{StdNamespacePrefix}.{nameof(FFXIVClientStructs.STD.Helper)}"; + public const string InteropNamespacePrefix = $"{nameof(FFXIVClientStructs)}.{nameof(FFXIVClientStructs.Interop)}"; + public const string HavokNamespacePrefix = $"{nameof(FFXIVClientStructs)}.{nameof(FFXIVClientStructs.Havok)}"; + public const string StdCppNamespace = "std::"; public static readonly BindingFlags BindingFlags = BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance; public static readonly BindingFlags StaticBindingFlags = BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static; public static readonly string[] IgnoredTypeNames = ["MemberFunctionPointers", "StaticAddressPointers", "Addresses", "VirtualTable"]; - public static readonly string[] BaseTypeNames = ["byte", "wchar_t", "char", "float", "double", "__int16", "int", "__int64", "unsigned __int16", "unsigned int", "unsigned __int64", "__int8", "unsigned __int8"]; + public static readonly string[] BaseTypeNames = ["byte", "wchar_t", "char", "float", "double", "bool", "int8_t", "uint8_t", "int16_t", "uint16_t", "int32_t", "uint32_t", "int64_t", "uint64_t"]; public static readonly List WarningList = []; public static readonly List ErrorList = []; public static readonly string Separator = "::"; #pragma warning restore CA2211 - public static Type[] GetXIVTypes() { - var assembly = AppDomain.CurrentDomain.Load(nameof(FFXIVClientStructs)); - - Type[] definedTypes; - try { - definedTypes = assembly.DefinedTypes.Select(ti => ti.AsType()).ToArray(); - } catch (ReflectionTypeLoadException ex) { - definedTypes = ex.Types.Where(t => t != null).ToArray()!; - } + public static Type[] GetXIVTypes() => GetTypes(FFXIVNamespacePrefix); - return definedTypes.Where(t => t.FullName!.StartsWith(FFXIVNamespacePrefix) && !t.FullName.EndsWith("VirtualTable") && t.GetCustomAttribute() == null).ToArray(); - } + public static Type[] GetHavokTypes() => GetTypes(HavokNamespacePrefix); - public static Type[] GetHavokTypes() { + private static Type[] GetTypes(string namespacePrefix) { var assembly = AppDomain.CurrentDomain.Load(nameof(FFXIVClientStructs)); Type[] definedTypes; try { - definedTypes = assembly.DefinedTypes.Select(ti => ti.AsType()).ToArray(); + definedTypes = assembly.DefinedTypes.Where(ti => ti.IsValueType && !ti.IsEnum && !ti.IsNested).Select(ti => ti.AsType()).ToArray(); } catch (ReflectionTypeLoadException ex) { definedTypes = ex.Types.Where(t => t != null).ToArray()!; } - return definedTypes.Where(t => t.FullName!.StartsWith(HavokNamespacePrefix) && !t.FullName.EndsWith("VirtualTable") && t.GetCustomAttribute() == null).ToArray(); + return definedTypes.Where(t => t.FullName!.StartsWith(namespacePrefix) && !t.FullName.EndsWith("VirtualTable") && t.GetCustomAttribute() == null).ToArray(); } public static Type GetBestMatchFromSize(int size) => size switch { diff --git a/CExporter/Extensions.cs b/CExporter/Extensions.cs index 424ad7f471..90dd3fbf5c 100644 --- a/CExporter/Extensions.cs +++ b/CExporter/Extensions.cs @@ -17,248 +17,278 @@ public static partial class TypeExtensions { [GeneratedRegex(@"^`[1-9]\+Node", RegexOptions.Compiled)] public static partial Regex StdNodeRegex(); - public static bool IsFixedBuffer(this Type type) { - return type.Name.EndsWith("e__FixedBuffer"); - } + extension(Type type) { + public bool IsFixedBuffer() { + return type.Name.EndsWith("e__FixedBuffer"); + } - public static bool IsStruct(this Type type) { - return type != typeof(decimal) && type is { IsValueType: true, IsPrimitive: false, IsEnum: false }; - } + public bool IsStruct() { + return type != typeof(decimal) && type is { IsValueType: true, IsPrimitive: false, IsEnum: false }; + } - public static bool IsBaseType(this Type type) { - while (true) { - if (type.IsPointer) { - type = type.GetElementType()!; - continue; + public bool IsBaseType() { + while (true) { + if (type.IsPointer) { + type = type.GetElementType()!; + continue; + } + if (type.IsGenericPointer()) { + type = type.GenericTypeArguments[0]; + continue; + } + return type == typeof(void) || type == typeof(bool) || type == typeof(char) || + type == typeof(sbyte) || type == typeof(byte) || type == typeof(short) || + type == typeof(ushort) || type == typeof(int) || type == typeof(uint) || + type == typeof(long) || type == typeof(ulong) || type == typeof(float) || + type == typeof(double) || type == typeof(decimal) || type == typeof(nint) || + type == typeof(nuint) || type == typeof(Half) || type == typeof(CStringPointer); } - if (type.IsGenericPointer()) { - type = type.GenericTypeArguments[0]; - continue; + } + + public Type[] GetInheritsTypes() { + const string inheritsAttribute = "InheritsAttribute`1"; + Type[] inheritances = []; + foreach (var attr in type.GetCustomAttributes()) { + if (attr.GetType().Name.Contains(inheritsAttribute)) { + inheritances = [.. inheritances, attr.GetType().GetGenericArguments()[0]]; + } } - return type == typeof(void) || type == typeof(bool) || type == typeof(char) || - type == typeof(sbyte) || type == typeof(byte) || type == typeof(short) || - type == typeof(ushort) || type == typeof(int) || type == typeof(uint) || - type == typeof(long) || type == typeof(ulong) || type == typeof(float) || - type == typeof(double) || type == typeof(decimal) || type == typeof(nint) || - type == typeof(nuint) || type == typeof(Half) || type == typeof(CStringPointer); + return inheritances; } - } - public static Type[] GetInheritsTypes(this Type type) { - const string inheritsAttribute = "InheritsAttribute`1"; - Type[] inheritances = []; - foreach (var attr in type.GetCustomAttributes()) { - if (attr.GetType().Name.Contains(inheritsAttribute)) { - inheritances = [.. inheritances, attr.GetType().GetGenericArguments()[0]]; + public bool IsInheritance(FieldInfo field) { + var inheritances = type.GetInheritsTypes(); + if (inheritances.Length == 0) return false; + foreach (var inheritance in inheritances) { + if (inheritance.IsFieldInType(field)) return true; + if (IsInheritance(inheritance, field)) return true; } + return false; } - return inheritances; - } - public static bool IsInheritance(this Type type, FieldInfo field) { - var inheritances = type.GetInheritsTypes(); - if (inheritances.Length == 0) return false; - foreach (var inheritance in inheritances) { - if (inheritance.IsFieldInType(field)) return true; - if (IsInheritance(inheritance, field)) return true; + public bool IsFieldInType(FieldInfo field) { + var nameStrings = field.Name.Split('_'); + var index = Array.IndexOf(nameStrings, type.Name); + if (index <= 0) return type.GetFields(ExporterStatics.BindingFlags).Any(f => f.Name == field.Name && f.FieldType == field.FieldType); + var name = string.Join("_", nameStrings[(index + 1)..]); + return type.GetFields(ExporterStatics.BindingFlags).Any(f => f.Name == name && f.FieldType == field.FieldType) || type.GetFields(ExporterStatics.BindingFlags).Any(f => f.Name == field.Name && f.FieldType == field.FieldType); } - return false; - } - public static bool IsFieldInType(this Type type, FieldInfo field) { - var nameStrings = field.Name.Split('_'); - var index = Array.IndexOf(nameStrings, type.Name); - if (index <= 0) return type.GetFields(ExporterStatics.BindingFlags).Any(f => f.Name == field.Name && f.FieldType == field.FieldType); - var name = string.Join("_", nameStrings[(index + 1)..]); - return type.GetFields(ExporterStatics.BindingFlags).Any(f => f.Name == name && f.FieldType == field.FieldType) || type.GetFields(ExporterStatics.BindingFlags).Any(f => f.Name == field.Name && f.FieldType == field.FieldType); - } + public string FixTypeName(bool shouldLower = true) { + using var builderPooled = StringBuilderPool.Get(500); + var builder = builderPooled.Builder; + var name = type switch { + _ when type == typeof(void) => "void", + _ when type == typeof(char) => "wchar_t", + _ when type == typeof(CStringPointer) => "char*", + _ when type == typeof(float) => "float", + _ when type == typeof(double) => "double", + _ when type == typeof(bool) => "bool", + _ when type == typeof(byte) => "uint8_t", + _ when type == typeof(sbyte) => "int8_t", + _ when type == typeof(ushort) || type == typeof(Half) => "uint16_t", + _ when type == typeof(short) => "int16_t", + _ when type == typeof(uint) => "uint32_t", + _ when type == typeof(int) => "int32_t", + _ when type == typeof(long) || type == typeof(nint) => "int64_t", + _ when type == typeof(ulong) || type == typeof(nuint) => "uint64_t", + _ when type.IsPointer() && !type.IsGenericPointer() => $"{type.GetElementType()!.FixTypeName(shouldLower)}*", // able to handle infinite amount of pointers until stack size says no but this should never happen + _ when type.IsGenericPointer() => $"{type.GenericTypeArguments[0].FixTypeName(shouldLower)}*", + _ when type.IsGenericTypeParameter || type.IsGenericParameter => type.Name, + _ when type.IsConstructedGenericType || type.IsGenericTypeDefinition => type.SanitizeName(true), + _ => type.SanitizeName() + }; + return builder.Append(name).Replace("+", ExporterStatics.Separator).Replace(".", ExporterStatics.Separator).ToString(); + } - public static string FixTypeName(this Type type, bool shouldLower = true) { - using var builderPooled = StringBuilderPool.Get(500); - var builder = builderPooled.Builder; - var name = type switch { - _ when type == typeof(void) => shouldLower ? type.Name.ToLower() : type.Name, - _ when type == typeof(byte) || type == typeof(byte*) || type == typeof(byte**) => shouldLower ? type.Name.ToLower() : type.Name, - _ when type == typeof(char) => "wchar_t", - _ when type == typeof(bool) => "char", - _ when type == typeof(float) => "float", - _ when type == typeof(double) => "double", - _ when type == typeof(short) => "__int16", - _ when type == typeof(int) => "int", - _ when type == typeof(long) || type == typeof(nint) => "__int64", - _ when type == typeof(ushort) => "unsigned __int16", - _ when type == typeof(uint) => "unsigned int", - _ when type == typeof(ulong) || type == typeof(nuint) => "unsigned __int64", - _ when type == typeof(sbyte) => "__int8", - _ when type == typeof(char*) => "wchar_t*", - _ when type == typeof(short*) => "__int16*", - _ when type == typeof(ushort*) => "unsigned __int16*", - _ when type == typeof(int*) => "int*", - _ when type == typeof(uint*) => "unsigned int*", - _ when type == typeof(long*) || type == typeof(nint*) => "__int64*", - _ when type == typeof(ulong*) || type == typeof(nuint*) => "unsigned __int64*", - _ when type == typeof(float*) => "float*", - _ when type == typeof(Half) => "__int16", // Half is a struct that is 2 bytes long and does not exist in C so we just use __int16 - _ when type == typeof(void*) => "__int64", - _ when type == typeof(void**) => "__int64*", - _ when type == typeof(CStringPointer) => "char*", - _ => type.SanitizeName() - }; - return builder.Append(name).Replace("+", ExporterStatics.Separator).Replace(".", ExporterStatics.Separator).ToString(); - } + public int SizeOf() { + return type switch { + _ when type == typeof(sbyte) || type == typeof(byte) || type == typeof(bool) => 1, + _ when type == typeof(char) || type == typeof(short) || type == typeof(ushort) || type == typeof(Half) => 2, + _ when type == typeof(int) || type == typeof(uint) || type == typeof(float) => 4, + _ when type == typeof(long) || type == typeof(ulong) || type == typeof(double) || type.IsPointer || type.IsFunctionPointer || type.IsUnmanagedFunctionPointer || (type.Name == "Pointer`1" && type.Namespace.AsSpan().SequenceEqual(ExporterStatics.InteropNamespacePrefix)) || type == typeof(CStringPointer) => 8, + _ when type.Name.StartsWith("FixedSizeArray") => type.GetGenericArguments()[0].SizeOf() * int.Parse(type.Name[14..type.Name.IndexOf('`')]), + _ when type.GetCustomAttribute() is { Length: var length } => type.GetGenericArguments()[0].SizeOf() * length, + _ when type.IsStruct() && !type.IsGenericType && (type.StructLayoutAttribute?.Value ?? LayoutKind.Sequential) != LayoutKind.Sequential => type.StructLayoutAttribute?.Size ?? (int?)typeof(Unsafe).GetMethod("SizeOf")?.MakeGenericMethod(type).Invoke(null, null) ?? 0, + _ when type.IsEnum => Enum.GetUnderlyingType(type).SizeOf(), + _ when type.ContainsGenericParameters => 1, + _ when type.IsGenericType => Marshal.SizeOf(Activator.CreateInstance(type)!), + _ => type.GetSizeOf() + }; + } - public static int SizeOf(this Type type) { - return type switch { - _ when type == typeof(sbyte) || type == typeof(byte) || type == typeof(bool) => 1, - _ when type == typeof(char) || type == typeof(short) || type == typeof(ushort) || type == typeof(Half) => 2, - _ when type == typeof(int) || type == typeof(uint) || type == typeof(float) => 4, - _ when type == typeof(long) || type == typeof(ulong) || type == typeof(double) || type.IsPointer || type.IsFunctionPointer || type.IsUnmanagedFunctionPointer || (type.Name == "Pointer`1" && type.Namespace.AsSpan().SequenceEqual(ExporterStatics.InteropNamespacePrefix)) || type == typeof(CStringPointer) => 8, - _ when type.Name.StartsWith("FixedSizeArray") => type.GetGenericArguments()[0].SizeOf() * int.Parse(type.Name[14..type.Name.IndexOf('`')]), - _ when type.GetCustomAttribute() is { Length: var length } => type.GetGenericArguments()[0].SizeOf() * length, - _ when type.IsStruct() && !type.IsGenericType && (type.StructLayoutAttribute?.Value ?? LayoutKind.Sequential) != LayoutKind.Sequential => type.StructLayoutAttribute?.Size ?? (int?)typeof(Unsafe).GetMethod("SizeOf")?.MakeGenericMethod(type).Invoke(null, null) ?? 0, - _ when type.IsEnum => Enum.GetUnderlyingType(type).SizeOf(), - _ when type.IsGenericType => Marshal.SizeOf(Activator.CreateInstance(type)!), - _ => GetSizeOf(type) - }; - } + private int GetSizeOf() { + try { + return Marshal.SizeOf(Activator.CreateInstance(type)!); + } catch { + return 0; + } + } - private static int GetSizeOf(this Type type) { - try { - return Marshal.SizeOf(Activator.CreateInstance(type)!); - } catch { - return 0; + public string GetNamespace() { + var ns = type.Namespace!; + if (ns.StartsWith("System")) { + return ns; + } + var offset = ns.IndexOf('.') + 1; + ns = offset == 0 ? "" : ns[offset..]; + if (ns.StartsWith("SQEX")) return ns; + offset = ns.IndexOf('.') + 1; + return offset == 0 ? "" : ns[offset..]; } - } - public static string GetNamespace(this Type type) { - var ns = type.Namespace!; - var offset = ns.IndexOf('.', ns.IndexOf('.') + 1) + 1; - return offset == 0 ? "" : ns[offset..]; - } + public string GetFullname() { + return type.Namespace + "." + type.Name; + } - public static string GetFullname(this Type type) { - return type.Namespace + "." + type.Name; - } - private static bool IsHavokStdXivOrInterop(this Type type) { - var ns = type.Namespace.AsSpan(); - const string cs = "FFXIVClientStructs."; - if (!ns.StartsWith(cs)) - return false; - ns = ns[cs.Length..]; + private bool IsHavokStdXivOrInterop() { + var ns = type.Namespace.AsSpan(); + const string cs = "FFXIVClientStructs."; + if (!ns.StartsWith(cs)) + return false; + ns = ns[cs.Length..]; - return ns.StartsWith("FFXIV") - || ns.StartsWith("STD") - || ns.StartsWith("Interop") - || ns.StartsWith("Havok"); - } + return ns.StartsWith("FFXIV") + || ns.StartsWith("STD") + || ns.StartsWith("Interop") + || ns.StartsWith("Havok"); + } + + private bool IsSQEX() { + var ns = type.Namespace.AsSpan(); + const string cs = "FFXIVClientStructs."; + if (!ns.StartsWith(cs)) + return false; + ns = ns[cs.Length..]; - public static Type GetPointerType(this Type type) { - while (type.IsPointer()) { - if (type.IsPointer) type = type.GetElementType()!; - else if (type.IsGenericPointer()) type = type.GenericTypeArguments[0]; - else if (type.IsFunctionPointer) type = type.GetFunctionPointerReturnType(); - else if (type.IsUnmanagedFunctionPointer) type = type.GetFunctionPointerReturnType(); + return ns.StartsWith("SQEX"); } - return type; - } - public static bool IsPointer(this Type type) { - return type.IsPointer || type.IsFunctionPointer || type.IsUnmanagedFunctionPointer || type.IsGenericPointer(); - } + public Type GetPointerType() { + while (type.IsPointer()) { + if (type.IsPointer) type = type.GetElementType()!; + else if (type.IsGenericPointer()) type = type.GenericTypeArguments[0]; + else if (type.IsFunctionPointer || type.IsUnmanagedFunctionPointer) type = type.GetFunctionPointerReturnType(); + } + return type; + } - public static bool IsGenericPointer(this Type type) { - return (type.Name == "Pointer`1" && type.Namespace == ExporterStatics.InteropNamespacePrefix) || (type.Name == "hkRefPtr`1" && type.Namespace == ExporterStatics.HavokNamespacePrefix); - } + public bool IsPointer() { + return type.IsPointer || type.IsFunctionPointer || type.IsUnmanagedFunctionPointer || type.IsGenericPointer(); + } - public static string SanitizeName(this Type type) { - if (type.IsPointer || type.IsFunctionPointer || type.IsUnmanagedFunctionPointer) return type.GetElementType()!.FixTypeName() + "*"; - var name = (type.FullName ?? type.GetFullname()).AsSpan(); - if (type.IsHavokStdXivOrInterop()) { - name = name[(name.IndexOf('.') + 1)..]; - name = name[(name.IndexOf('.') + 1)..]; + public bool IsGenericPointer() { + return type is { Name: "Pointer`1", Namespace: ExporterStatics.InteropNamespacePrefix } || (type.Name == "hkRefPtr`1" && type.Namespace!.StartsWith(ExporterStatics.HavokNamespacePrefix)); } - if (!type.IsGenericType) return name.ToString(); - switch (type.Name) { - case "Pointer`1" when type.Namespace == ExporterStatics.InteropNamespacePrefix: - return type.GenericTypeArguments[0].FixTypeName() + "*"; - case "hkRefPtr`1" when type.Namespace == ExporterStatics.HavokNamespacePrefix: + + public string SanitizeName(bool ignoreGenericTypes = false) { + if (type.IsPointer || type.IsFunctionPointer || type.IsUnmanagedFunctionPointer) return type.GetElementType()!.FixTypeName() + "*"; + var name = (type.FullName ?? type.GetFullname()).AsSpan(); + if (type.IsHavokStdXivOrInterop()) { + name = name[(name.IndexOf('.') + 1)..]; + name = name[(name.IndexOf('.') + 1)..]; + } + if (type.IsSQEX()) { + name = name[(name.IndexOf('.') + 1)..]; + } + if (!type.IsGenericType) return name.ToString(); + if (type.IsGenericPointer()) return type.GenericTypeArguments[0].FixTypeName() + "*"; - } - var afterBacktick = name[name.IndexOf('`')..]; - name = name[..^afterBacktick.Length]; - using var builderPooled = StringBuilderPool.Get(500); - var builder = builderPooled.Builder; - builder.Append(name); - - var me = StdNodeRegex().EnumerateMatches(afterBacktick); - if (me.MoveNext()) - builder.Append('.').Append(afterBacktick.Slice(me.Current.Index + 3, me.Current.Length - 3)); - - builder.Append('<'); - var gta = type.GenericTypeArguments; - for (var i = 0; i < gta.Length; i++) { - if (i != 0) - builder.Append(", "); - builder.Append(gta[i].FixTypeName()); - } + using var builderPooled = StringBuilderPool.Get(500); + var builder = builderPooled.Builder; + if (name.IndexOf('`') != -1){ - builder.Append('>'); - return builder.ToString(); - } + var afterBacktick = name[name.IndexOf('`')..]; + name = name[..^afterBacktick.Length]; - public static int PackSize(this Type type) { - if (type.GetCustomAttribute() != null) return 1; // FixedSizeArrayAttribute is always packed to 1 as the generated struct gets generated with Pack = 1 - if (!type.IsStruct()) return type.SizeOf(); - var pack = type.StructLayoutAttribute?.Pack ?? 8; - if (pack == 0) pack = 8; - var fields = type.GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); - if (fields.Length == 0) return 1; - return fields.Max(t => Math.Min(pack, t.FieldType.PackSize())); - } + builder.Append(name); + + var me = StdNodeRegex().EnumerateMatches(afterBacktick); + if (me.MoveNext()) + builder.Append('.').Append(afterBacktick.Slice(me.Current.Index + 3, me.Current.Length - 3)); + } + else + builder.Append(name); - public static bool IsInStructList(this Type type, List structs) { - var name = type.FixTypeName(); - foreach (var str in structs) { - if (str.StructTypeName == name) return true; + if (!ignoreGenericTypes) + { + builder.Append('<'); + var gta = type.GenericTypeArguments; + for (var i = 0; i < gta.Length; i++) { + if (i != 0) + builder.Append(", "); + builder.Append(gta[i].FixTypeName()); + } + builder.Append('>'); + } + + return builder.ToString(); + } + public int PackSize() { + if (type.GetCustomAttribute() != null) return 1; // FixedSizeArrayAttribute is always packed to 1 as the generated struct gets generated with Pack = 1 + if (!type.IsStruct()) return type.SizeOf(); + var pack = type.StructLayoutAttribute?.Pack ?? 8; + if (pack == 0) pack = 8; + var fields = type.GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + if (fields.Length == 0) return 1; + return fields.Max(t => Math.Min(pack, t.FieldType.PackSize())); + } + public bool IsInStructList(List structs) { + var name = type.FixTypeName(); + foreach (var str in structs) { + if (str.StructTypeName == name) return true; + } + return false; } - return false; + + public bool ShouldNotExportType() => type.IsPointer() || type.IsPrimitive || type.IsFixedBuffer() || type.IsEnum || type.IsBaseType(); + + public bool IsGenericTypeParameterOrPointer() => type.IsGenericTypeParameter || (type.IsPointer() && type.GetPointerType().IsGenericTypeParameterOrPointer()); + + public bool IsStdDefiniton() => type.Namespace.AsSpan().SequenceEqual(ExporterStatics.StdNamespacePrefix); } + } public static class FieldInfoExtensions { - public static int GetFieldOffset(this FieldInfo info) { - var attrs = info.GetCustomAttributes(typeof(FieldOffsetAttribute), false); - return attrs.Length != 0 ? attrs.Cast().Single().Value : GetFieldOffsetSequential(info); - } + extension(FieldInfo info) { + + public int GetFieldOffset() { + var attrs = info.GetCustomAttributes(typeof(FieldOffsetAttribute), false); + return attrs.Length != 0 ? attrs.Cast().Single().Value : GetFieldOffsetSequential(info); + } - public static int GetFieldOffsetSequential(this FieldInfo info) { - if (info.DeclaringType is not { } declaring) - throw new Exception($"Unable to access declaring type of field {info.Name}"); - var pack = declaring.StructLayoutAttribute?.Pack ?? 0; // Default to 0 if no pack is specified - var fields = declaring.GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); - var offset = 0; - foreach (var field in fields) { - var fieldPack = field.FieldType.PackSize(); - if (pack != 0) { - fieldPack = Math.Min(pack, field.FieldType.PackSize()); + public int GetFieldOffsetSequential() { + if (info.DeclaringType is not { } declaring) + throw new Exception($"Unable to access declaring type of field {info.Name}"); + var pack = declaring.StructLayoutAttribute?.Pack ?? 0; // Default to 0 if no pack is specified + var fields = declaring.GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + var offset = 0; + foreach (var field in fields) { + var fieldPack = field.FieldType.PackSize(); + if (pack != 0) { + fieldPack = Math.Min(pack, field.FieldType.PackSize()); + } + offset = (offset + fieldPack - 1) / fieldPack * fieldPack; + if (field == info) { + return offset; + } + offset += field.FieldType.SizeOf(); } - offset = (offset + fieldPack - 1) / fieldPack * fieldPack; - if (field == info) { - return offset; - } - offset += field.FieldType.SizeOf(); + throw new Exception("Field not found"); } - throw new Exception("Field not found"); - } - public static bool IsDirectBase(this FieldInfo field) { - var bases = field.DeclaringType?.GetInheritsTypes() ?? []; - return bases.Any(b => field.FieldType == b && field.Name == (b.Name == field.DeclaringType?.Name ? b.Name + "Base" : b.Name)) || field.GetCustomAttribute() != null; - } + public bool IsDirectBase() { + var bases = info.DeclaringType?.GetInheritsTypes() ?? []; + return bases.Any(b => info.FieldType == b && info.Name == (b.Name == info.DeclaringType?.Name ? b.Name + "Base" : b.Name)) || info.GetCustomAttribute() != null; + } - public static bool IsBitArray(this FieldInfo type) { - return type.GetCustomAttributes().Any(t => t.GetType().Name.Contains("BitFieldAttribute`1")); + public bool IsBitArray() { + return info.GetCustomAttributes().Any(t => t.GetType().Name.Contains("BitFieldAttribute`1")); + } + + public bool IsStdDefinition() => info.FieldType.IsStdDefiniton(); } } public static class Extensions { @@ -267,10 +297,20 @@ public static void WriteFile(this FileInfo file, string content) { stream.Write(content); } - public static bool IsInStructList(this string name, List structs) { - foreach (var str in structs) { - if (str.StructTypeName == name) return true; + extension(string name) { + public bool IsInStructList(List structs) { + foreach (var str in structs) { + if (str.StructTypeName == name) return true; + } + return false; + } + + public string GetCorrectedName() => name is "Dtor" or "Ctor" ? name.ToLower() : name; + } + + extension(Type[] span){ + public bool GenericSequenceEqual(Type[] other){ + return span.Select(t => t.Name).SequenceEqual(other.Select(t => t.Name)); } - return false; } } diff --git a/CExporter/ProcessedData.cs b/CExporter/ProcessedData.cs new file mode 100644 index 0000000000..242d67beb6 --- /dev/null +++ b/CExporter/ProcessedData.cs @@ -0,0 +1,176 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using YamlDotNet.Serialization; + +namespace CExporter; + +public class YamlExport { + public required ProcessedEnum[] Enums; + public required ProcessedStruct[] Structs; +} + +public class ProcessedBitField { + public required int Size; + public required int Offset; + public required Type Type; + public required string Name; +} + +public class ProcessedFieldUnion { + public required Type FieldType; + public required string FieldName; + public required string? FieldTypeOverride; + public Type[]? FieldTypeOverrideTemplates; +} + +public class ProcessedField { + public string? FieldTypeOverride; + public bool? FieldTypeOverrideCheck; + public Type[]? FieldTypeOverrideTemplates; + public required Type FieldType; + public required string FieldName; + public required int FieldOffset; + public required bool IsUnion; + public required ProcessedFieldUnion[] UnionValues; + public bool IsBase; + public required ProcessedBitField[] Bits; + public virtual int FieldSize => FieldType.SizeOf(); +} + +public class ProcessedFixedField : ProcessedField { + public required int FixedSize; + public bool FixedString; + + public override int FieldSize => FixedSize * FieldType.SizeOf(); +} + +public class ProcessedFunctionField : ProcessedField { + public required Type FunctionReturnType; + public required ProcessedField[] FunctionParameters; +} + +public class ProcessedStaticMembers { + public required string Name; + public required string Signature; + public required ushort[] RelativeFollowOffsets; + public bool IsPointer; + public required Type ReturnType; +} + +public class ProcessedStruct { + public required string? StructTypeOverride; + public required bool IsUnion; + public required Type StructType; + public required string StructName; + public required string StructNamespace; + public required int StructSize; + public required ProcessedField[] Fields; + public required int VirtualFunctionSize; + public required ProcessedVirtualFunction[]? VirtualFunctions; + public required ProcessedMemberFunction[] MemberFunctions; + public required ProcessedMemberFunction[]? StaticMemberFunctions; + public required ProcessedStaticMembers[]? StaticMembers; + public required string[] TemplateTypes; + public required ProcessedStruct[]? InlineStructs; + public required ProcessedEnum[]? InlineEnums; + public required bool IsSequential; + [YamlIgnore] + public string StructTypeName => StructTypeOverride ?? StructType.FixTypeName(); + [YamlIgnore] + private string[] _dependencyNames = []; + [YamlIgnore] + public string[] DependencyNames { + get { + if (_dependencyNames.Length == 0) { + var unionMembers = Fields.SelectMany(unionField => unionField.UnionValues.Select(unionValue => new ProcessedField { + FieldType = unionValue.FieldType, + FieldTypeOverride = unionValue.FieldTypeOverride, + FieldTypeOverrideTemplates = unionValue.FieldTypeOverrideTemplates, + FieldName = unionValue.FieldName, + FieldOffset = unionField.FieldOffset, + Bits = [], + IsUnion = false, + UnionValues = [] + })); + _dependencyNames = Fields + .Concat(unionMembers) + .Where(t => { + if (t.FieldTypeOverride == null) { + if (t.FieldType.IsEnum) { + var enumOwner = t.FieldType.DeclaringType; + return enumOwner != null && !IsNestedWithin(StructType, enumOwner); + } + return (t.GetType() == typeof(ProcessedField) || t.GetType() == typeof(ProcessedFixedField)) && !t.FieldType.ShouldNotExportType() && !ExporterStatics.BaseTypeNames.Contains(t.FieldType.FixTypeName()) && !t.FieldType.ContainsGenericParameters; + } + if (t.FieldTypeOverride.EndsWith('*') || t.FieldTypeOverride.StartsWith("Component::Exd::Sheets::") || ExporterStatics.BaseTypeNames.Contains(t.FieldTypeOverride)) return false; + return !t.FieldTypeOverrideCheck.HasValue || !t.FieldTypeOverrideCheck.Value; + }) + .SelectMany(t => { + if (t.FieldTypeOverride == null && t.FieldType.IsEnum) + return [t.FieldType.DeclaringType!.FixTypeName()]; + if (t.FieldTypeOverride?.StartsWith(ExporterStatics.StdCppNamespace) ?? false) return t.FieldTypeOverrideTemplates!.Where(k => !k.ShouldNotExportType()).Select(k => k.FixTypeName()).Where(k => !ExporterStatics.BaseTypeNames.Contains(k)); + return [t.FieldTypeOverride ?? t.FieldType.FixTypeName()]; + }) + .Concat(InlineStructs?.SelectMany(t => t.DependencyNames) ?? Enumerable.Empty()) + .Concat(InlineStructs?.Select(t => t.StructTypeName) ?? Enumerable.Empty()) + .Distinct() + .ToArray(); + } + return _dependencyNames; + } + } + + private static bool IsNestedWithin(Type type, Type possibleOwner) { + if (type == possibleOwner) return true; + for (var declaringType = type.DeclaringType; declaringType != null; declaringType = declaringType.DeclaringType) { + if (declaringType == possibleOwner) return true; + } + return false; + } + + public ProcessedStruct[] ChildrenAndSelf => InlineStructs != null && InlineStructs.Any() ? [this, .. InlineStructs.SelectMany(t => t.ChildrenAndSelf)] : [this]; + public ProcessedStruct FixOrder() { + Fields = [.. Fields.OrderBy(t => t.FieldOffset)]; + if (InlineStructs != null) InlineStructs = [.. InlineStructs.Select(t => t.FixOrder())]; + return this; + } +} + +public class ProcessedEnum { + public required Type EnumType; + public required string EnumName; + public required string EnumNamespace; + public required bool IsFlags; + public required Dictionary EnumValues; +} + +public class ProcessedVirtualFunction { + public required string VirtualFunctionName; + public required int Offset; + public required Type? VirtualFunctionReturnType; + public required ProcessedField[]? VirtualFunctionParameters; + public required string? VirtualFunctionReturnTypeOverride; +} + +public class ProcessedMemberFunction { + public required string MemberFunctionSignature; + public required string MemberFunctionName; + public required ProcessedMemberFunctionReturn MemberFunctionReturnType; + public required ProcessedField[] MemberFunctionParameters; +} + +public class ProcessedMemberFunctionReturn(Type type, string? overrideType) { + public Type Type = type; + public string? OverrideType = overrideType; + public static implicit operator ProcessedMemberFunctionReturn((Type Type, string OverrideType) value) => new(value.Type, value.OverrideType); + public static implicit operator (Type Type, string? OverrideType)(ProcessedMemberFunctionReturn value) => (value.Type, value.OverrideType); + public static implicit operator ProcessedMemberFunctionReturn(Type type) => new(type, null); + public static implicit operator Type(ProcessedMemberFunctionReturn value) => value.Type; + public static implicit operator string(ProcessedMemberFunctionReturn value) => value.OverrideType ?? value.Type.FixTypeName(); +} + +public record ProcessingType(Type Type, string? OverrideTypeName) { + public static implicit operator Type(ProcessingType t) => t.Type; + public static implicit operator ProcessingType(Type t) => new(t, null); +} diff --git a/CExporter/Program.cs b/CExporter/Program.cs index 4976271a72..020dab46e8 100644 --- a/CExporter/Program.cs +++ b/CExporter/Program.cs @@ -13,11 +13,11 @@ public static void Main(string[] args) { var noWrite = args.Contains("--no-write"); var timeStart = DateTime.Now; var dir = new DirectoryInfo(Environment.CurrentDirectory); - while (dir.FullName.Contains("ida") && !dir.FullName.EndsWith("ida")) { + while (dir.FullName.Contains("scripts") && !dir.FullName.EndsWith("scripts")) { dir = dir.Parent!; } - while (!dir.FullName.Contains("ida") && !dir.FullName.EndsWith("ida")) { - dir = dir.GetDirectories("ida", SearchOption.AllDirectories).FirstOrDefault() ?? dir.Parent!; + while (!dir.FullName.Contains("scripts") && !dir.FullName.EndsWith("scripts")) { + dir = dir.GetDirectories("scripts", SearchOption.AllDirectories).FirstOrDefault() ?? dir.Parent!; } Exporter.ProcessTypes(quiet); @@ -37,6 +37,7 @@ public static void Main(string[] args) { Exporter.VerifyNoNameOverlap(dataCheck); Exporter.ProcessDefinedVTables(data); + Exporter.VerifyDependencies(); if (!quiet) Console.WriteLine($"Processed all types in: {DateTime.Now - timeStart}"); timeStart = DateTime.Now; diff --git a/CExporter/YamlConverters.cs b/CExporter/YamlConverters.cs new file mode 100644 index 0000000000..fe5c3c920b --- /dev/null +++ b/CExporter/YamlConverters.cs @@ -0,0 +1,311 @@ +using System; +using System.Linq; +using System.Runtime.InteropServices; +using YamlDotNet.Core; +using YamlDotNet.Core.Events; +using YamlDotNet.Serialization; + +namespace CExporter; + +public class ProcessedEnumConverter : IYamlTypeConverter { + public bool Accepts(Type type) => type == typeof(ProcessedEnum); + public object? ReadYaml(IParser parser, Type type, ObjectDeserializer rootDeserializer) => throw new NotImplementedException(); + public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializer serializer) { + if (value is not ProcessedEnum e) return; + emitter.Emit(new MappingStart()); + emitter.Emit(new Scalar("type")); + emitter.Emit(new Scalar(e.EnumType.FixTypeName())); + emitter.Emit(new Scalar("name")); + emitter.Emit(new Scalar(e.EnumName)); + emitter.Emit(new Scalar("underlying")); + emitter.Emit(new Scalar(e.EnumType.GetEnumUnderlyingType().FixTypeName())); + emitter.Emit(new Scalar("namespace")); + emitter.Emit(new Scalar(e.EnumNamespace)); + emitter.Emit(new Scalar("flags")); + emitter.Emit(e.IsFlags ? new Scalar("True") : new Scalar("False")); + emitter.Emit(new Scalar("values")); + emitter.Emit(new MappingStart()); + foreach (var (key, val) in e.EnumValues) { + emitter.Emit(new Scalar(AnchorName.Empty, TagName.Empty, key, ScalarStyle.DoubleQuoted, true, false)); + emitter.Emit(new Scalar(val)); + } + emitter.Emit(new MappingEnd()); + emitter.Emit(new MappingEnd()); + } + + public static readonly IYamlTypeConverter Instance = new ProcessedEnumConverter(); +} + +public class ProcessedFieldConverter : IYamlTypeConverter { + public bool Accepts(Type type) => type == typeof(ProcessedField); + public object? ReadYaml(IParser parser, Type type, ObjectDeserializer rootDeserializer) => throw new NotImplementedException(); + public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializer serializer) { + if (value is not ProcessedField f) return; + emitter.Emit(new MappingStart()); + emitter.Emit(new Scalar("type")); + if (f.FieldType.IsFunctionPointer || f.FieldType.IsUnmanagedFunctionPointer) { + emitter.Emit(new Scalar("__fastcall")); + } + else { + emitter.Emit(new Scalar(f.FieldTypeOverride ?? f.FieldType.FixTypeName())); + } + emitter.Emit(new Scalar("name")); + emitter.Emit(new Scalar(f.FieldName)); + if (f.FieldOffset >= 0) { + emitter.Emit(new Scalar("offset")); + emitter.Emit(new Scalar(f.FieldOffset.ToString())); + } + if (f.Bits.Length > 0) { + emitter.Emit(new Scalar("bits")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); + foreach (var bit in f.Bits) { + emitter.Emit(new MappingStart()); + emitter.Emit(new Scalar("type")); + emitter.Emit(new Scalar(bit.Type.FixTypeName())); + emitter.Emit(new Scalar("offset")); + emitter.Emit(new Scalar(bit.Offset.ToString())); + emitter.Emit(new Scalar("size")); + emitter.Emit(new Scalar(bit.Size.ToString())); + emitter.Emit(new Scalar("name")); + emitter.Emit(new Scalar(bit.Name)); + emitter.Emit(new MappingEnd()); + } + emitter.Emit(new SequenceEnd()); + } + if (f.IsBase) { + emitter.Emit(new Scalar("base")); + emitter.Emit(new Scalar("true")); + } + switch (f) { + case ProcessedFunctionField func: { + emitter.Emit(new Scalar("return_type")); + emitter.Emit(new Scalar(func.FunctionReturnType.FixTypeName())); + emitter.Emit(new Scalar("parameters")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); + foreach (var p in func.FunctionParameters) { + emitter.Emit(new MappingStart()); + emitter.Emit(new Scalar("type")); + emitter.Emit(new Scalar(p.FieldTypeOverride ?? p.FieldType.FixTypeName())); + emitter.Emit(new Scalar("name")); + emitter.Emit(new Scalar(p.FieldName)); + emitter.Emit(new MappingEnd()); + } + emitter.Emit(new SequenceEnd()); + break; + } + case ProcessedFixedField fix: + emitter.Emit(new Scalar("size")); + emitter.Emit(new Scalar(fix.FixedSize.ToString())); + emitter.Emit(new Scalar("is_string")); + emitter.Emit(new Scalar(fix.FixedString.ToString())); + break; + } + if (f.IsUnion) { + emitter.Emit(new Scalar("union_values")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); + foreach (var unionValue in f.UnionValues) { + emitter.Emit(new MappingStart()); + emitter.Emit(new Scalar("type")); + emitter.Emit(new Scalar(unionValue.FieldTypeOverride ?? unionValue.FieldType.FixTypeName())); + emitter.Emit(new Scalar("name")); + emitter.Emit(new Scalar(unionValue.FieldName)); + emitter.Emit(new MappingEnd()); + } + emitter.Emit(new SequenceEnd()); + } + if(f.FieldType.IsConstructedGenericType && !f.FieldType.IsStdDefiniton()) + { + emitter.Emit(new Scalar("template_args")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Flow)); + foreach (var templateArg in f.FieldType.GenericTypeArguments){ + emitter.Emit(new Scalar(templateArg.FixTypeName())); + } + emitter.Emit(new SequenceEnd()); + } + else if (f.FieldTypeOverrideTemplates != null && f.FieldTypeOverrideTemplates.Any()) + { + emitter.Emit(new Scalar("template_args")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Flow)); + foreach (var templateArg in f.FieldTypeOverrideTemplates){ + emitter.Emit(new Scalar(templateArg.FixTypeName())); + } + emitter.Emit(new SequenceEnd()); + } + emitter.Emit(new MappingEnd()); + } + + public static readonly IYamlTypeConverter Instance = new ProcessedFieldConverter(); +} + +public class ProcessedStructConverter : IYamlTypeConverter { + public bool Accepts(Type type) => type == typeof(ProcessedStruct); + public object? ReadYaml(IParser parser, Type type, ObjectDeserializer rootDeserializer) => throw new NotImplementedException(); + public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializer serializer) { + if (value is not ProcessedStruct s) return; + emitter.Emit(new MappingStart()); + emitter.Emit(new Scalar("type")); + emitter.Emit(new Scalar(!string.IsNullOrWhiteSpace(s.StructTypeOverride) ? s.StructTypeOverride : s.StructType.FixTypeName())); + emitter.Emit(new Scalar("name")); + emitter.Emit(new Scalar(s.StructName)); + emitter.Emit(new Scalar("namespace")); + emitter.Emit(new Scalar(s.StructNamespace)); + emitter.Emit(new Scalar("union")); + emitter.Emit(new Scalar(s.IsUnion.ToString())); + if (s.StructSize > 1 || s.Fields.Any()) { + emitter.Emit(new Scalar("size")); + emitter.Emit(new Scalar(s.StructSize.ToString())); + } + emitter.Emit(new Scalar("fields")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); + foreach (var field in s.Fields) { + ProcessedFieldConverter.Instance.WriteYaml(emitter, field, field.GetType(), serializer); + } + emitter.Emit(new SequenceEnd()); + if (s.VirtualFunctionSize != 0) { + emitter.Emit(new Scalar("vtable_size")); + emitter.Emit(new Scalar(s.VirtualFunctionSize.ToString())); + } + if (s.VirtualFunctions != null) { + emitter.Emit(new Scalar("virtual_functions")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); + foreach (var virtualFunction in s.VirtualFunctions.OrderBy(t => t.Offset)) { + ProcessedVirtualFunctionConverter.Instance.WriteYaml(emitter, virtualFunction, virtualFunction.GetType(), serializer); + } + emitter.Emit(new SequenceEnd()); + } + if (s.MemberFunctions.Any()){ + emitter.Emit(new Scalar("member_functions")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); + foreach (var memberFunction in s.MemberFunctions) { + ProcessedMemberFunctionConverter.Instance.WriteYaml(emitter, memberFunction, memberFunction.GetType(), serializer); + } + emitter.Emit(new SequenceEnd()); + } + if (s.StaticMembers != null) { + emitter.Emit(new Scalar("static_members")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); + foreach (var staticMember in s.StaticMembers) { + ProcessedStaticMembersConverter.Instance.WriteYaml(emitter, staticMember, staticMember.GetType(), serializer); + } + emitter.Emit(new SequenceEnd()); + } + if (s.StaticMemberFunctions != null) { + emitter.Emit(new Scalar("static_member_functions")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); + foreach (var staticMemberFunction in s.StaticMemberFunctions) { + ProcessedMemberFunctionConverter.Instance.WriteYaml(emitter, staticMemberFunction, staticMemberFunction.GetType(), serializer); + } + emitter.Emit(new SequenceEnd()); + } + if (s.TemplateTypes.Any()) { + emitter.Emit(new Scalar("template_types")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Flow)); + foreach (var templateType in s.TemplateTypes) { + emitter.Emit(new Scalar(templateType)); + } + emitter.Emit(new SequenceEnd()); + } + if (s.InlineStructs != null) { + emitter.Emit(new Scalar("inline_structs")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); + foreach (var inlineStruct in s.InlineStructs) { + Instance.WriteYaml(emitter, inlineStruct, inlineStruct.GetType(), serializer); + } + emitter.Emit(new SequenceEnd()); + } + if (s.InlineEnums != null) { + emitter.Emit(new Scalar("inline_enums")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); + foreach (var inlineEnum in s.InlineEnums) { + ProcessedEnumConverter.Instance.WriteYaml(emitter, inlineEnum, inlineEnum.GetType(), serializer); + } + emitter.Emit(new SequenceEnd()); + } + if (s.IsSequential) { + emitter.Emit(new Scalar("is_sequential")); + emitter.Emit(new Scalar(s.IsSequential.ToString())); + } + emitter.Emit(new MappingEnd()); + } + + public static readonly IYamlTypeConverter Instance = new ProcessedStructConverter(); +} + +public class ProcessedMemberFunctionConverter : IYamlTypeConverter { + public bool Accepts(Type type) => type == typeof(ProcessedMemberFunction); + public object? ReadYaml(IParser parser, Type type, ObjectDeserializer rootDeserializer) => throw new NotImplementedException(); + public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializer serializer) { + if (value is not ProcessedMemberFunction m) return; + emitter.Emit(new MappingStart()); + emitter.Emit(new Scalar("signature")); + emitter.Emit(new Scalar(m.MemberFunctionSignature)); + emitter.Emit(new Scalar("return_type")); + emitter.Emit(new Scalar(m.MemberFunctionReturnType)); + emitter.Emit(new Scalar("name")); + emitter.Emit(new Scalar(m.MemberFunctionName)); + emitter.Emit(new Scalar("parameters")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); + foreach (var parameter in m.MemberFunctionParameters) { + ProcessedFieldConverter.Instance.WriteYaml(emitter, parameter, parameter.GetType(), serializer); + } + emitter.Emit(new SequenceEnd()); + emitter.Emit(new MappingEnd()); + } + + public static readonly IYamlTypeConverter Instance = new ProcessedMemberFunctionConverter(); +} + +public class ProcessedVirtualFunctionConverter : IYamlTypeConverter { + + public bool Accepts(Type type) => type == typeof(ProcessedVirtualFunction); + public object? ReadYaml(IParser parser, Type type, ObjectDeserializer rootDeserializer) => throw new NotImplementedException(); + public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializer serializer) { + if (value is not ProcessedVirtualFunction v) return; + emitter.Emit(new MappingStart()); + emitter.Emit(new Scalar("name")); + emitter.Emit(new Scalar(v.VirtualFunctionName)); + emitter.Emit(new Scalar("offset")); + emitter.Emit(new Scalar(v.Offset.ToString())); + if (v.VirtualFunctionReturnType != null) { + emitter.Emit(new Scalar("return_type")); + emitter.Emit(new Scalar(v.VirtualFunctionReturnTypeOverride ?? v.VirtualFunctionReturnType.FixTypeName())); + } + if (v.VirtualFunctionParameters != null) { + emitter.Emit(new Scalar("parameters")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); + foreach (var parameter in v.VirtualFunctionParameters) { + ProcessedFieldConverter.Instance.WriteYaml(emitter, parameter, parameter.GetType(), serializer); + } + emitter.Emit(new SequenceEnd()); + } + emitter.Emit(new MappingEnd()); + } + + public static readonly IYamlTypeConverter Instance = new ProcessedVirtualFunctionConverter(); +} + +public class ProcessedStaticMembersConverter : IYamlTypeConverter { + public bool Accepts(Type type) => type == typeof(ProcessedStaticMembers); + public object? ReadYaml(IParser parser, Type type, ObjectDeserializer rootDeserializer) => throw new NotImplementedException(); + public void WriteYaml(IEmitter emitter, object? value, Type type, ObjectSerializer serializer) { + if (value is not ProcessedStaticMembers s) return; + emitter.Emit(new MappingStart()); + emitter.Emit(new Scalar("name")); + emitter.Emit(new Scalar(s.Name)); + emitter.Emit(new Scalar("signature")); + emitter.Emit(new Scalar(s.Signature)); + emitter.Emit(new Scalar("relative_follow_offsets")); + emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block)); + foreach (var offset in s.RelativeFollowOffsets) { + emitter.Emit(new Scalar(offset.ToString())); + } + emitter.Emit(new SequenceEnd()); + emitter.Emit(new Scalar("is_pointer")); + emitter.Emit(new Scalar(s.IsPointer.ToString())); + emitter.Emit(new Scalar("return_type")); + emitter.Emit(new Scalar(s.ReturnType.FixTypeName())); + emitter.Emit(new MappingEnd()); + } + + public static readonly IYamlTypeConverter Instance = new ProcessedStaticMembersConverter(); +} diff --git a/FFXIVClientStructs/FFXIV/Client/Game/UI/ContentsFinder.cs b/FFXIVClientStructs/FFXIV/Client/Game/UI/ContentsFinder.cs index 0e43258bc3..4df138be04 100644 --- a/FFXIVClientStructs/FFXIV/Client/Game/UI/ContentsFinder.cs +++ b/FFXIVClientStructs/FFXIV/Client/Game/UI/ContentsFinder.cs @@ -10,7 +10,7 @@ public unsafe partial struct ContentsFinder { [StaticAddress("0F B6 FA 48 8D 0D ?? ?? ?? ?? E8 ?? ?? ?? ?? 80 78 59 02", 6)] public static partial ContentsFinder* Instance(); - [FieldOffset(0x18)] public LootRule LootRules; + [FieldOffset(0x18)] public ContentsFinderQueueInfo.LootRule LootRules; [FieldOffset(0x19)] public bool IsUnrestrictedParty; [FieldOffset(0x1A)] public bool IsMinimalIL; @@ -22,12 +22,6 @@ public unsafe partial struct ContentsFinder { [MemberFunction("E8 ?? ?? ?? ?? 33 FF 0F B6 48")] public partial ContentsFinderQueueInfo* GetQueueInfo(); - - public enum LootRule : byte { - Normal = 0, - GreedOnly = 1, - Lootmaster = 2 - } } [GenerateInterop] @@ -48,7 +42,7 @@ public unsafe partial struct ContentsFinderQueueInfo { [FieldOffset(0x5A)] public byte QueuedContentRouletteId; [FieldOffset(0x5B)] public sbyte ClampedPositionInQueue; [FieldOffset(0x5C)] public sbyte PositionInQueue; - [FieldOffset(0x5D)] public ContentsFinder.LootRule PoppedContentLootRule; + [FieldOffset(0x5D)] public LootRule PoppedContentLootRule; [FieldOffset(0x60)] public bool PoppedContentIsInProgress; @@ -101,7 +95,7 @@ public partial void OnQueuePop( uint contentId, nint a4, bool isInProgressParty, - ContentsFinder.LootRule lootRule, + LootRule lootRule, ulong inProgressPartyStartTimestamp, nint a8, bool isUnrestrictedParty, @@ -110,6 +104,12 @@ public partial void OnQueuePop( bool isExplorerMode, bool isLevelSync, bool isLimitedLeveling); + + public enum LootRule : byte { + Normal = 0, + GreedOnly = 1, + Lootmaster = 2 + } } [StructLayout(LayoutKind.Explicit, Size = 0x10)] diff --git a/FFXIVClientStructs/FFXIV/Common/Component/Excel/ExcelSheet.cs b/FFXIVClientStructs/FFXIV/Common/Component/Excel/ExcelSheet.cs index e4b097738e..37ef4b2591 100644 --- a/FFXIVClientStructs/FFXIV/Common/Component/Excel/ExcelSheet.cs +++ b/FFXIVClientStructs/FFXIV/Common/Component/Excel/ExcelSheet.cs @@ -74,12 +74,12 @@ public struct LinkedListContainer { public struct WaiterContainer { [FieldOffset(0x00), CExporterBaseType] public LinkList Base; [FieldOffset(0x18)] public int Count; + } - [StructLayout(LayoutKind.Explicit, Size = 0x18)] - public struct WaiterEntry { - [FieldOffset(0x00)] public ExcelSheetWaiter* Waiter; - [FieldOffset(0x08)] public int Index; // ? - [FieldOffset(0x10)] public WaiterEntry* Next; - } + [StructLayout(LayoutKind.Explicit, Size = 0x18)] + public struct WaiterEntry { + [FieldOffset(0x00)] public ExcelSheetWaiter* Waiter; + [FieldOffset(0x08)] public int Index; // ? + [FieldOffset(0x10)] public WaiterEntry* Next; } } diff --git a/FFXIVClientStructs/STD/StdLinkedList.cs b/FFXIVClientStructs/STD/StdLinkedList.cs index 90901a7c76..40d72892cb 100644 --- a/FFXIVClientStructs/STD/StdLinkedList.cs +++ b/FFXIVClientStructs/STD/StdLinkedList.cs @@ -15,6 +15,7 @@ public struct Node { public Enumerator GetEnumerator() => new(this); + [CExporterIgnore] public struct Enumerator(StdLinkedList list) { private Node* _current = list.Start; diff --git a/Ghidra/Getting Started.md b/Ghidra/Getting Started.md index e4414b3323..843a19c3dd 100644 --- a/Ghidra/Getting Started.md +++ b/Ghidra/Getting Started.md @@ -40,18 +40,23 @@ Note that after each patch, you'll have to re-import, re-analyze, and re-run any

# Script dependency installation: -We use [..\ida\ffxiv_idarename.py](../ida/ffxiv_idarename.py) to apply data.yml. The same script works for both IDA and Ghidra, howver it's slightly more complicated in Ghidra as we need to add dependencies to the PyGhidra environment manually. +We use [rename.py](../scripts/ghidra/rename.py) to apply data.yml and [struct_import.py](../scripts/ghidra/struct_import.py) to import the structs. They follow the same flow as their IDA counterparts in `scripts/ida`, however it's slightly more complicated in Ghidra as we need to add dependencies to the PyGhidra environment manually. - Install a copy of [Python 3](https://www.python.org/downloads/).
-Note: it has to be major version 3, so something like [Python 3.14.3](https://www.python.org/downloads/release/python-3143/) works.
+Note: any Python version that is still listed on the [Python versions status page](https://devguide.python.org/versions/) is supported, so something like [Python 3.14.3](https://www.python.org/downloads/release/python-3143/) works.
+Ghidra only ships JPype wheels up to Python 3.13, on newer versions PyGhidra builds JPype from source and needs a C++ compiler for that (the MSVC Build Tools on Windows).
- **Check your PyGhidra log for which environment is being used.** For example, if you've using a "virtual environment", it should be something similar to either of the following two paths:
Windows: `%AppData%\ghidra\ghidra_VERSION\venv`
Linux: `/home/user/.config/ghidra/ghidra_VERSION/venv` - Execute the following in your terminal, using Python from the same environment as PyGhidra:
-venv: `.../venv/bin/python -m pip install pyyaml==6.0.3 anytree==2.13.0`
-system: `python -m pip install pyyaml==6.0.3 anytree==2.13.0` +venv: `.../venv/bin/python -m pip install pyyaml anytree pythonnet libclang`
+system: `python -m pip install pyyaml anytree pythonnet libclang`
+`pythonnet` is only used by struct_import.py to read the Excel headers with Lumina, it needs the [.NET 10 runtime](https://dotnet.microsoft.com/download/dotnet/10.0) installed, the same version FFXIVClientStructs uses.
+`libclang` is only used by struct_import.py to work out the layout of the structs. Outside of Windows it also needs the MSVC and Windows SDK headers, download them with [mstorsjo/msvc-wine](https://github.com/mstorsjo/msvc-wine)'s `vsdownload.sh`.
+The `libclang` package ships libclang 18 while the MSVC headers want clang 20 or newer, set the `LIBCLANG_LIBRARY_FILE` environment variable to a libclang 20 or newer (ex. `/usr/lib/llvm-20/lib/libclang-20.so.1`) to avoid the errors this reports.
+struct_import.py looks for the game in the [XIVLauncher](https://goatcorp.github.io/) or XLCore config, the default install folder and the Steam libraries, and asks for the folder holding both `game` and `boot` if it is not found. It also needs internet access to download Lumina and [EXDSchema](https://github.com/xivdev/EXDSchema). - Open the Script Manager

@@ -59,13 +64,13 @@ system: `python -m pip install pyyaml==6.0.3 anytree==2.13.0` - Click the plus to add a directory

-- Add both `FFXIVClientStructs/ida` (for ffxiv_idarename.py) and `FFXIVClientStructs/Ghidra/scripts` (for ffxiv_exdgetters and SigScanner) +- Add `FFXIVClientStructs/scripts/ghidra` (for rename.py, struct_import.py and full_run.py) After adding both, select `__UserScripts` and you should see all of the scripts. SigScanner should already have a key binding assigned of `Ctrl-Alt-S`. Assign any others you wish, and ensure the checkbox is ticked.

# The Scripts -## ffxiv_idarename.py +## rename.py First a quick intro. The analyzer reads through the code and attempts to determine things like what blocks of code are instructions vs pointers vs data, etc. When it identifies things, it'll add labels to them: - FUN_14#######: for what it believes to be functions - PTR_FUN_14#######: for what it believes to be pointers to functions @@ -81,7 +86,7 @@ As an example, if you look in Dalamud's Data window You'll see the vtbl has an address like`7FF6DDEAD480 ffxiv_dx11.exe+1A0D480`. In Ghidra, if you were to `Navigation -> Go To...` (or simply press `g`) and typed 141A0D480, you'd arrive at the vtable. -What you'll find in [data.yml](../ida/data.yml) are definitions like: +What you'll find in [data.yml](../scripts/data.yml) are definitions like: ```yaml Client::System::Framework::Framework: instances: @@ -101,7 +106,7 @@ What you'll find in [data.yml](../ida/data.yml) are definitions like: ... ``` -When you run the ffxiv_idarename.py script, it'll rename `FUN_1400901d0` to `Client::System::Framework::Framework.ctor` (per the first line under `funcs:`). It'll turn: +When you run the rename.py script, it'll rename `FUN_1400901d0` to `Client::System::Framework::Framework.ctor` (per the first line under `funcs:`). It'll turn:

Into: @@ -111,8 +116,8 @@ And much more. So go ahead and run it. Either by opening the script manager, r You can re-run this script as many times as you like, it'll just re-process data.yml, so it's much easier to update data.yml when you map out a new vtable and re-run than to try and set all the labels yourself. -## ffxiv_exdgetters.java -After having run ffxiv_idarename.py, we'll have a bunch of functions referencing Exd methods like: +## struct_import.py +After having run rename.py, we'll have a bunch of functions referencing Exd methods like: ```cpp undefined8 FUN_1406cb6c0(undefined4 param_1) { @@ -129,7 +134,12 @@ undefined8 FUN_1406cb6c0(undefined4 param_1) } ``` -The label `Component::Exd::ExdModule.GetRowBySheetIndexAndRowIndex` came from data.yml, but unfortunately it's still not obvious exactly what that method is fetching. The sheet indexes are fairly well-known, and this script parses these and relabels the function from `FUN_1406cb6c0` to `undefined8 Client::ExdData::GetClassJob(undefined4 param_1)`, which is much more useful. Go ahead and run this too. +The label `Component::Exd::ExdModule.GetRowBySheetIndexAndRowIndex` came from data.yml, but unfortunately it's still not obvious exactly what that method is fetching. The sheet indexes are fairly well-known, and this script parses these and relabels the function from `FUN_1406cb6c0` to `Component::Exd::ExdModule_GetClassJobRow_1`, returning a `Component::Exd::Sheets::ClassJob*`, which is much more useful. + +It also imports the Excel sheets and everything in [structs.yml](../scripts/structs.yml) as data types, and can apply the types of the virtual and member functions. It asks you to confirm the game version of data.yml, for the folder of the MSVC headers (outside of Windows) and which function types to update. Go ahead and run this too. + +## full_run.py +Runs rename.py and struct_import.py in sequence, then lets Ghidra analyze the changes. ## SigScanner.java In plugins (and FFXIVClientStructs) we use signatures rather than offsets. This is because offsets change with every patch, while signatures are far more durable. diff --git a/README.md b/README.md index 76e7174c1a..3a545caf6f 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ We make extensive use of [C# Source Generators](https://learn.microsoft.com/en-u ### Reverse Engineering Rename Database -A database and script(s) are maintained in the [ida](https://github.com/aers/FFXIVClientStructs/tree/main/ida) folder which can be used to import a large number of location names to IDA or Ghidra. This database is updated with every patch, although keep in mind this is volunteer work and some patches require more effort than others. There is more info in the readme in the folder itself. +A database and script(s) are maintained in the [scripts](https://github.com/aers/FFXIVClientStructs/tree/main/scripts) folder which can be used to import a large number of location names to IDA or Ghidra. This database is updated with every patch, although keep in mind this is volunteer work and some patches require more effort than others. There is more info in the readme in the folder itself. ### Credits @@ -96,7 +96,7 @@ namespace FFXIVClientStructs.FFXIV.Component.GUI; public unsafe partial struct AtkResNode : ICreatable { } ``` -Native game classes are represented as explicit layout structs. If the official name of the class is available (via [old RTTI](https://github.com/aers/FFXIVClientStructs/blob/main/ida/classinformer.csv)) use that for the name and namespace. For new classes or classes without virtual functions, make up a name that seems appropriate. +Native game classes are represented as explicit layout structs. If the official name of the class is available (via [old RTTI](https://github.com/aers/FFXIVClientStructs/blob/main/scripts/classinformer.csv)) use that for the name and namespace. For new classes or classes without virtual functions, make up a name that seems appropriate. If the struct has unsafe members, mark the struct unsafe rather than the individual members. If you are using a generator, the struct must also be partial. If you are unable to get the exact size, use your best estimate. diff --git a/ida/README.md b/ida/README.md deleted file mode 100644 index 176730e650..0000000000 --- a/ida/README.md +++ /dev/null @@ -1,48 +0,0 @@ -## ffxiv_idarename.py -Despite the name being a misnomer at this point, this script supports both Ghidra and IDA. -This script ingests the `data.yml` file in the same directory, and renames various offsets as globals, functions, vtables and such. - -#### IDA dependency installation: -`pip install pyyaml anytree` or `python -m pip install pyyaml anytree` in whichever version of python you are currently using with IDA Pro. -If you are familiar with Poetry, the `idarename` extras package will do the same. -This could be either Python2 or Python3. - -Additionally, you will need to place "idauser.cfg" from this repository in "%AppData%\Hex-Rays\IDA Pro\". IDA doesn't allow you to use certain characters in names and this config changes that. - -#### Ghidra dependency installation: -See [Getting Started](../Ghidra/Getting%20Started.md) for information on how to set up PyGhidra. - -## ffxiv_sigmaker.py -> [!WARNING] -> Script file is no longer maintained by anyone. Most people use most people use this [sigmaker](https://github.com/caraxi/sigmaker-x64) instead - -This script ingests the data.yml file in the same directory, and generates automatic signatures for consumption by the source-gen modules. - -This support **Python3** and **IDA** only. - -#### Dependency installation: -`pip install dacite pyyaml ruamel-yaml` or `python -m pip install dacite pyyaml ruamel-yaml` in whichever version of python you are currently using with IDA Pro. -If you are familiar with Poetry, the `sigmaker` extras package will do the same. - -## ffxiv_exdgetters.py -> [!WARNING] -> Does not work with IDA 9 due to changes made on the application - -This script ingests the `exh` files from the base game and renames various functions, including setting the return type of the functions to the propper sheet struct. - -## ffxiv_structimporter.py -> [!WARNING] -> Does not work with IDA 9 due to changes made on the application - -This script ingests the `ffxiv_structs.yml` file and forms the proper structs and assigns the member function return and param types correctly - -## ffxiv_fullrun_ida.py -> [!WARNING] -> Does not work with IDA 9 due to changes made on the application - -This script runs `ffxiv_idarename.py`, `ffxiv_exdgetters.py`, `ffxiv_structimporter.py` in sequence - -## classinformer.csv -Once upon a time, someone did a bad thing and released FFXIV with the RTTI data intact. -This is an export of that RTTI data using the ClassInformer IDA Pro plugin. -It is useful as a baseline for many hierarchies and class names, be aware however it is several years old at this point. diff --git a/ida/data_schema.py b/ida/data_schema.py deleted file mode 100644 index b0ea1b96b8..0000000000 --- a/ida/data_schema.py +++ /dev/null @@ -1,34 +0,0 @@ -class DefinedDataClassInstance: - def __init__(self, ea, pointer=False, name=None): - # type: (int, bool, str | None) -> None - self.ea = ea - self.pointer = pointer - self.name = name - -class DefinedDataClassVtable: - def __init__(self, ea, base=None, size=None): - # type: (int, None | str, None | int) -> None - self.ea = ea - self.base = base - self.size = size - -class DefinedDataClassFunction: - def __init__(self, num, name): - # type: (int, str) -> None - self.num = num - self.name = name - -class DefinedDataClass: - def __init__(self, name, instances=[], vtbls=[], functions=[], vfuncs=[]): - # type: (str, list[DefinedDataClassInstance], list[DefinedDataClassVtable], list[DefinedDataClassFunction], list[DefinedDataClassFunction]) -> None - self.instances = instances - self.vtbls = vtbls - self.functions = functions - self.vfuncs = vfuncs - self.name = name - -class DefinedData: - def __init__(self, classes=[]): - # type: (list[DefinedDataClass]) -> None - self.classes = classes - pass \ No newline at end of file diff --git a/ida/ffxiv_data_struct_matcher.py b/ida/ffxiv_data_struct_matcher.py deleted file mode 100644 index ec204c7370..0000000000 --- a/ida/ffxiv_data_struct_matcher.py +++ /dev/null @@ -1,212 +0,0 @@ -import idaapi -import idc -import ida_bytes -import ida_search -import ida_typeinf -import ida_funcs -import ida_name -import ida_kernwin -import os -import sys -from ida_wrapper import IdaInterface -from structs_schema import * -from data_schema import * -from yaml import load - -if sys.version_info[0] >= 3: - long = int - -try: - from yaml import CSafeLoader as Loader -except ImportError: - from yaml import SafeLoader as Loader - - -def get_image_base(): - return idaapi.get_imagebase() - - -dataDir = os.path.dirname(os.path.realpath(__file__)) -structFileName = os.path.join(dataDir, "ffxiv_structs.yml") -dataFileName = os.path.join(dataDir, "data.yml") - -ida = IdaInterface() - -def get_structs() -> DefinedStructExport: - dic: dict[str, dict[str, list[dict[str, str | int | list[dict[str, str | int]]]]]] = load(open(structFileName), Loader=Loader) - enums = [] - structs = [] - for enum in dic["enums"]: - enums.append( - DefinedStructEnum( - enum["name"], - enum["type"], - enum["underlying"], - enum["namespace"], - enum["flags"], - enum["values"], - ) - ) - for struct in dic["structs"]: - fields = [] - virtual_functions = None - member_functions = [] - static_member_functions = None - static_members = None - for field in struct["fields"]: - base = field["base"] if "base" in field else False - if "size" in field: - fields.append( - DefinedStructFixedField( - field["name"], - field["type"], - field["offset"], - base, - field["size"], - ) - ) - elif "return_type" in field: - parameters = [] - for param in field["parameters"]: - parameters.append( - DefinedStructFuncParam(param["name"], param["type"]) - ) - fields.append( - DefinedStructFuncField( - field["name"], - field["type"], - field["offset"], - base, - field["return_type"], - parameters, - ) - ) - else: - fields.append( - DefinedStructField( - field["name"], field["type"], field["offset"], base - ) - ) - if "virtual_functions" in struct: - virtual_functions = [] - for vfunc in struct["virtual_functions"]: - parameters = ( - [ - DefinedStructFuncParam(param["name"], param["type"]) - for param in vfunc["parameters"] - ] - if "parameters" in vfunc - else None - ) - virtual_functions.append( - DefinedStructVFunc( - vfunc["name"], - vfunc["return_type"] if "return_type" in vfunc else None, - vfunc["offset"], - parameters, - ) - ) - for memfunc in struct["member_functions"]: - parameters = [] - for param in memfunc["parameters"]: - parameters.append(DefinedStructFuncParam(param["name"], param["type"])) - member_functions.append( - DefinedStructMemFunc( - memfunc["signature"], - memfunc["return_type"], - parameters, - memfunc["name"], - ) - ) - if "static_member_functions" in struct: - static_member_functions = [] - for smemfunc in struct["static_member_functions"]: - parameters = [] - for param in smemfunc["parameters"]: - parameters.append( - DefinedStructFuncParam(param["name"], param["type"]) - ) - static_member_functions.append( - DefinedStructMemFunc( - smemfunc["signature"], - smemfunc["return_type"], - parameters, - smemfunc["name"], - ) - ) - if "static_members" in struct: - static_members = [] - for sm in struct["static_members"]: - static_members.append( - DefinedStructStaticMember( - sm["name"], - sm["signature"], - sm["relative_follow_offsets"], - sm["return_type"], - sm["is_pointer"] if "is_pointer" in sm else False, - ) - ) - size = None - if "size" in struct: - size = struct["size"] - vtable_size = None - if "vtable_size" in struct: - vtable_size = struct["vtable_size"] - structs.append( - DefinedStruct( - struct["name"], - struct["type"], - struct["namespace"], - fields, - size, - vtable_size, - virtual_functions, - member_functions, - struct["union"], - static_member_functions, - static_members, - ) - ) - return DefinedStructExport(enums, structs) - - -def get_data(): - dic: dict[str, str | dict[int, str] | dict[str, dict[str, list[dict[str, int | bool | str]] | dict[int, str]]]] = load(open(dataFileName), Loader=Loader) - classes = [] - for className, classInstance in dic["classes"].items(): - if classInstance == None: - classes.append(DefinedDataClass(className)) - continue - instances = [] - if "instances" in classInstance: - for instance in classInstance["instances"]: - instances.append( - DefinedDataClassInstance( - instance["ea"], - instance["pointer"] if "pointer" in instance else False, - instance["name"] if "name" in instance else None, - ) - ) - classes.append(DefinedDataClass(className, instances)) - return DefinedData(classes) - - -def load_data(): - structs = get_structs().structs - for data in get_data().classes: - struct = next((struc for struc in structs if struc.type == data.name), None) - if struct is None or struct.static_members is None: - continue - for static_member in struct.static_members: - ea = ida.search_binary(0, static_member.signature, ida_search.SEARCH_DOWN) - for offset in static_member.relative_offsets: - ea = ea + offset - dword = ida.get_dword(ea) - ea = ea + 4 + dword - instance = next((inst for inst in data.instances if inst.ea == ea), None) - if instance is None: - continue - if ea == instance.ea and static_member.is_pointer != instance.pointer: - print(f"Found pointer discrepancy at {ea:X} on {data.name}") - -load_data() diff --git a/ida/ffxiv_exdgetters.py b/ida/ffxiv_exdgetters.py deleted file mode 100644 index 8c66eb37c8..0000000000 --- a/ida/ffxiv_exdgetters.py +++ /dev/null @@ -1,501 +0,0 @@ -# Automagically labels most exd getter functions along with a hint indicating which sheet/sheet id its fetching from -# @category __UserScripts -# @menupath Tools.Scripts.ffxiv_exdgetters -# @runtime PyGhidra - -from json import load -from os import getenv -from os.path import join -import sys -from luminapie.game_data import GameData, ParsedFileName -from luminapie.excel import ExcelListFile, ExcelHeaderFile -from abc import abstractmethod - - -class BaseApi: - @abstractmethod - def create_enum_struct(self, name, values, width=0): - # type: (str, dict[int, str], int) -> None - pass - - @abstractmethod - def create_struct(self, name, fields): - # type: (str, dict[str, str]) -> None - pass - - @abstractmethod - def set_func_name(self, ea, name, cmt): - # type: (int, str, str) -> None - pass - - @abstractmethod - def process_pattern(self, pattern): - # type: (str) -> None - pass - - -api = None - -if api is None: - try: - import idaapi - import ida_bytes - import ida_search - import ida_typeinf - import ida_hexrays - import ida_name - import idc - from ida_wrapper import IdaInterface - except ImportError as e: - print("Warning: Unable to load IDA failed with {0}".format(e)) - else: - # noinspection PyUnresolvedReferences - class IdaApi(BaseApi, IdaInterface): - def do_pattern(self, pattern, suffix, sheetSearchPattern, sheetSearchOffset): - ea = 0 - - while True: - ea = self.search_binary(ea + 1, pattern, ida_search.SEARCH_DOWN) - - if ea == 0xFFFFFFFFFFFFFFFF: - break - - if sheetSearchPattern == None: - sheetEa = ea + sheetSearchOffset - else: - sheetEa = self.search_binary(ea, sheetSearchPattern, ida_search.SEARCH_DOWN) + sheetSearchOffset - sheetIdx = self.get_dword(sheetEa) - origName = idc.get_func_name(ea) - - # don't rename any funcs that are already named - if origName[0:4] == "sub_": - if exd_map.get(sheetIdx) == None: - print( - f"Func @ 0x{ea:X} references unknown sheet {sheetIdx}! sheet dword offset = 0x{sheetEa:X}" - ) - continue - - sheetName = exd_map[sheetIdx] - - if suffix == None: - suffix = "" - - fnName = "Component::Exd::ExdModule_Get%s%s" % ( - exd_map[sheetIdx], - suffix, - ) - - uniquifier = 0 - while True: - uniqueName = fnName + ( - f"_{uniquifier}" if uniquifier > 0 else "" - ) - - # check if this name is unique now - if ( - idc.get_name_ea_simple(uniqueName) == idc.BADADDR - and uniquifier > 0 - ): - fnName = uniqueName - break - - uniquifier += 1 - - self.set_func_name( - ea, fnName, "Sheet: {0} ({1})".format(sheetName, sheetIdx) - ) - - if suffix == "SheetIndex": - continue - - tif, funcdata = ( - ida_typeinf.tinfo_t(), - ida_typeinf.func_type_data_t(), - ) - - ida_typeinf.guess_tinfo(tif, ea) - if not tif.get_func_details(funcdata): - ida_hexrays.decompile(ea) - ida_typeinf.guess_tinfo(tif, ea) - if not tif.get_func_details(funcdata): - print( - "Failed to get func details for %s @ %X" - % (fnName, ea) - ) - continue - - - if suffix == "RowAndSubRowId": - tif = self.get_tinfo_from_type( - f"{exd_struct_map[sheetIdx]} *__fastcall func(unsigned int rowId, unsigned int subRowId);" - ) - elif suffix == "RowCount": - tif = self.get_tinfo_from_type( - f"{exd_struct_map[sheetIdx]} *__fastcall func();" - ) - else: - tif = self.get_tinfo_from_type( - f"{exd_struct_map[sheetIdx]} *__fastcall func(unsigned int rowIdOrIndex);" - ) - - ida_typeinf.apply_tinfo(ea, tif, ida_typeinf.TINFO_DEFINITE) - - def create_enum_struct(self, name, values, width = 0): - # type: (str, dict[int, str], int) -> None - if len(name.split("::")) > 3: - sheet_name = name.split("::")[-2] - else: - sheet_name = name.split("::")[-1] - - # create or reset if exists - enum_id = self.get_enum_id(name) - if enum_id == idaapi.BADADDR: - enum_id = self.create_enum(name) - else: - self.delete_enum_members(enum_id) - idc.set_enum_bf(enum_id, False) - - self.set_enum_width(enum_id, width) - if width == 1: - if idaapi.IDA_SDK_VERSION < 900: - self.add_enum_member(enum_id, f"{sheet_name}.tmp", self.get_enum_default_mask(enum_id)) - self.set_enum_as_bf(enum_id) - - for key in values: - self.add_enum_member(enum_id, f"{sheet_name}.{values[key]}", key) - - if width == 1 and idaapi.IDA_SDK_VERSION < 900: - self.remove_enum_member(enum_id, "tmp", sheet_name) - - def create_struct(self, name, fields): - idaapi.begin_type_updating(idaapi.UTP_STRUCT) - struct_id = self.get_struct_id(name) - if struct_id == idaapi.BADADDR: - struct_id = self.create_struct_type(name) - else: - self.remove_struct_members(struct_id) - struct_type = self.get_struct(struct_id) - for [index, [type, name]] in fields.items(): - if ( - self.get_idc_type_from_ida_type( - self.clean_struct_name(type) - ) - == self.get_struct_flag() - ): - type = self.clean_struct_name(type) - self.create_struct_member( - struct_type, - name, - index, - self.get_idc_type_from_ida_type(type), - self.get_struct_opinfo_from_type(type), - self.get_size_from_ida_type(type), - ) - elif ( - self.get_idc_type_from_ida_type(type) - == self.get_enum_flag() - ): - self.create_struct_member( - struct_type, - name, - index, - self.get_idc_type_from_ida_type(type), - self.get_enum_opinfo_from_type(type), - self.get_size_from_ida_type(type), - ) - else: - self.create_struct_member( - struct_type, - name, - index, - self.get_idc_type_from_ida_type(type), - None, - self.get_size_from_ida_type(type), - ) - meminfo = self.get_struct_member_by_name(struct_type, name) - self.set_struct_member_info( - struct_type, meminfo, 0, self.get_tinfo_from_type(type), 0 - ) - idaapi.end_type_updating(idaapi.UTP_STRUCT) - - def set_func_name(self, ea, name, cmt): - ida_name.set_name(ea, name) - ida_bytes.set_cmt(ea, cmt, 0) - - def process_pattern(self, pattern: str): - (suffix, search) = exd_func_patterns[pattern] - if suffix is not None: - print(f"Finding exd funcs of {suffix}... please wait.") - (searchPattern, searchOffset) = search - self.do_pattern(pattern, suffix, searchPattern, searchOffset) - - def get_all_eas(self, pattern: str): - ea = self.search_binary(0, pattern, ida_search.SEARCH_DOWN) - while ea != idc.BADADDR: - yield ea - ea = ea + 1 - ea = self.search_binary(ea, pattern, ida_search.SEARCH_DOWN) - - def comment_rows(self, pattern: dict[str, int], values: dict[int, str]): - for pattern_key in pattern: - for ea in list(self.get_all_eas(pattern_key)): - try: - sheetIdx = self.get_dword(ea + pattern[pattern_key]) - sheetName = values[sheetIdx] - ida_bytes.set_cmt(ea, "Sheet: {0} ({1})".format(sheetName, sheetIdx), 0) - except Exception as e: - print(f"An unexpected error occurred at 0x{hex(ea)}: {e}") - pass - - api = IdaApi() - -if api is None: - try: - import ghidra - import re - try: - from ghidra.ghidra_builtins import * # ghidra-stubs - except ImportError: - pass - - from ghidra.program.model.data import * - from ghidra.program.model.listing import * - from ghidra.program.model.symbol import SourceType - from ghidra.app.util import SymbolPathParser - from java.util import ArrayList - - except ImportError: - print("Warning: Unable to load Ghidra") - else: - # noinspection PyUnresolvedReferences - class GhidraApi(BaseApi): - def create_enum_struct(self, name, values, width = 0): - # type: (str, dict[int, str], int) -> None - path = self.get_datatype_path(name) - enum_dt = EnumDataType(path.getCategoryPath(), path.getDataTypeName(), width or 8) - is_sheets_enum = name == "Component::Exd::SheetsEnum" - for enum_value, enum_name in values.items(): - if monitor.isCancelled(): - break - if not is_sheets_enum and "_" in enum_name: - enum_name = "".join(enum_name.split("_")[1:]) - enum_dt.add(enum_name, enum_value) - if width == 0: - enum_dt.setLength(enum_dt.getMinimumPossibleLength()) - if enum_dt.getName(0) is None: - enum_dt.add("None", 0) - dt = currentProgram.getDataTypeManager().getDataType(path) - if dt is None: - currentProgram.getDataTypeManager().addDataType(enum_dt, None) - else: - dt.replaceWith(enum_dt) - - def create_struct(self, name, fields): - # type: (str, dict[str, str]) -> None - dt_path = self.get_datatype_path(name) - struct = StructureDataType(dt_path.getCategoryPath(), dt_path.getDataTypeName(), 0) - struct.setExplicitMinimumAlignment(4) - for [offset, [type_name, field_name]] in fields.items(): - if monitor.isCancelled(): - break - field_dt = currentProgram.getDataTypeManager().getDataType(self.get_datatype_path(type_name)) - if field_dt is None: - print(f"Warning: Data type {name} field {field_name} has missing type {type_name} ({dt_path})") - continue - struct.insertAtOffset(offset, field_dt, -1, field_name, None) - dt = currentProgram.getDataTypeManager().getDataType(dt_path) - if dt is None: - currentProgram.getDataTypeManager().addDataType(struct, None) - else: - dt.replaceWith(struct) - - def set_func_name(self, ea, name, cmt): - # type: (int, str, str) -> None - func = getFunctionAt(ea) - if func is not None: - func.setName(None, SourceType.DEFAULT) - func.setName(name, SourceType.USER_DEFINED) - func.setComment(cmt) - - def process_pattern(self, pattern): - # type: (str) -> None - (suffix, _) = exd_func_patterns[pattern] - pattern = "".join(["\\x" + x if x != "?" else "." for x in pattern.split(" ")]) - if suffix is not None: - print(f"Finding exd funcs of {suffix}... please wait.") - self.do_pattern(pattern, suffix) - - def do_pattern(self, pattern, suffix): - # type: (string, string) -> None - block = currentProgram.getMemory().getBlock(".text") - address_set = currentProgram.getAddressFactory().getAddressSet(block.getStart(), block.getEnd()) - result_list = findBytes(address_set, pattern, 8192, 1) - for ea in result_list: - if monitor.isCancelled(): - break - sheet_index = self.get_sheet_index(ea) - sheet_name = exd_map.get(sheet_index) - if sheet_name is None: - print(f"Warning: sheet index {sheet_index} is undefined at {ea}") - continue - func_name = f"Component::Exd::ExdModule.Get{sheet_name}{suffix}" - self.set_func_name(ea, func_name, f"Sheet: {sheet_name} ({sheet_index})") - if suffix == "SheetIndex": - return - - return_type = PointerDataType(VoidDataType.dataType) - if suffix == "RowCount": - return_type = IntegerDataType() - else: - path = self.get_datatype_path(f"{exd_struct_map[sheet_index]}") - dt = currentProgram.getDataTypeManager().getDataType(path) - if dt is not None: - return_type = PointerDataType(dt) - - return_var = ReturnParameterImpl(return_type, currentProgram) - arg_vars = ArrayList() - arg_vars.add(ParameterImpl("rowId", UnsignedIntegerDataType(), currentProgram)) - if suffix == "RowAndSubRowId": - arg_vars.add(ParameterImpl("subRowId", UnsignedIntegerDataType(), currentProgram)) - elif suffix == "RowCount": - arg_vars.clear() - - update_type = Function.FunctionUpdateType.DYNAMIC_STORAGE_ALL_PARAMS - getFunctionAt(ea).updateFunction("__fastcall", return_var, arg_vars, update_type, False, SourceType.USER_DEFINED) - - def get_sheet_index(self, ea): - # type: (Address) -> int - func = getFunctionAt(ea) - if func is None and disassemble(ea): - func = createFunction(ea, None) - if func is None: - return -1 - min_address = func.getBody().getMinAddress().getOffset() - instructions = currentProgram.getListing().getInstructions(func.getBody(), True) - for insn in instructions: - if monitor.isCancelled(): - break - if insn.getFlowType().isCall(): - return self.get_rdx_arg(insn, min_address) - return -1 - - def get_rdx_arg(self, insn, min_address): - # type: (Instruction, int) -> int - target = insn.getRegister("RDX") - while insn is not None and insn.getMinAddress().getOffset() >= min_address: - if monitor.isCancelled(): - break - insn = insn.getPrevious() - if insn is None: - break - outp = insn.getResultObjects() - if outp.length == 0 or outp[0] != target: - continue - value = insn.getScalar(1) - if value is not None: - return value.getValue() - return -1 - - def get_datatype_path(self, name): - # type: (str) -> DataTypePath - if name == "__int8": name = "char" - if name == "__int16": name = "short" - if name == "__int32": name = "int" - if name == "__int64": name = "longlong" - if name == "unsigned __int8" or name == "unsigned char": name = "byte" - if name == "unsigned __int16" or name == "unsigned short": name = "ushort" - if name == "unsigned __int32" or name == "unsigned int": name = "uint" - if name == "unsigned __int64" or name == "unsigned long long": name = "ulonglong" - path_parts = SymbolPathParser.parse(name) - return DataTypePath("/" + "/".join(path_parts[:-1]), path_parts[-1]) - - def comment_rows(self, pattern: dict[str, int], values: dict[int, str]): - pass - - api = GhidraApi() - -if api is None: - print("Warning: No API available, exiting.") - exit(1) - - -if sys.platform.startswith("linux"): - launcher_ini = join(getenv("HOME"), ".xlcore", "launcher.ini") - game_path = None - - f = open(launcher_ini, "r") - try: - for line in f: - if line.startswith("GamePath="): - game_path = line.split("=", 1)[1].strip() - break - finally: - f.close() - - if not game_path: - raise ValueError("GamePath key not found in {0}".format(launcher_ini)) -else: - f = open(join(getenv("APPDATA"), "XIVLauncher", "launcherConfigV3.json"), "r") - try: - config = load(f) - finally: - f.close() - game_path = config["GamePath"] - -game_data = GameData(join(game_path, "game")) - -# nb: "pattern": ("func suffix", ("instance pointer sig" or None, offset from sig start or func start)) -exd_func_patterns: dict[str, tuple[str, tuple[str | None, int]]] = { - "48 83 EC 28 48 8B 05 ? ? ? ? 44 8B C1 BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ("Row", ("48 8B 05 ? ? ? ? 44 8B C1 BA", 11)), - "48 83 EC 28 85 C9 74 20 48 8B 05 ? ? ? ? 44 8B C1 BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ("Row", ("48 8B 05 ? ? ? ? 44 8B C1 BA", 11)), - "48 83 EC 28 48 8B 05 ? ? ? ? BA ? ? ? ? 44 0F B6 C1 48 8B 88 ? ? ? ? E8": ("Row", ("48 8B 05 ? ? ? ? BA", 8)), - "48 83 EC 28 48 8B 05 ? ? ? ? BA ? ? ? ? 44 0F B7 C1 48 8B 88 ? ? ? ? E8": ("Row", ("48 8B 05 ? ? ? ? BA", 8)), - "48 83 EC 28 48 8B 05 ? ? ? ? BA ? ? ? ? 44 0F B7 81 ? ? ? ? 48 8B 88 ? ? ? ? E8": ("Row", ("48 8B 05 ? ? ? ? BA", 8)), - "40 53 48 83 EC 20 48 8B 05 ? ? ? ? 44 8B C1 8B DA BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ("Row", ("44 8B C1 8B DA BA", 6)), - "48 83 EC 28 48 8B 05 ? ? ? ? 44 8B C1 84 D2 BA ? ? ? ? 48 8B 88 ? ? ? ? 74 07 E8": ("Row", ("44 8B C1 84 D2 BA", 6)), - "48 83 EC 38 48 8B 05 ? ? ? ? 44 8B CA 44 8B C1 48 C7 44 24 ? ? ? ? ? BA ? ? ? ? 48 C7 44 24 ? ? ? ? ? 48 8B 88 ? ? ? ? E8": ("RowAndSubRowId", ("C1 48 C7 44 24 ? ? ? ? ? BA", 11)), - "48 83 EC 38 48 8B 05 ? ? ? ? 44 8B C1 44 0F ? CA BA ? ? ? ? 48 C7 44 24 28 ? ? ? ? 48 C7 44 24 20 ? ? ? ? 48 8B 88 ? ? ? ? E8": ("RowAndSubRowId", ("44 8B C1 44 0F ? CA BA", 8)), - "48 83 EC 28 48 8B 05 ? ? ? ? 44 8D 81 ? ? ? ? BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ("RowIndex", ("05 ? ? ? ? 44 8D 81 ? ? ? ? BA", 13)), - "48 83 EC 28 8D 41 ? 3D ? ? ? ? 77 20 48 8B 05 ? ? ? ? 44 8B C1 BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ("RowIndex", ("48 8B 05 ? ? ? ? 44 8B C1 BA", 11)), - "48 83 EC 28 48 8B 05 ? ? ? ? BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ("SheetIndex", ("48 8B 05 ? ? ? ? BA", 8)), - "48 8B 05 ? ? ? ? BA ? ? ? ? 48 8B 88 ? ? ? ? E9 ? ? ? ?": ("RowCount", (None, 8)) -} - -exd_comment_patterns: dict[str, int] = { - "48 8B 05 ? ? ? ? BA ? ? ? ? 48 8B 88 40 2B 00 00": 8, - "48 8B 05 ? ? ? ? BA ? ? ? ? 44 0F": 8, - "48 8B 05 ? ? ? ? 44 8B C1 8B DA BA": 13, - "48 8B 05 ? ? ? ? 44 8B C1 BA": 11, - "48 8B 05 ? ? ? ? 44 8B CA 44 8B C1 48 C7 44 24 ? ? ? ? ? BA": 23, - "48 8B 05 ? ? ? ? 44 8B C1 84 D2 BA": 13, - "48 8B 05 ? ? ? ? 44 8B C1 44 0F ? CA BA": 15, - "48 8B 05 ? ? ? ? 44 8D 81 ? ? ? ? BA": 15 -} - -exd_map = ExcelListFile(game_data.get_file(ParsedFileName("exd/root.exl"))).dict -exd_struct_map: dict[str, str] = {} -exd_headers: tuple[dict[int, tuple[str, str]], dict[str, dict[int, str]], int] = {} - -api.create_enum_struct("Component::Exd::SheetsEnum", exd_map, 4) - -for key in exd_map: - print(f"Parsing schema for {exd_map[key]}.") - exd_headers[key] = ExcelHeaderFile( - game_data.get_file(ParsedFileName("exd/" + exd_map[key] + ".exh")), - exd_map[key] - ).map_names(game_data.get_exd_schema(exd_map[key])) - -for key in exd_headers: - [exd_header, exd_header_enums, exd_header_count] = exd_headers[key] - struct_name = f"Component::Exd::Sheets::{exd_map[key]}" - print(f'Creating struct {struct_name}.') - exd_struct_map[key] = struct_name - for enum_key in exd_header_enums: - api.create_enum_struct(f"{struct_name}::{enum_key}", exd_header_enums[enum_key], 1) - api.create_struct(struct_name, exd_header) - -# if a new pattern is found, add it to the exd_func_patterns dict -for func_pattern in exd_func_patterns: - api.process_pattern(func_pattern) - -api.comment_rows(exd_comment_patterns, exd_map) diff --git a/ida/ffxiv_extract_uld_names.py b/ida/ffxiv_extract_uld_names.py deleted file mode 100644 index d706ba4e1a..0000000000 --- a/ida/ffxiv_extract_uld_names.py +++ /dev/null @@ -1,76 +0,0 @@ -import idaapi -import ida_bytes -import ida_nalt -import ida_struct -import ida_enum -import ida_kernwin -import ida_search -import ida_ida -import ida_typeinf -import ida_hexrays -import ida_name -import os - -patterns = [ - "48 8D 15 ?? ?? ?? ?? 41 B9 ?? ?? ?? ?? 45 33 C0 E8 ?? ?? ?? ??", - "48 8D 15 ?? ?? ?? ?? 45 33 C0 48 8B CB 48 83 C4 20 5B E9 ?? ?? ?? ??", - "48 8D 15 ?? ?? ?? ?? 45 33 C0 E9 ?? ?? ?? ??", -] - -strings = [] - - -def search_binary(ea, pattern, flag): - return ida_search.find_binary( - ea, - flag & 1 and ida_ida.cvar.inf.max_ea or ida_ida.cvar.inf.min_ea, - pattern, - 16, - flag, - ) - - -def get_string(offset): - return ida_bytes.get_strlit_contents(offset, -1, ida_nalt.STRTYPE_TERMCHR).decode( - "UTF-8" - ) - - -def do_pattern(pattern): - ea = 0 - while True: - ea = search_binary( - ea + 1, - pattern, - ida_search.SEARCH_DOWN, - ) - - if ea == 0xFFFFFFFFFFFFFFFF: - break - - offset = ida_bytes.get_original_dword(ea + 3) - - string = get_string(offset + ea + 0x7) - - strings.append(string) - - -for pattern in patterns: - do_pattern(pattern) - -txtFile = os.path.join(os.path.dirname(os.path.realpath(__file__)), "uld_names.txt") - -if os.path.exists(txtFile): - os.remove(txtFile) - -strings.sort() - -filestr = "" - -for string in strings: - filestr = f'{filestr},\n"ui/uld/{string}.uld"' - -with open(txtFile, "w") as f: - f.write(filestr) - -print(strings) diff --git a/ida/ffxiv_fullrun_ida.py b/ida/ffxiv_fullrun_ida.py deleted file mode 100644 index ece68d0026..0000000000 --- a/ida/ffxiv_fullrun_ida.py +++ /dev/null @@ -1,32 +0,0 @@ -from os import path -import idautils -import ida_auto -import idc -import ida_kernwin -import idaapi -import sys - -print("Loading ffxiv_idarename.py") -exec(open(path.join(path.dirname(path.realpath(__file__)), "ffxiv_idarename.py")).read()) - -print("Loading ffxiv_exdgetters.py") -exec(open(path.join(path.dirname(path.realpath(__file__)), "ffxiv_exdgetters.py")).read()) - -ida_base = ida_kernwin.ask_buttons( - "ffxiv_dx11.exe", - "__ImageBase", - "", - ida_kernwin.ASKBTN_YES, - "HIDECANCEL\nWhich name would you prefer from base offset?\n\nffxiv_dx11.exe : CheatEngine and a few other utilities will use the offset of ffxiv_dx11.exe+ as the base.\n__ImageBase : This is the default IDA behavior.", -) == ida_kernwin.ASKBTN_YES - -print("Loading ffxiv_structimporter.py") -exec(open(path.join(path.dirname(path.realpath(__file__)), "ffxiv_structimporter.py")).read()) - -if (ida_base): - idc.set_name(idaapi.get_imagebase(), "ffxiv_dx11.exe") - -for s in idautils.Segments(): - start = idc.get_segm_start(s) - end = idc.get_segm_end(s) - ida_auto.plan_range(start, end) \ No newline at end of file diff --git a/ida/ffxiv_idarename.py b/ida/ffxiv_idarename.py deleted file mode 100644 index 076ca9d0b4..0000000000 --- a/ida/ffxiv_idarename.py +++ /dev/null @@ -1,977 +0,0 @@ -# current exe version: 2020.12.29.0000.0000 -# @category __UserScripts -# @menupath Tools.Scripts.ffxiv_idarename -# @runtime PyGhidra - -from __future__ import print_function -import os -import yaml -from anytree import Node, RenderTree, PreOrderIter - -try: - from typing import Any, Dict, List, Optional, Union # noqa -except ImportError: - pass - -import sys -import itertools -from abc import abstractmethod -from collections import deque - -if sys.version_info[0] >= 3: - long = int - - -# region Api - -class BaseApi(object): - @property - @abstractmethod - def data_file_path(self): - """ - Get the Path to the data.yml File - :return: Path to the data.yml File - :rtype: str - """ - - @abstractmethod - def get_image_base(self): - """ - Get the image base ea - :return: Image base ea - :rtype: int - """ - - @abstractmethod - def is_offset(self, ea): - """ - Is the given address an offset to something else - :param ea: Effective address - :type ea: int - :return: Is offset or not - :rtype: bool - """ - - @abstractmethod - def xrefs_to(self, ea): - """ - Retrieve all xrefs to the given address - :param ea: Effective address - :type ea: int - :return: List of addresses - :rtype: List[int] - """ - - @abstractmethod - def get_qword(self, ea): - """ - Read a qword of data from an address - :param ea: Effective address - :type ea: int - :return: 64bits of data - :rtype: int - """ - - @abstractmethod - def get_addr_name(self, ea): - """ - Get the name of a given address - :param ea: Effective address - :type ea: int - :return: Name or empty - :rtype: str - """ - - @abstractmethod - def set_addr_name(self, ea, name): - """ - Set the name of a given address (function) - :param ea: Effective address - :type ea: int - :param name: Name to set - :type name: str - :return: Success/failure - :rtype: bool - """ - - @abstractmethod - def get_comment(self, ea): - """ - Get the comment at an address - :param ea: Effective address - :type ea: int - :return: Comment - :rtype: str - """ - - @abstractmethod - def set_comment(self, ea, comment): - """ - Add a comment to an address - :param ea: Effective address - :type ea: int - :param comment: Comment - :type comment: str - :return: None - """ - - def format_class_name_for_vtbl(self, class_name): - """ - Format a class name for special representation in the vtbl - By default, looks similar to vtbl_Component::Gui::AtkResNode - :param class_name: Name as contained in data.yml - :type class_name: str - :return: Formatted class name - :rtype: str - """ - return "vtbl_{0}".format(class_name) - - def format_class_name_for_secondary_vtbl(self, class_name, secondary_name): - """ - Format a class name for special representation in the vtbl - By default, looks similar to vtbl_classname_secondaryname - :param class_name: Name as contained in data.yml - :type class_name: str - :param secondary_name: Name as contained in data.yml - :type secondary_name: str - :return: Formatted class name - :rtype: str - """ - return "vtbl_{0}___{1}".format(class_name, secondary_name) - - def format_class_name(self, class_name): - """ - Format a class name for representation in the toolset - By default, looks similar to Component::Gui::AtkResNode - :param class_name: Name as contained in data.yml - :type class_name: str - :return: Formatted class name - :rtype: str - """ - return class_name - - @abstractmethod - def format_vfunc_name(self, ea, current_func_name, proposed_func_name, class_name, base_class_names): - """ - Name a function (vfunc) based on how it is currently named, what it is proposed to be named, - and the names of the current and base classes. If None is returned, it is assumed - that the current func was named something unexpectedly and will be warned about. Return - an empty string "" if no renaming should take place. - :param ea: Func effective address - :type ea: int - :param current_func_name: Current func name as in the disassembler - :type current_func_name: str - :param proposed_func_name: Either a name from data.yml or vfX where X is the vfunc index - :type proposed_func_name: str - :param class_name: Class name - :type class_name: str - :param base_class_names: base class names - :type base_class_names: List[str] - :return: Formatted vfunc name - :rtype: Optional[str] - """ - - @abstractmethod - def format_func_name(self, ea, current_func_name, proposed_func_name, class_name): - """ - Name a function based on how it is currently named, what it is proposed to be named, - and the name of the current class. If None is returned, it is assumed that the current - func was named something unexpectedly and will be warned about. Return an empty string "" - if no renaming should take place. - :param ea: Func effective address - :type ea: int - :param current_func_name: Current func name as in the disassembler - :type current_func_name: str - :param proposed_func_name: A name from data.yml - :type proposed_func_name: str - :param class_name: Class name - :type class_name: str - :return: Formatted func name - :rtype: Optional[str] - """ - - -api = None - -# region IDA Api - -if api is None: - try: - import idaapi # noqa - import idc # noqa - import idautils # noqa - import ida_funcs # noqa - import ida_bytes # noqa - except ImportError: - print("Warning: Unable to load IDA") - else: - # noinspection PyUnresolvedReferences - class IdaApi(BaseApi): - - @property - def data_file_path(self): - return os.path.join(os.path.dirname(os.path.realpath(__file__)), "data.yml") - - def get_image_base(self): - return idaapi.get_imagebase() - - def is_offset(self, ea): - return idc.is_off0(idc.get_full_flags(ea)) - - def xrefs_to(self, ea): - return [xref.to for xref in idautils.XrefsTo(ea)] - - def get_qword(self, ea): - return idc.get_qword(ea) - - def get_addr_name(self, ea): - name = idc.get_name(ea) - # if name is None or '' and segment name is .text then create the function and get the name of it. - # this is done for parts of the code that has no xrefs to it due to how IDA detects function boundaries - if (name is None or name == '') and idc.get_segm_name(ea) == '.text': - finf = ida_funcs.func_t() - finf.start_ea = ea - finf.end_ea = idc.BADADDR - ida_funcs.add_func_ex(finf) - name = idc.get_name(ea) - return name - - def set_addr_name(self, ea, name): - # print("{0} {1}".format(ea, name)) - if ida_bytes.get_item_head(ea) != ea: - return True - result = idc.set_name(ea, name) - return bool(result) - - def get_comment(self, ea): - idc.get_cmt(ea, False) - - def set_comment(self, ea, comment): - idc.set_cmt(ea, comment, False) - - def format_vfunc_name(self, ea, current_func_name, proposed_func_name, class_name, base_class_names): - if current_func_name.startswith("j_"): # jump - current_func_name = current_func_name.lstrip("j_") - - if current_func_name.startswith("qword_"): - idc.auto_mark_range(ea, ea + 1, idc.AU_CODE) - idc.create_insn(ea) - idc.add_func(ea) - current_func_name = api.get_addr_name(ea) - print("Info: qword in vtbl of {1} at 0x{0:X}, it may be an offset to undefined code".format(ea, - class_name)) - - # Previously renamed as a vfunc - if current_func_name.startswith(class_name): - # return the proposed func name in case it was updated since last run - current_class_name = current_func_name.rsplit(".", 1)[0] - return "{0}.{1}".format(current_class_name, proposed_func_name) - - # This should have been handled in the base class - if any(current_func_name.startswith(name) for name in base_class_names): - return "" - - if current_func_name.startswith("sub_"): - return "{0}.{1}".format(class_name, proposed_func_name) - - if current_func_name.startswith("unknown_libname_"): - return "{0}.{1}".format(class_name, proposed_func_name) - - if current_func_name.startswith("nullsub_"): - return "{0}.{1}_nullsub".format(class_name, proposed_func_name) - - if current_func_name.startswith("loc_"): - return "{0}.{1}_loc".format(class_name, proposed_func_name) - - if current_func_name.startswith("locret_"): - return "{0}.{1}_locret".format(class_name, proposed_func_name) - - # Name it later in a child class when it gets overridden - if current_func_name == "_purecall": - return "" - - # Mangled func names, thanks IDA - if current_func_name.startswith("?") or current_func_name.startswith("_"): - return "{0}.{1}".format(class_name, proposed_func_name) - - return None - - def format_func_name(self, ea, current_func_name, proposed_func_name, class_name): - if current_func_name.startswith("j_"): # jump - current_func_name = current_func_name.lstrip("j_") - - # Mangled func names, thanks IDA - if proposed_func_name.startswith("?") or proposed_func_name.startswith("_"): - return proposed_func_name - - proposed_qualified_func_name = "{0}.{1}".format(class_name, proposed_func_name) - if current_func_name == proposed_qualified_func_name: - return "" - - if any(current_func_name.startswith(prefix) for prefix in ("sub_", "nullsub_", "loc_", "qword_", "unknown_libname_", "?", "_")): - return proposed_qualified_func_name - - return None - - - api = IdaApi() - -# endregion -# region Ghidra Api - -if api is None: - try: - import ghidra - - from ghidra.program.model.data import CategoryPath # noqa - from ghidra.program.model.data import StructureDataType # noqa - from ghidra.program.model.data import PointerDataType # noqa - from ghidra.program.model.symbol import SourceType # noqa - except ImportError: - print("Warning: Unable to load Ghidra") - else: - def toAddr(offset): # noqa - # type: (int) -> Address - return currentProgram.getAddressFactory().getDefaultAddressSpace().getAddress(offset) # noqa - - # noinspection PyUnresolvedReferences - class GhidraApi(BaseApi): - @property - def data_file_path(self): - return os.path.join(os.path.dirname(str(sourceFile)), "data.yml") - - def get_image_base(self): - return currentProgram.getImageBase().getOffset() - - def is_offset(self, ea): - data = getDataAt(toAddr(ea)) - if not data: return False - return data.isPointer() - - def xrefs_to(self, ea): - return [xref.getFromAddress().getOffset() for xref in getReferencesTo(toAddr(ea))] - - def get_qword(self, ea): - return getLong(toAddr(ea)) - - def get_addr_name(self, ea): - sym = getSymbolAt(toAddr(ea)) - if not sym and getByte(toAddr(ea).subtract(1)) & 0xFF == 0xCC: - disassemble(toAddr(ea)) - createFunction(toAddr(ea), None) - sym = getSymbolAt(toAddr(ea)) - if not sym: - return "" - return sym.getName(True) - - def set_addr_name(self, ea, name): - createLabel(toAddr(ea), name, True, SourceType.ANALYSIS) - return True - # return createLabel(toAddr(ea), name, True).checkIsValid() - - def get_comment(self, ea): - return getEOLComment(toAddr(ea)) - - def set_comment(self, ea, comment): - if getEOLComment(toAddr(ea)) is None: - setEOLComment(toAddr(ea), comment) - - def format_vfunc_name(self, ea, current_func_name, proposed_func_name, class_name, base_class_names): - if current_func_name.startswith("thunk_"): # jump - current_func_name = current_func_name.lstrip("thunk_") - - # Previously renamed as a vfunc - if current_func_name.startswith(class_name): - # return the proposed func name in case it was updated since last run - current_class_name = current_func_name.rsplit(".", 1)[0] - return "{0}.{1}".format(current_class_name, proposed_func_name) - - # This should have been handled in the base class - if any(current_func_name.startswith(name) for name in base_class_names): - return "" - - if any(current_func_name.startswith(prefix) for prefix in ("FUN_", "LAB_", "SUB_", "LOC_", "DAT_")): - return "{0}.{1}".format(class_name, proposed_func_name) - - if current_func_name == "_purecall": - return "" - - return None - - def format_func_name(self, ea, current_func_name, proposed_func_name, class_name): - # override default thunk names and thunks of already named funcs with the default - func = getFunctionAt(toAddr(ea)) - if func and func.isThunk(): - current_func_name = "FUN_{0:x}".format(ea) - - proposed_qualified_func_name = "{0}.{1}".format(class_name, proposed_func_name) - if current_func_name == proposed_qualified_func_name: - return "" - - if any(current_func_name.startswith(prefix) for prefix in ("FUN_", "LAB_", "SUB_", "LOC_", "DAT_")): - return proposed_qualified_func_name - - return None - - - api = GhidraApi() - -# endregion -# region Binja Api - -if api is None: - try: - import binaryninja # noqa - except ImportError: - print("Warning: Unable to load Binary Ninja") - else: - # noinspection PyUnresolvedReferences - class BinjaApi(BaseApi): - @property - def data_file_path(self): - return os.path.join(os.path.dirname(__file__), "data.yml") - - def get_image_base(self): - return bv.start - - def is_offset(self, ea): - return bv.get_data_var_at(ea) is not None - - def xrefs_to(self, ea): - return [xref.address for xref in bv.get_code_refs(ea)] - - def get_qword(self, ea): - bytes = bv.read(ea, 8) - return int.from_bytes(bytes, byteorder='little') - - def get_addr_name(self, ea): - func = bv.get_function_at(ea) - if func: - return func.name - - sym = bv.get_symbol_at(ea) - if sym: - return sym.name - - return "" - - def set_addr_name(self, ea, name): - func = bv.get_function_at(ea) - if func: - func.name = name - return True - - sym = bv.get_symbol_at(ea) - if sym: - # No good way to clone a symbol (afaik), so remake it - new_sym = binaryninja.types.Symbol( - sym.type, - ea, - name, - sym.full_name, - sym.raw_name, - sym.binding, - sym.namespace, - sym.ordinal - ) - - if sym.auto: - bv.undefine_auto_symbol(sym) - else: - bv.undefine_user_symbol(sym) - bv.define_user_symbol(new_sym) - return True - - data_var = bv.get_data_var_at(ea) - if data_var: - new_sym = binaryninja.types.Symbol( - binaryninja.types.SymbolType.DataSymbol, - ea, - name - ) - bv.define_user_symbol(new_sym) - return True - - return False - - def get_comment(self, ea): - return bv.get_comment_at(ea) - - def set_comment(self, ea, comment): - bv.set_comment_at(ea, comment) - - def format_vfunc_name(self, ea, current_func_name, proposed_func_name, class_name, base_class_names): - # Previously renamed as a vfunc - if current_func_name.startswith(class_name): - # return the proposed func name in case it was updated since last run - current_class_name = current_func_name.rsplit(".", 1)[0] - return "{0}.{1}".format(current_class_name, proposed_func_name) - - # This should have been handled in the base class - if any(current_func_name.startswith(name) for name in base_class_names): - return "" - - return "{0}.{1}".format(class_name, proposed_func_name) - - def format_func_name(self, ea, current_func_name, proposed_func_name, class_name): - proposed_qualified_func_name = "{0}.{1}".format(class_name, proposed_func_name) - if current_func_name == proposed_qualified_func_name: - return "" - return proposed_qualified_func_name - - - api = BinjaApi() - -# endregion - -if api is None: - raise Exception("Unable to load API (supported: IDA, Ghidra, Binary Ninja)") - - -# endregion - -def load_data(): - with open(api.data_file_path, "r") as fd: - data = yaml.safe_load(fd) - - if data.get("globals"): - for ea, name in data["globals"].items(): - if not isinstance(ea, (int, long)): - print('Warning: {0} has an invalid address {1}'.format(name, ea)) - continue - api.set_addr_name(ea, name) - - if data.get("functions"): - for ea, name in data["functions"].items(): - if not isinstance(ea, (int, long)): - print('Warning: {0} has an invalid address {1}'.format(name, ea)) - continue - api.set_addr_name(ea, name) - - - if data.get("classes"): - factory = FfxivClassFactory() - for class_name, class_data in data["classes"].items(): - if not class_data: - class_data = {} - - vtbls_raw = class_data.pop("vtbls", []) - vtbls = [(vtbl["ea"], vtbl["base"] if "base" in vtbl else None, vtbl["size"] if "size" in vtbl else None) for vtbl in vtbls_raw] - vfuncs = class_data.pop("vfuncs", {}) - funcs = class_data.pop("funcs", {}) - instances_raw = class_data.pop("instances", []) - instances = [(instance["ea"], instance["name"] if "name" in instance else "Instance") for instance in instances_raw] if instances_raw is not None else [] - for leftover in class_data: - print("Warning: Extra key \"{0}\" present in {1}".format(leftover, class_name)) - - factory.register( - class_name=class_name, vtbls=vtbls, vfuncs=vfuncs, funcs=funcs, instances=instances) - - factory.finalize() - - -class FfxivClassFactory: - _vtbl_addresses = [] # type: List[int] - _classes = {} # type: Dict[str, FfxivClass] - - def register(self, class_name, vtbls=None, vfuncs=None, funcs=None, instances=None): - """ - Register a class - :param class_name: Class name - :type class_name: str - :param vtbls: List of (vtbl_ea, base_name) pairs - :type vtbls: list[(int, str)] - :param funcs: Mapping of effective addresses to func names - :type funcs: Dict[int, str] - :param vfuncs: Mapping of vtbl index to func names - :type funcs: Dict[int, str] - :param instances: List of global instances of the object - :type instances: List[(int, str)] - :return: None - :rtype: None - """ - if vtbls is None: - vtbls = [] - - if not vfuncs: - vfuncs = {} - - if not funcs: - funcs = {} - - for (vtbl_ea, _, _) in vtbls: - if vtbl_ea != 0x0 and vtbl_ea in self._vtbl_addresses: - print("Error: Multiple vtables are defined at 0x{0:X}".format(vtbl_ea)) - return - - if class_name in self._classes: - print("Error: Multiple classes are registered with the name \"{0}\"".format(class_name)) - return - for (vtbl_ea, _, _) in vtbls: - self._vtbl_addresses.append(vtbl_ea) - - if not instances: - instances = [] - - self._classes[class_name] = FfxivClass( - class_name=class_name, vtbls=vtbls, vfuncs=vfuncs, funcs=funcs, instances=instances) - - def finalize(self): - """ - Perform the class naming - :return: None - :rtype: None - """ - self._resolve_base_classes() - - # We write the vtbl names first for an added check so that - # the size finder will not advance past a named offset. - for cls in self._classes.values(): - if cls.vtbls: - cls.write_vtbl_names() - - for cls in self._classes.values(): - self._finalize_class(cls) - - def _resolve_base_classes(self): - """ - Resolve base classes for a class - If missing, warn the user and add a stub entry. - :return: None - """ - for class_name, cls in list(self._classes.items()): - if cls.vtbls: - for idx, vtbl in enumerate(cls.vtbls): - if vtbl.resolved_base is None and vtbl.base_name: - if vtbl.base_name not in self._classes: - print("Warning: Inherited class \"{0}\" is not documented, add a placeholder entry".format( - vtbl.base_name)) - self.register(class_name=vtbl.base_name) - vtbl.resolved_base = self._classes[vtbl.base_name] - if idx == 0: - cls.first_base = self._classes[vtbl.base_name] - - _finalize_stack = deque() - - def _finalize_class(self, cls): - """ - Perform a single class naming - :param cls: Class object - :type cls: FfxivClass - :return: None - :rtype: None - """ - if cls in self._finalize_stack: - names = [c.name for c in self._finalize_stack] + [cls.name] - names = "\n".join([" - {0}".format(name) for name in names]) - raise ValueError("Inheritance cycle detected: \n{0}".format(names)) - - if not cls.finalized: - self._finalize_stack.append(cls) - if cls.vtbls: - for vtbl in cls.vtbls: - if vtbl.resolved_base and not vtbl.resolved_base.finalized: - self._finalize_class(vtbl.resolved_base) - cls.finalize() - - self._finalize_stack.pop() - - -class Vtbl: - resolved_base = None # type: FfxivClass - - def __init__(self, ea, base_name=None, size=None): - """ - Object representing a class's vtbl - :param ea: Address of vtbl - :type ea: int - :param base_name: Name of the base class this vtbl inherits, if it exists - :type base_name: str - :param size: Explicit size of vtbl used to override cases where size detection fails - :type size: int - """ - self.ea = ea - self.base_name = base_name - self.size = size - - -class FfxivClass: - STANDARD_IMAGE_BASE = 0x140000000 - - vtbls = None # type: list[Vtbl] - - def __init__(self, class_name, vtbls, vfuncs, funcs, instances): - """ - Object representing a class - :param class_name: Class name - :type class_name: str - :param vtbls: List of (vtbl_ea, base_name) pairs - :type vtbls: list[(int, str)] - :param vfuncs: Mapping of vtbl index to func names - :type vfuncs: Dict[int, str] - :param funcs: Mapping of effective addresses to func names - :type funcs: Dict[int, str] - :param instances: List of global instances of the object - :type instances: List[(int, str)] - """ - self.name = class_name - if vtbls: - self.vtbls = [Vtbl(ea, base_name, size) for (ea, base_name, size) in vtbls] - self.vfuncs = vfuncs - self.funcs = funcs - self.instances = instances - - # Offset the vtbl and funcs if the program has been rebased - current_image_base = api.get_image_base() - if self.STANDARD_IMAGE_BASE != current_image_base: - rebase_offset = current_image_base - self.STANDARD_IMAGE_BASE - if self.vtbls: - for vtbl in self.vtbls: - vtbl.ea += rebase_offset - for ea in list(funcs.keys()): - funcs[ea + rebase_offset] = funcs.pop(ea) - - # region vtbl_size - - _main_vtbl_size = 0 - - @property - def main_vtbl_size(self): - """ - Iterate from the vtbl start until a non-offset or xref is encountered. - This strategy implies that the only xref in a vtbl is the first vfunc. - Can be overriden by explicit size in the vtbl definition. - :return: VTable func count - :rtype: int - """ - if not self.vtbls: - return self._main_vtbl_size - - if self.vtbls[0].size: - self._main_vtbl_size = self.vtbls[0].size - return self._main_vtbl_size - - if self._main_vtbl_size == 0: - self._main_vtbl_size = 1 # Set to 1, skip the first entry - for ea in itertools.count(self.vtbls[0].ea + 8, 8): - if api.get_addr_name(ea) != '': - break - - if api.is_offset(ea) and api.xrefs_to(ea) == []: - self._main_vtbl_size += 1 - else: - break - - if self.vtbls[0].resolved_base and self._main_vtbl_size < self.vtbls[0].resolved_base._main_vtbl_size: - print( - "Error: The sum of \"{0}\"'s base vtbl sizes ({1}) is greater than the actual class itself ({2})" - .format(self.name, self.vtbls[0].resolved_base._main_vtbl_size, self._main_vtbl_size)) - - return self._main_vtbl_size - - # endregion - - _inheritance_tree = None - - @property - def inheritance_tree(self): - def recurse_tree(current_node, current_class, all_bases): - if current_class.vtbls: - for vtbl in current_class.vtbls: - if vtbl.resolved_base and vtbl.resolved_base.name not in all_bases: - new_node = Node(vtbl.resolved_base.name, parent=current_node) - all_bases.add(vtbl.resolved_base.name) - recurse_tree(new_node, vtbl.resolved_base, all_bases) - - if not self._inheritance_tree: - self._inheritance_tree = Node(self.name) - base_set = set() - recurse_tree(self._inheritance_tree, self, base_set) - - return self._inheritance_tree - - # region finalized - - _finalized = False - - @property - def finalized(self): - """ - Has this class and its hierarchy been written out or not - :return: bool yes/no - :rtype: bool - """ - if self.vtbls: - return self._finalized and all( - vtbl.resolved_base is not None and vtbl.resolved_base._finalized for vtbl in self.vtbls) - else: - return self._finalized - - @finalized.setter - def finalized(self, value): - """ - Set if finalized or not - :param value: Finalized state - :type value: bool - :return: None - :rtype: None - """ - self._finalized = value - - # endregion - - # region finalize - - def finalize(self): - """ - Write out this class - :return: None - :rtype: None - """ - self._inherit_func_names_from_main_base() - self._comment_vtbls_with_inheritance_tree() - - self._write_vtbl_functions() - self._write_funcs() - - self._write_instances() - - self.finalized = True - - def _inherit_func_names_from_main_base(self): - """ - A base is guaranteed to be finalized before the child, - so a base has all the vfunc names of its base already. - :return: None - :rtype: None - """ - if self.vtbls and self.vtbls[0].resolved_base: - for idx, base_vfunc_name in self.vtbls[0].resolved_base.vfuncs.items(): - if idx in self.vfuncs: - print("Warning: 0x{0:X} \"{1}\" overwrites the name of inherited function \"{2}\"".format( - self.vtbls[0].ea, self.name, base_vfunc_name)) - pass - else: - self.vfuncs[idx] = base_vfunc_name - - def _comment_vtbls_with_inheritance_tree(self): - """ - Adds the inheritance tree as a comment to the start of the vtbl. - grandbase_name - base_name - self_name - :return: None - :rtype: None - """ - if self.vtbls: - for idx, vtbl in enumerate(self.vtbls): - comment = api.get_comment(vtbl.ea) or "" - - for pre, fill, node in RenderTree(self.inheritance_tree): - if comment: - comment += "\n" - comment += pre + api.format_class_name(node.name) - api.set_comment(vtbl.ea, comment) - - def write_vtbl_names(self): - """ - Write out the vtbl name. - :return: None - """ - api.set_addr_name(self.vtbls[0].ea, api.format_class_name_for_vtbl(self.name)) - for vtbl in self.vtbls[1:]: - api.set_addr_name(vtbl.ea, api.format_class_name_for_secondary_vtbl(self.name, - vtbl.base_name)) - - def _write_vtbl_functions(self): - """ - Write out the vtbl function names - :return: None - :rtype: None - """ - - def collect_vtbl_functions(ea, size, class_name, vfunc_names, base_class_names): - """ - :type ea: int - :type size: int - :type class_name: str - :type vfunc_names: dict[int, str] - :type base_class_names: list[str] - """ - vtbl_builder = [] - # Iterate through each offset - for func_idx in range(0, size): - vtbl_vfunc_ea = ea + func_idx * 8 - vfunc_ea = api.get_qword(vtbl_vfunc_ea) # type: int - - current_func_name = api.get_addr_name(vfunc_ea) # type: str - proposed_func_name = vfunc_names.get(func_idx, "vf{0}".format(func_idx)) - formatted_class_name = api.format_class_name(class_name) - - formatted_func_name = api.format_vfunc_name(vfunc_ea, current_func_name, proposed_func_name, - formatted_class_name, - base_class_names) - - if formatted_func_name == "": - pass - elif formatted_func_name is None: - print( - "Error: Function at 0x{0:X} had unexpected name \"{1}\" during naming of {2}.{3} (vtbl[{4}])" - .format(vfunc_ea, current_func_name, self.name, proposed_func_name, func_idx)) - else: - vtbl_builder.append((vfunc_ea, formatted_func_name)) - - return vtbl_builder - - if self.vtbls: - formatted_base_class_names = [api.format_class_name(node.name) for node in - PreOrderIter(self.inheritance_tree)] - formatted_base_class_names.remove(self.name) - for idx, vtbl in enumerate(self.vtbls): - if idx == 0: - funcs = collect_vtbl_functions(vtbl.ea, self.main_vtbl_size, self.name, self.vfuncs, - formatted_base_class_names) - else: - funcs = collect_vtbl_functions(vtbl.ea, vtbl.resolved_base.main_vtbl_size, - self.name + "___" + vtbl.resolved_base.name, - vtbl.resolved_base.vfuncs, formatted_base_class_names) - for (func_ea, func_name) in funcs: - api.set_addr_name(func_ea, func_name) - - def _write_funcs(self): - """ - Write the names of all non-vtbl funcs - :return: None - """ - for func_ea, proposed_func_name in self.funcs.items(): - current_func_name = api.get_addr_name(func_ea) # type: str - - func_name = api.format_func_name(func_ea, current_func_name, proposed_func_name, self.name) - if func_name == "": - pass - elif func_name is None: - print("Error: Function at 0x{0:X} had unexpected name \"{1}\" during naming of {2}.{3}" - .format(func_ea, current_func_name, self.name, proposed_func_name)) - else: - api.set_addr_name(func_ea, func_name) - - def _write_instances(self): - """ - Write the names of all instances - :return: None - """ - for (instance_ea, instance_name) in self.instances: - name = "g_{}_{}".format(self.name, instance_name) - api.set_addr_name(instance_ea, name) - - # endregion - - def __repr__(self): - return "<{0}(\"{1}\")>".format(self.__class__.__name__, self.name) - - -# endregion - -print("Executing") -load_data() -print("Done") diff --git a/ida/ffxiv_luafuncs_ida.py b/ida/ffxiv_luafuncs_ida.py deleted file mode 100644 index eb7b3eac90..0000000000 --- a/ida/ffxiv_luafuncs_ida.py +++ /dev/null @@ -1,109 +0,0 @@ -import idc -import ida_name -import ida_funcs - -namespace = "Client::Game::Event::Lua" - -def get_string_from_ptr(ea): - val = idc.get_strlit_contents(ea) - if val == None: - val = idc.get_strlit_contents(idc.get_qword(ea)) - return val.decode('utf-8') - -def get_asm_string(ea): - ins = idc.print_insn_mnem(ea) - - op = idc.print_operand(ea, 0) - if ins == 'push' or ins == 'call' or ins == 'jz': - val = idc.get_operand_value(ea, 0) - return {'ins': ins, 'op': op, 'val': val} - - val_text = idc.print_operand(ea, 1) - val = idc.get_operand_value(ea, 1) - if val_text.startswith('cs:'): - val_text = get_string_from_ptr(val) - return {'ins': ins, 'reg': op, 'val_text': val_text, 'val': val} - -def get_registers(ea): - regs = {} - loop = True - while loop: - asm = get_asm_string(ea) - if asm['ins'] == 'call' and asm['op'] == 'Common::Lua::LuaState.SetStringField': - loop = False - break - if asm['ins'] == 'mov': - regs[asm['reg']] = {'val_text': asm['val_text'], 'val': asm['val']} - ea = idc.next_head(ea) - return regs - -def get_func_registers(ea, reg_count): - regs = {} - while reg_count > 0: - ea = idc.prev_head(ea) - asm = get_asm_string(ea) - regs[asm['reg']] = {'val_text': asm['val_text'], 'val': asm['val']} - reg_count = reg_count - 1 - return regs - -def get_func_xrefs(ea): - eas = [] - xref = idc.get_first_cref_to(ea) - while xref != idc.BADADDR: - if idc.print_insn_mnem(xref).lower() == "call": - eas.append(xref) - xref = idc.get_next_cref_to(ea, xref) - return eas - -def set_func_name(ea, name, count: None | int = None): - act_name = name - if count != None: - act_name = f"{name}{count}" - - cur_name = idc.get_func_name(ea) - if cur_name.startswith('sub_'): - print(act_name) - found_ea = ida_name.get_name_ea(0, act_name) - if found_ea == idc.BADADDR or found_ea == ea: - idc.set_name(ea, act_name) - else: - set_func_name(ea, name, 2 if count is None else count + 1) - -def run(): - string_eas = get_func_xrefs(ida_name.get_name_ea(0, 'Common::Lua::LuaState.SetStringField')) - func_eas = get_func_xrefs(ida_name.get_name_ea(0, 'Common::Lua::LuaState.SetFunctionField')) - - string_eas = [ida_funcs.get_func(x).start_ea for x in string_eas] - - eas = {} - for x in func_eas: - regs = get_func_registers(x, 4) - regs['r8'] = get_string_from_ptr(regs['r8']['val']) - eas.setdefault(ida_funcs.get_func(x).start_ea, {'calls': []})['calls'].append({'off': x, 'regs': regs}) - - eas = {k: v for k,v in eas.items() if k in string_eas} - - for func_ea in eas: - eas[func_ea]['regs'] = get_registers(func_ea) - - for func_ea in eas: - eas[func_ea]['name'] = eas[func_ea]['regs'][eas[func_ea]['regs']['r9']['val_text']]['val_text'] - eas[func_ea]['name_val'] = eas[func_ea]['regs'][eas[func_ea]['regs']['r9']['val_text']]['val'] - - for func_ea in eas: - if eas[func_ea]['name'] == 'r8': - eas[func_ea]['name'] = idc.get_func_name(func_ea).split('.')[0].split(':')[-1] - eas[func_ea]['name_val'] = func_ea - if eas[func_ea]['name'] == 'rax': - eas[func_ea]['name_val'] = get_func_xrefs(func_ea)[0] - eas[func_ea]['name'] = idc.get_func_name(eas[func_ea]['name_val']).split('.')[0].split(':')[-1] - - for func_ea in eas: - name = f"{namespace}::{eas[func_ea]['name']}" - idc.set_name(func_ea, f"{name}.SetupLuaClass") - for call in eas[func_ea]['calls']: - regs = call['regs'] - func_name = regs['r8'] - set_func_name(regs['r9']['val'], f"{name}.{func_name}") - -run() \ No newline at end of file diff --git a/ida/ffxiv_structimporter.py b/ida/ffxiv_structimporter.py deleted file mode 100644 index 10d4943eef..0000000000 --- a/ida/ffxiv_structimporter.py +++ /dev/null @@ -1,2400 +0,0 @@ -# @category __UserScripts -# @menupath Tools.Scripts.ffxiv_structimport -# @runtime PyGhidra - -from yaml import load - -try: - from yaml import CSafeLoader as Loader -except ImportError: - from yaml import SafeLoader as Loader - -import os -from abc import abstractmethod -from time import time -from structs_schema import * - - -class BaseApi: - @abstractmethod - def can_run(self): - # type: () -> None - """ - Checks if exdgetters has run before this is allowed to continue - """ - - @abstractmethod - def create_enum_struct(self, enum): - # type: (DefinedStructEnum) -> None - """ - Create an enum in the database. - """ - - @abstractmethod - def delete_enum(self, enum): - # type: (DefinedStructEnum) -> None - """ - Delete an enum in the database. - """ - - @abstractmethod - def delete_struct(self, struct): - # type: (DefinedStruct) -> None - """ - Delete a struct in the database. - """ - - @abstractmethod - def create_struct(self, struct): - # type: (DefinedStruct) -> None - """ - Create a struct in the database. - """ - - @abstractmethod - def create_struct_members(self, struct): - # type: (DefinedStruct) -> None - """ - Create members for a struct in the database. - """ - - @abstractmethod - def create_vtable(self, struct): - # type: (DefinedStruct) -> None - """ - Create a vtable in the database. - """ - - @abstractmethod - def finalise_struct(self, struct): - # type: (DefinedStruct) -> None - """ - Finalise a struct in the database. - """ - - @abstractmethod - def create_union(self, struct): - # type: (DefinedStruct) -> None - """ - Create a union in the database. - """ - - @abstractmethod - def update_member_func(self, member_func, struct): - # type: (DefinedStructMemFunc, DefinedStruct) -> None - """ - Updates a member function in the database. - """ - - @abstractmethod - def update_virt_func(self, virt_func, struct): - # type: (DefinedStructVFunc, DefinedStruct) -> None - """ - Updates a virtual function in the database. - """ - - @abstractmethod - def update_static_member(self, static_member, struct): - # type: (DefinedStructStaticMember, DefinedStruct) -> None - """ - Updates a static member in the database. - """ - - @abstractmethod - def should_update_member_func(self): - # type: () -> bool - """ - Returns if the member function types should be updated. - """ - - @abstractmethod - def should_update_virt_func(self): - # type: () -> bool - """ - Returns if the virtual function types should be updated. - """ - - @property - @abstractmethod - def get_file_path(self): - """ - Retrieve the file path of the yaml file. - """ - - def get_yaml(self): - # type: () -> DefinedStructExport - with open(self.get_file_path, "r") as fd: - dic = load(fd, Loader=Loader) # type: dict[str, dict[str, list[dict[str, str | int | list[dict[str, str | int]]]]]] - enums = [] - structs = [] - for enum in dic["enums"]: - enums.append( - DefinedStructEnum( - enum["name"], - enum["type"], - enum["underlying"], - enum["namespace"], - enum["flags"], - enum["values"], - ) - ) - for struct in dic["structs"]: - fields = [] - virtual_functions = None - member_functions = [] - static_member_functions = None - static_members = None - for field in struct["fields"]: - base = field["base"] if "base" in field else False - if "size" in field: - fields.append( - DefinedStructFixedField( - field["name"], - field["type"], - field["offset"], - base, - field["size"], - field["is_string"] - ) - ) - elif "return_type" in field: - parameters = [] - for param in field["parameters"]: - parameters.append( - DefinedStructFuncParam(param["name"], param["type"]) - ) - fields.append( - DefinedStructFuncField( - field["name"], - field["type"], - field["offset"], - base, - field["return_type"], - parameters, - ) - ) - else: - fields.append( - DefinedStructField( - field["name"], field["type"], field["offset"], base - ) - ) - if "virtual_functions" in struct: - virtual_functions = [] - for vfunc in struct["virtual_functions"]: - parameters = ( - [ - DefinedStructFuncParam(param["name"], param["type"]) - for param in vfunc["parameters"] - ] - if "parameters" in vfunc - else None - ) - virtual_functions.append( - DefinedStructVFunc( - vfunc["name"], - vfunc["return_type"] if "return_type" in vfunc else None, - vfunc["offset"], - parameters, - ) - ) - for memfunc in struct["member_functions"]: - parameters = [] - for param in memfunc["parameters"]: - parameters.append(DefinedStructFuncParam(param["name"], param["type"])) - member_functions.append( - DefinedStructMemFunc( - memfunc["signature"], - memfunc["return_type"], - parameters, - memfunc["name"], - ) - ) - if "static_member_functions" in struct: - static_member_functions = [] - for smemfunc in struct["static_member_functions"]: - parameters = [] - for param in smemfunc["parameters"]: - parameters.append( - DefinedStructFuncParam(param["name"], param["type"]) - ) - static_member_functions.append( - DefinedStructMemFunc( - smemfunc["signature"], - smemfunc["return_type"], - parameters, - smemfunc["name"], - ) - ) - if "static_members" in struct: - static_members = [] - for sm in struct["static_members"]: - static_members.append( - DefinedStructStaticMember( - sm["name"], - sm["signature"], - sm["relative_follow_offsets"], - sm["return_type"], - sm["is_pointer"] if "is_pointer" in sm else False, - ) - ) - size = None - if "size" in struct: - size = struct["size"] - vtable_size = None - if "vtable_size" in struct: - vtable_size = struct["vtable_size"] - structs.append( - DefinedStruct( - struct["name"], - struct["type"], - struct["namespace"], - fields, - size, - vtable_size, - virtual_functions, - member_functions, - struct["union"], - static_member_functions, - static_members, - ) - ) - return DefinedStructExport(enums, structs) - - @abstractmethod - def preprocess_yaml(self, yml: DefinedStructExport): - """ - Preprocesses the YAML data before importing. - - For the IDA srclang importer, this fixes issues with generic base classes. - """ - - def load_data_yaml(self): - # type: () -> dict - path = os.path.join(os.path.dirname(self.get_file_path), "data.yml") - if not os.path.exists(path): - return None - with open(path, "r") as fd: - return load(fd, Loader=Loader) - -api = None - -if api is None: - try: - import idaapi - import idc - import ida_bytes - import ida_search - import ida_typeinf - import ida_funcs - import ida_name - import ida_kernwin - import ida_srclang - import hashlib - import copy - from ida_wrapper import IdaInterface - except ImportError: - print("Warning: Unable to load IDA") - else: - # noinspection PyUnresolvedReferences - class IdaApi(BaseApi, IdaInterface): - def __init__(self, full_padding, srclang_importer): - # type: (bool, bool) -> None - self.full_padding = full_padding - self.srclang_importer = srclang_importer - if self.srclang_importer: - self.srclang_types = {} - - def validate_name_cfg(self): - """Verifies that the user's IDA config allows template characters for type names""" - temporary_name = "struc_name_test" - template_name = "OuterStructTest*>" - - def delete_test_type(name): - sid = self.get_struct_id(name) - if sid == idaapi.BADADDR: - return - if idaapi.IDA_SDK_VERSION >= 900: - ida_typeinf.del_named_type(idaapi.get_idati(), name, ida_typeinf.NTF_TYPE) - else: - idc.del_struc(sid) - - created_name = None - try: - template_sid = self.get_struct_id(template_name) - temporary_sid = self.get_struct_id(temporary_name) - - if template_sid != idaapi.BADADDR: - if temporary_sid != idaapi.BADADDR: - return True - - self.rename_struct(template_sid, temporary_name) - if self.get_struct_id(template_name) != idaapi.BADADDR: - raise RuntimeError("could not rename existing template test type") - - temporary_sid = self.get_struct_id(temporary_name) - - if temporary_sid == idaapi.BADADDR: - temporary_sid = self.create_struct_type(temporary_name) - - if temporary_sid == idaapi.BADADDR: - raise RuntimeError("could not create temporary struct") - - created_name = temporary_name - - self.rename_struct(temporary_sid, template_name) - if self.get_struct_id(template_name) == idaapi.BADADDR: - raise RuntimeError("could not rename temporary struct to template type name") - - created_name = template_name - except Exception as exc: - return False # dropping exception, but will prompt the user to fix their cfg and exit - - finally: - if created_name is not None: - delete_test_type(created_name) - delete_test_type(temporary_name) - - return True - - def get_fallback_vfunc_name(self, class_name, index, visited=None): - # type: (str, int, set) -> str - if not hasattr(self, "data_yaml") or not self.data_yaml or "classes" not in self.data_yaml: - return None - - if visited is None: - visited = set() - - if class_name in visited: - return None - visited.add(class_name) - - if class_name not in self.data_yaml["classes"]: - return None - - class_data = self.data_yaml["classes"][class_name] - if not class_data: - return None - - if "vfuncs" in class_data and index in class_data["vfuncs"]: - return class_data["vfuncs"][index] - - if "vtbls" in class_data and isinstance(class_data["vtbls"], list) and len(class_data["vtbls"]) > 0: - vtbl = class_data["vtbls"][0] - if "base" in vtbl: - res = self.get_fallback_vfunc_name(vtbl["base"], index, visited) - if res: - return res - - return None - - def get_vft_from_data(self, class_name): - # type: (str) -> dict | None - if not self.data_yaml or "classes" not in self.data_yaml: - return None - - class_data = self.data_yaml["classes"].get(class_name) or {} - vtables = class_data.get("vtbls", []) - if not isinstance(vtables, list) or not vtables: - return None - - vtable = vtables[0] - return vtable if isinstance(vtable, dict) else None - - def get_vfunc_ea_from_data(self, class_name, offset): - # type: (str, int) -> int - vtable = self.get_vft_from_data(class_name) - if vtable is None or "ea" not in vtable: - return idc.BADADDR - - # assuming data.yml will continue using default windows base - vtable_ea = vtable["ea"] + idaapi.get_imagebase() - 0x140000000 - return idc.get_qword(vtable_ea + offset) - - def delete_struct_members(self, fullname): - # type: (str) -> None - self.remove_struct_members(self.get_struct_id(fullname)) - - @property - def get_file_path(self): - return os.path.join( - os.path.dirname(os.path.realpath(__file__)), "ffxiv_structs.yml" - ) - - def generate_hashed_type_name(self, name: str) -> str: - name = self.clean_struct_name(name) - - name = hashlib.sha1(name.encode()).hexdigest() - - return "struc_" + name - - def get_srclang_type_name(self, name: str) -> str: - if not self.srclang_importer: - return name - - ptr_count = 0 - i = len(name) - 1 - while i >= 0 and name[i] == '*': - ptr_count += 1 - i -= 1 - - full_name = name - if ptr_count > 0: - name = name.strip("*") - - cname = self.srclang_types.get(name) - if cname is None: - cname = self.srclang_types.get(self.clean_struct_name(name)) - - if cname is not None: - return cname + "*" * ptr_count - - return full_name - - def can_run(self): - return self.enum_exists("Component::Exd::SheetsEnum") - - def create_enum_struct(self, enum): - # type: (DefinedStructEnum) -> None - fullname = enum.type - - e = self.get_enum_id(fullname) - if e == idaapi.BADADDR: - e = self.create_enum(fullname) - - self.set_enum_width(e, self.get_size_from_ida_type(enum.underlying)) - if self.is_signed(enum.underlying): - self.set_enum_flag(e, 0x20000) - if enum.flags: - if idaapi.IDA_SDK_VERSION < 900: - self.add_enum_member(e, "{0}.{1}".format(enum.name, "tmp"), self.get_enum_default_mask(e)) - self.set_enum_as_bf(e) - for value in enum.values: - self.add_enum_member( - e, "{0}.{1}".format(enum.name, value), enum.values[value] - ) - if enum.flags and idaapi.IDA_SDK_VERSION < 900: - self.remove_enum_member(e, "tmp", enum.name) - - def delete_enum(self, enum): - # type: (DefinedStructEnum) -> None - eid = idc.get_enum(enum.type) - if eid != idaapi.BADADDR: - self.delete_enum_members(eid) - idc.set_enum_bf(eid, False) - - def delete_struct(self, struct): - # type: (DefinedStruct) -> None - idaapi.begin_type_updating(idaapi.UTP_STRUCT) - fullname = self.clean_struct_name(struct.type) - self.delete_struct_members(fullname) - self.delete_struct_members(fullname + "_vtbl") - - # also check C types in case of an incomplete C run - if self.srclang_importer: - cname = self.get_srclang_type_name(fullname) - self.delete_struct_members(cname) - self.delete_struct_members(cname + "_vtbl") - - idaapi.end_type_updating(idaapi.UTP_STRUCT) - - def create_struct(self, struct): - # type: (DefinedStruct) -> None - fullname = self.clean_struct_name(struct.type) - - if self.srclang_importer: - # rename C++ -> C or create new C struct - cname = self.generate_hashed_type_name(fullname) - self.srclang_types[fullname] = cname - - sid = self.get_struct_id(fullname) - if sid == idaapi.BADADDR: - if self.get_struct_id(cname) == idaapi.BADADDR: - self.create_struct_type(cname, struct.union) - else: - self.rename_struct(sid, cname) - - if not struct.virtual_functions: - return - - sid = self.get_struct_id(fullname + "_vtbl") - if sid == idaapi.BADADDR: - if self.get_struct_id(cname + "_vtbl") == idaapi.BADADDR: - self.create_struct_type(cname + "_vtbl") - else: - self.rename_struct(sid, cname + "_vtbl") - - return - - if self.get_struct_id(fullname) == idaapi.BADADDR: - self.create_struct_type(fullname, struct.union) - - if struct.virtual_functions: - self.create_struct_type(fullname + "_vtbl") - - def validate_srclang_struct(self, struct: DefinedStruct): - cname = self.get_srclang_type_name(self.clean_struct_name(struct.type)) - sid = self.get_struct_id(cname) - if sid == idaapi.BADADDR: - ida_kernwin.warning(f"Struct {cname} ({struct.type}) not found during validation") - exit() - - ti = self.get_struct(sid) - - expected_base_offsets = {} - for field in struct.fields: - if not field.srclang_is_baseclass: - continue - expected_base_offsets[field.offset] = ( - expected_base_offsets.get(field.offset, 0) + 1 - ) - - if expected_base_offsets: - udt = ida_typeinf.udt_type_data_t() - if not ti.get_udt_details(udt): - ida_kernwin.warning( - f"Could not find baseclass for {cname} ({struct.type}) during validation" - ) - exit() - - actual_base_offsets = {} - for udm in udt: - if not udm.is_baseclass(): - continue - offset = int(udm.offset / 8) - actual_base_offsets[offset] = actual_base_offsets.get(offset, 0) + 1 - - for offset, expected_count in expected_base_offsets.items(): - actual_count = actual_base_offsets.get(offset, 0) - if actual_count < expected_count: - ida_kernwin.warning( - f"Baseclass offset mismatch in struct {cname} ({struct.type}).\n" - f"Expected {expected_count} member(s) for baseclass at {offset}, got {actual_count}" - ) - exit() - - last_offset = -1 - for field in struct.fields: - if field.offset == last_offset: - continue - - last_offset = field.offset - - if field.offset is None or field.srclang_is_baseclass: - continue - - if field.offset == 0 and field.name == "_vtable": - continue - - # ugly hack to workaround duplicate field names in structs - field_name = field.srclang_field_name - - (idx, udm) = ti.get_udm(field_name) - if idx == -1: - ida_kernwin.warning(f"Field {field_name} not found in struct {cname} ({struct.type}) during validation") - exit() - - if (udm.offset / 8) != field.offset: - ida_kernwin.warning(f"Field \"{field_name}\" offset mismatch in struct {cname} ({struct.type}) during validation.\nExpected {field.offset}, got {(udm.offset/8)}") - exit() - - def get_srclang_fill_type(self, available_bytes: int, current_offset: int) -> tuple[str, int]: - if available_bytes >= 8 and (current_offset % 8) == 0: - return ("__int64", 8) - elif available_bytes >= 4 and (current_offset % 4) == 0: - return ("__int32", 4) - elif available_bytes >= 2 and (current_offset % 2) == 0: - return ("__int16", 2) - else: - return ("char", 1) - - def append_srclang_padding(self, decl: list[str], current_size: int, target_size: int) -> int: - while current_size < target_size: - if self.full_padding: - (fill_type, fill_size) = self.get_srclang_fill_type( - target_size - current_size, current_size - ) - decl.append(f"{fill_type} field_{current_size:X};") - current_size += fill_size - else: - arr_size = target_size - current_size - decl.append(f"char field_{current_size:X}[{arr_size}];") - current_size += arr_size - - return current_size - - def create_srclang_decl(self, struct: DefinedStruct) -> str: - fullname = self.get_srclang_type_name(self.clean_struct_name(struct.type)) - - decl = [ "_" ] # placeholder, filled after we determine base classes - - cur_size = 0 - - contiguous_fields = True - - seen_fields = {} - - inherits_from = [] - - has_explicit_vtable = (len(struct.fields) != 0 and struct.fields[0].name == "_vtable") - - if struct.virtual_functions != None or has_explicit_vtable: - # the placeholder will force IDA to mark the _vtbl struct as a VFT - # and attach it to this struct - decl.append("virtual void _placeholder();") - - # offset for vfptr if needed - if struct.srclang_needs_vfptr or has_explicit_vtable: - cur_size += 8 - - last_field_offset = -1 - for field in struct.fields: - offset = field.offset - - # skip explicit vtable fields - if offset == 0 and field.name == "_vtable": - continue - - if offset == last_field_offset and not struct.union: - # NOTE In IDA versions < 9.0 you could have overlapping fields or an automatically created union. - # We're not able to support this for srclang, so overlapping fields should ideally be - # treated as a layout error which requires a union to resolve. - # - # I've made the decision here to drop these with a warning, but it is probably worth evaluating - # whether it's worthwhile to make this an error later. - print(f"Skipping {struct.type}.{field.name} as it is at a duplicate offset.") - continue - - last_field_offset = offset - - if offset > cur_size: - contiguous_fields = False - cur_size = self.append_srclang_padding(decl, cur_size, offset) - - field_is_base = field.base and contiguous_fields - field_name = ( - field.name - if not field_is_base - else f"baseclass_{offset:X}" - ) - field.srclang_is_baseclass = field_is_base - - # ugly hack to workaround duplicate field names in structs - if field_name in seen_fields: - next_index = seen_fields[field_name] + 1 - seen_fields[field_name] = next_index - - field_name += f"_{next_index}" - else: - seen_fields[field_name] = 1 - - field.srclang_field_name = field_name - - array_size = field.size if hasattr(field, "size") else 0 - - field_type = self.clean_name(field.type) - if field_type == "__fastcall": - field_decl = self.get_srclang_type_name(self.clean_name(field.return_type)) - field_decl = field_decl + "(__fastcall* " + field_name + ")(" - for param in field.parameters: - field_decl = field_decl + self.get_srclang_type_name(self.clean_name(param.type)) + "" - field_decl = field_decl + param.name + "," - field_decl = field_decl[:-2] + ")" - - decl.append(field_decl) - cur_size += 8 - - continue - - field_size = 0 - - # struct type - if self.get_idc_type_from_ida_type( - self.get_srclang_type_name(self.clean_struct_name(field_type)) - ) == self.get_struct_flag(): - field_type = self.get_srclang_type_name(self.clean_struct_name(field_type)) - - tinfo = self.get_tinfo_from_type(field_type) - field_size = tinfo.get_size() - - # enum type - elif ( - self.get_idc_type_from_ida_type(field_type) - == self.get_enum_flag() - ): - field_size = idc.get_enum_width(self.get_enum_id(field_type)) - - # primitive type - else: - field_size = self.get_size_from_ida_type(field_type) - - if field_type.endswith("*"): - field_type = self.get_srclang_type_name(field_type) - - field_decl = f"{field_type} {field_name}" - if array_size > 0: - field_size *= array_size - field_decl += f"[{array_size}];" - else: - field_decl += ";" - - if field_is_base: - inherits_from.append(field_type) - else: - decl.append(field_decl) - - cur_size += field_size - - if struct.size is not None and struct.size != 0: - cur_size = self.append_srclang_padding(decl, cur_size, struct.size) - - decl.append("};") - - # set struct type - if struct.union: - decl[0] = f"union {fullname} " - else: - decl[0] = f"struct __attribute__((packed)) {fullname} " - if len(inherits_from) > 0: - inheritances = ", ".join(inherits_from) - decl[0] += f": {inheritances}" - - decl[0] += " {" - - return "\n".join(decl) - - def create_struct_member_fill(self, struct_name, offset): - # type: (str, int) -> None - s = self.get_struct(self.get_struct_id(struct_name)) - prev_size = self.get_struct_size(s) - if self.full_padding: - flag = self.get_idc_type_from_size(prev_size) - size = self.get_size_from_idc_type(flag) - if size > offset - prev_size: - flag = self.get_idc_type_from_size( - offset - prev_size, prev_size - ) - size = self.get_size_from_idc_type(flag) - - self.create_struct_member( - s, "field_{0:X}".format(prev_size), prev_size, flag, None, size - ) - else: - self.create_struct_member( - s, - "field_{0:X}".format(prev_size), - prev_size, - ida_bytes.byte_flag(), - None, - offset - prev_size, - ) - - def create_struct_members(self, struct): - # type: (DefinedStruct) -> None - idaapi.begin_type_updating(idaapi.UTP_STRUCT) - - if self.srclang_importer: - idaapi.begin_type_updating(idaapi.UTP_STRUCT) - - decl = self.create_srclang_decl(struct) - num_errors = ida_srclang.parse_decls_for_srclang( - ida_srclang.SRCLANG_C, - None, - decl, - False - ) - - if num_errors != 0: - # show messagebox - print(f"above errors occurred while parsing the following:\n---\n{decl}\n---") - ida_kernwin.warning(f"Error parsing srclang decl for {struct.type}, please see errors in Output window.") - exit() - - if struct.virtual_functions is not None: - # delete the _placeholder function from the VFT - cname = self.srclang_types[self.clean_struct_name(struct.type)] - sid = self.get_struct_id(f"{cname}_vtbl") - tinfo = self.get_struct(sid) - tinfo.del_udm(0) - - idaapi.end_type_updating(idaapi.UTP_STRUCT) - - self.validate_srclang_struct(struct) - return - - fullname = self.clean_struct_name(struct.type) - - tid = self.get_struct_id(fullname) - if tid == idaapi.BADADDR: - print("Error: Struct {0} not found when trying to create members".format(fullname)) - return - - s = self.get_struct(tid) - - if struct.virtual_functions != None and ( - struct.fields == [] or struct.fields[0].offset > 0 - ): - self.create_struct_member( - s, "__vftable", 0, ida_bytes.qword_flag(), None, 8 - ) - type = fullname + "_vtbl*" if struct.virtual_functions else "void**" - meminfo = self.get_struct_member_by_name(s, "__vftable") - self.set_struct_member_info( - s, meminfo, 0, self.get_tinfo_from_type(type), 0, False - ) - - contiguous_fields = True - for field in struct.fields: - offset = field.offset - - prev_size = self.get_struct_size(s) - while offset > prev_size: - contiguous_fields = False - self.create_struct_member_fill(fullname, offset) - prev_size = self.get_struct_size(s) - - field_is_base = field.base and contiguous_fields - field_name = ( - field.name - if not field_is_base - else "baseclass_{0:X}".format(offset) - ) - field_type = self.clean_name(field.type) - if field_type == "__fastcall": - self.create_struct_member( - s, - field_name, - offset, - self.get_idc_type_from_ida_type("__int64"), - None, - self.get_size_from_ida_type("__int64"), - ) - field_type = self.clean_name(field.return_type) - field_type = field_type + "(__fastcall* " + field_name + ")(" - for param in field.parameters: - field_type = field_type + self.clean_name(param.type) + "" - field_type = field_type + param.name + "," - field_type = field_type[:-2] + ")" - elif ( - self.get_idc_type_from_ida_type( - self.clean_struct_name(field_type) - ) - == self.get_struct_flag() - ): - field_type = self.clean_struct_name(field_type) - self.create_struct_member( - s, - field_name, - offset, - self.get_idc_type_from_ida_type(field_type), - self.get_struct_opinfo_from_type(field_type), - self.get_size_from_ida_type(field_type), - ) - elif ( - self.get_idc_type_from_ida_type(field_type) - == self.get_enum_flag() - ): - self.create_struct_member( - s, - field_name, - offset, - self.get_idc_type_from_ida_type(field_type), - self.get_enum_opinfo_from_type(field_type), - self.get_size_from_ida_type(field_type), - ) - else: - self.create_struct_member( - s, - field_name, - offset, - self.get_idc_type_from_ida_type(field_type), - None, - self.get_size_from_ida_type(field_type), - ) - - meminfo = self.get_struct_member_by_name(s, field_name) - if meminfo is not None: - if field_is_base: - if idaapi.IDA_SDK_VERSION >= 900: - meminfo.set_baseclass() - else: - meminfo.props |= self.get_base_class_flag() - - array_size = field.size if hasattr(field, "size") else 0 - self.set_struct_member_info( - s, - meminfo, - 0, - self.get_tinfo_from_type(field_type, array_size), - 0, - field.is_string if hasattr(field, "is_string") and (field_type == "char" or field_type == "wchar_t") else False - ) - - if struct.size is not None and struct.size != 0: - prev_size = self.get_struct_size(s) - while struct.size > prev_size: - self.create_struct_member_fill(fullname, struct.size) - prev_size = self.get_struct_size(s) - - idaapi.end_type_updating(idaapi.UTP_STRUCT) - - def create_vtable(self, struct): - # type: (DefinedStruct) -> None - fullname = self.clean_name(struct.type) - s = self.get_struct(self.get_struct_id(fullname + "_vtbl")) - for virt_func in struct.virtual_functions: - if virt_func is None: - continue - - offset = virt_func.offset - field_name = virt_func.name - self.create_struct_member( - s, - field_name, - offset, - self.get_idc_type_from_ida_type("__int64"), - None, - self.get_size_from_ida_type("__int64"), - ) - if virt_func.return_type == None or virt_func.parameters == None: - continue - - meminfo = self.get_struct_member_by_name(s, field_name) - if meminfo is None: - raise RuntimeError("Failed to find member {0} in struct {1}".format(field_name, fullname)) - - field_type = self.clean_name(virt_func.return_type) - field_type = field_type + "(__fastcall* " + field_name + ")(" - for param in virt_func.parameters: - field_type = field_type + self.clean_name(param.type) + " " - field_type = field_type + param.name + "," - field_type = field_type[:-1] + ")" - - self.set_struct_member_info( - s, meminfo, 0, self.get_tinfo_from_type(field_type), 0, False - ) - if struct.vtable_size: - size = int(struct.vtable_size / 8) - else: - size = int(self.get_struct_size(s) / 8) - - for i in range(size): - offset = i * 8 - member_id = self.get_struct_member_id(s, offset) - if idaapi.IDA_SDK_VERSION >= 900: - udm_idx, udm = s.get_udm_by_offset(offset * 8) - - # srclang ends up with gap UDMs in VFTs that need to be deleted and filled back out as vf## placeholders - is_vft_gap = ( - udm is not None - and udm.name.startswith("gap") - and udm.type.dstr().startswith("_BYTE[") - ) - if is_vft_gap: - result = s.del_udm(udm_idx) - if result == ida_typeinf.TERR_OK: - member_id = idc.BADADDR - else: - print( - f"Error: failed to delete VFT gap UDM {fullname}.{udm.name} " - f"at slot {i}: {ida_typeinf.tinfo_errstr(result)}" - ) - if member_id == idc.BADADDR: - name = "vf{0}".format(i) - fallback_name = self.get_fallback_vfunc_name(struct.type, i) - if fallback_name: - name = fallback_name - - self.create_struct_member( - s, - name, - i * 8, - self.get_idc_type_from_ida_type("__int64"), - None, - self.get_size_from_ida_type("__int64"), - ) - meminfo = self.get_struct_member_by_name(s, name) - self.set_struct_member_info( - s, meminfo, 0, self.get_tinfo_from_type("__int64"), 0, False - ) - - def finalise_struct(self, struct: DefinedStruct): - if not self.srclang_importer: - return - - fullname = self.clean_struct_name(struct.type) - cname = self.get_srclang_type_name(fullname) - - sid = self.get_struct_id(cname) - if sid == idaapi.BADADDR: - ida_kernwin.warning(f"Failed to find and finalise struct {cname}") - - self.rename_struct(sid, fullname) - - if not struct.virtual_functions: - return - - sid = self.get_struct_id(cname + "_vtbl") - if sid == idaapi.BADADDR: - ida_kernwin.warning(f"Failed to find and finalise vtable for struct {cname}") - return - - self.rename_struct(sid, fullname + "_vtbl") - - def create_union(self, struct): - # type: (DefinedStruct) -> None - pass - - def update_member_func(self, member_func, struct): - # type: (DefinedStructMemFunc, DefinedStruct) -> None - func_name = "{0}.{1}".format( - self.clean_name(struct.type), member_func.name - ) - ea = self.get_func_ea_by_name(func_name) - if ea == idc.BADADDR: - ea = self.get_func_ea_by_sig(member_func.signature) - if ea == idc.BADADDR: - print( - "Error: {0} not found bad sig? {1}".format( - func_name, member_func.signature - ) - ) - return - if ida_funcs.get_func_name(ea) == "sub_{0:X}".format(ea): - idc.set_name(ea, func_name) - tif = ida_typeinf.tinfo_t() - ida_typeinf.guess_tinfo(tif, ea) - func_data = ida_typeinf.func_type_data_t() - tif.get_func_details(func_data) - func_data.clear() - func_data.cc = ida_typeinf.CM_CC_FASTCALL - func_data.rettype = self.get_tinfo_from_type(member_func.return_type) - for param in member_func.parameters: - arg = ida_typeinf.funcarg_t() - try: - arg.type = self.get_tinfo_from_type(param.type) - except ValueError as exc: - print( - "Error: update_member_func: function={!r}, ea={:#x}, " - "parameter={!r}, type={!r}, error={}".format( - func_name, ea, - param.name, param.type, exc - ) - ) - raise - arg.name = param.name - func_data.push_back(arg) - tif.create_func(func_data) - ida_typeinf.apply_tinfo(ea, tif, ida_typeinf.TINFO_DEFINITE) - - def update_virt_func(self, virt_func, struct): - # type: (DefinedStructVFunc, DefinedStruct) -> None - func_name = "{0}.{1}".format( - self.clean_name(struct.type), virt_func.name - ) - - ea = self.get_func_ea_by_name(func_name) - if ea == idc.BADADDR: - # ignore if added during preprocess - if virt_func.inherited_from_preprocess: - return - - ea = self.get_vfunc_ea_from_data(struct.type, virt_func.offset) - if ea in (0, idc.BADADDR): - print(f"Error: {func_name} not found and its VFT slot could not be resolved") - return - - actual_name = ida_funcs.get_func_name(ea) or "" - normalized_name = actual_name - for prefix in ("j_", "thunk_"): - if normalized_name.startswith(prefix): - normalized_name = normalized_name[len(prefix):] - - # pure virtual, nothing to do - if normalized_name == "_purecall": - return - - primary_vtable = self.get_vft_from_data(struct.type) - primary_base = primary_vtable.get("base") if primary_vtable else None - if primary_base: - base_ea = self.get_vfunc_ea_from_data(primary_base, virt_func.offset) - # if the derived vft slot points to the same as the baseclass, leave baseclass type - if base_ea == ea: - return - - # does it look inherited based on name? leave it alone - if ( - normalized_name.endswith("." + virt_func.name) - and not normalized_name.startswith(self.clean_name(struct.type) + ".") - ): - return - - tif = ida_typeinf.tinfo_t() - ida_typeinf.guess_tinfo(tif, ea) - - func_data = ida_typeinf.func_type_data_t() - tif.get_func_details(func_data) - - func_data.clear() - func_data.cc = ida_typeinf.CM_CC_FASTCALL - func_data.rettype = self.get_tinfo_from_type(virt_func.return_type) - - for param in virt_func.parameters: - arg = ida_typeinf.funcarg_t() - try: - arg.type = self.get_tinfo_from_type(param.type) - except ValueError as exc: - print( - "Error: update_virt_func: function={!r}, ea={:#x}, " - "parameter={!r}, type={!r}, error={}".format( - func_name, ea, param.name, param.type, exc - ) - ) - raise - - arg.name = param.name - func_data.push_back(arg) - - tif.create_func(func_data) - ida_typeinf.apply_tinfo(ea, tif, ida_typeinf.TINFO_DEFINITE) - - def update_static_member(self, static_member, struct): - # type: (DefinedStructStaticMember, DefinedStruct) -> None - ea = self.search_binary( - 0, static_member.signature, ida_search.SEARCH_DOWN - ) - if ea == idc.BADADDR: - print( - "Error: {0} not found something is wrong".format( - static_member.signature - ) - ) - return - for follows in static_member.relative_offsets: - ea = ea + follows - ea = ea + 4 + self.get_dword(ea) - tif = ida_typeinf.tinfo_t() - ida_typeinf.guess_tinfo(tif, ea) - return_type = static_member.return_type - if static_member.is_pointer: - return_type = return_type + "*" - ida_typeinf.apply_tinfo( - ea, - self.get_tinfo_from_type(return_type), - ida_typeinf.TINFO_DEFINITE, - ) - ida_name.set_name( - ea, - "g_{0}_{1}".format(self.clean_name(struct.type), static_member.name), - ) - - def should_update_member_func(self): - return ( - ida_kernwin.ask_yn( - ida_kernwin.ASKBTN_YES, "Update member function types?" - ) - == ida_kernwin.ASKBTN_YES - ) - - def should_update_virt_func(self): - return ( - ida_kernwin.ask_yn( - ida_kernwin.ASKBTN_YES, "Update virtual function types?" - ) - == ida_kernwin.ASKBTN_YES - ) - - def preprocess_yaml(self, yaml: DefinedStructExport) -> None: - # Attempts to identify and fix missing baseclass flags and VFT inconsistencies, checks - # for issues that were previously 'hidden' by the IDA API, then builds an inheritance - # graph to propagate virtual functions and validate layouts & inheritance chains - - if not self.srclang_importer: - return - - data_classes = {} - if hasattr(self, "data_yaml") and self.data_yaml: - data_classes = self.data_yaml.get("classes", {}) or {} - - def get_data_vtables(class_name: str) -> list[dict]: - class_data = data_classes.get(class_name) or {} - vtables = class_data.get("vtbls", []) - return vtables if isinstance(vtables, list) else [] - - class BaseEdge: - def __init__(self, derived: 'Node', base: 'Node', field: DefinedStructField): - self.derived = derived - self.base = base - self.field = field - self.offset = field.offset - - class Node: - def __init__(self, struct: DefinedStruct): - self.struct: DefinedStruct = struct - self.declared_vtable_size = struct.vtable_size - self.bases: list['BaseEdge'] = [] - self.children: list['BaseEdge'] = [] - - def is_virtual(self) -> bool: - return self.struct.virtual_functions is not None - - def primary_base(self) -> 'BaseEdge | None': - primary_bases = [edge for edge in self.bases if edge.offset == 0] - if len(primary_bases) > 1: - base_names = ", ".join(edge.base.struct.type for edge in primary_bases) - raise RuntimeError( - f"Multiple baseclasses defined at offset 0 for '{self.struct.type}': {base_names}" - ) - - return primary_bases[0] if primary_bases else None - - nodes: dict[str, Node] = {} - for struct in yaml.structs: - if struct.type in nodes: - raise RuntimeError(f"Duplicate type '{struct.type}'") - - nodes[struct.type] = Node(struct) - - for node in nodes.values(): - declared_vtable_size = node.declared_vtable_size - if declared_vtable_size is not None: - if type(declared_vtable_size) is not int or declared_vtable_size < 0: - raise RuntimeError( - f"Invalid VFT size {declared_vtable_size} " - f"for '{node.struct.type}'" - ) - - if declared_vtable_size % 8 != 0: - raise RuntimeError( - f"Invalid VFT size {declared_vtable_size:#x} " - f"for '{node.struct.type}'" - ) - - vfuncs_by_offset = {} - for vfunc in node.struct.virtual_functions or []: - if vfunc is None: - continue - - if type(vfunc.offset) is not int or vfunc.offset < 0: - raise RuntimeError( - f"Invalid VFT offset {vfunc.offset} for " - f"'{node.struct.type}.{vfunc.name}'" - ) - - if vfunc.offset % 8 != 0: - raise RuntimeError( - f"Unaligned VFT offset {vfunc.offset:#x} for " - f"'{node.struct.type}.{vfunc.name}'" - ) - - previous_vfunc = vfuncs_by_offset.get(vfunc.offset) - if previous_vfunc is not None: - raise RuntimeError( - f"Duplicate VFT slot {vfunc.offset // 8} " - f"for '{node.struct.type}': " - f"'{previous_vfunc.name}' and '{vfunc.name}'" - ) - - vfuncs_by_offset[vfunc.offset] = vfunc - - # normalize - for node in nodes.values(): - if node.struct.fields and node.struct.fields[0].name == "_vtable": - node.struct.fields.pop(0) - if not node.is_virtual(): - node.struct.virtual_functions = [] - node.struct.vtable_size = 0 - - if not node.is_virtual() and get_data_vtables(node.struct.type): - node.struct.virtual_functions = [] - node.struct.vtable_size = 0 - - def add_base_edge(derived: Node, field: DefinedStructField) -> BaseEdge: - parent = nodes.get(field.type) - if parent is None: - raise RuntimeError( - f"Could not resolve baseclass '{field.type}' " - f"for '{derived.struct.type}'" - ) - - edge = BaseEdge(derived, parent, field) - derived.bases.append(edge) - parent.children.append(edge) - return edge - - # add every explicit baseclass - for node in nodes.values(): - for field in node.struct.fields: - if field.base: - add_base_edge(node, field) - - def is_or_has_primary(node: Node, class_name: str, visited=None) -> bool: - if node.struct.type == class_name: - return True - - if visited is None: - visited = set() - - if node.struct.type in visited: - return False - - visited.add(node.struct.type) - - edge = node.primary_base() - return ( - edge is not None - and is_or_has_primary(edge.base, class_name, visited) - ) - - for node in nodes.values(): - if node.primary_base() is not None: - continue - - vtables = get_data_vtables(node.struct.type) - primary_vtable = vtables[0] if vtables else None - primary_vtable_base = ( - primary_vtable.get("base") - if isinstance(primary_vtable, dict) - else None - ) - - if not primary_vtable_base: - continue - - candidates = [] - for field in node.struct.fields: - candidate = nodes.get(field.type) - if ( - field.offset == 0 - and not field.base - and candidate is not None - and is_or_has_primary(candidate, primary_vtable_base) - ): - candidates.append((field, candidate)) - - if len(candidates) != 1: - continue - - field, _ = candidates[0] - - print( - f"Warning: Fixing missing baseclass '{field.type}' " - f"for '{node.struct.type}'" - ) - - field.base = True - add_base_edge(node, field) - - # validate inheritance graph to make sure we don't have any circular dependants - visit_state: dict[str, int] = {} - visit_stack: list[str] = [] - - def validate_acyclic(node: Node) -> None: - state = visit_state.get(node.struct.type, 0) - if state == 2: - return - - if state == 1: - cycle_start = visit_stack.index(node.struct.type) - cycle = visit_stack[cycle_start:] + [node.struct.type] - cycle_path = " -> ".join(cycle) - raise RuntimeError(f"Circular inheritance detected: {cycle_path}") - - visit_state[node.struct.type] = 1 - - visit_stack.append(node.struct.type) - for edge in node.bases: - validate_acyclic(edge.base) - visit_stack.pop() - - visit_state[node.struct.type] = 2 - - for node in nodes.values(): - validate_acyclic(node) - - - inherited_vfts_visited: set[str] = set() - - def inherit_vft(node: Node) -> None: - if node.struct.type in inherited_vfts_visited: - return - - edge = node.primary_base() - if edge is not None: - inherit_vft(edge.base) - if edge.base.is_virtual() and not node.is_virtual(): - node.struct.virtual_functions = [] - node.struct.vtable_size = 0 - - inherited_vfts_visited.add(node.struct.type) - - for node in nodes.values(): - inherit_vft(node) - - for node in nodes.values(): - primary_base = node.primary_base() - primary_base_has_vft = ( - primary_base is not None and primary_base.base.is_virtual() - ) - - node.struct.srclang_needs_vfptr = ( - node.is_virtual() and not primary_base_has_vft - ) - - if ( - node.struct.srclang_needs_vfptr - and primary_base is not None - ): - # TODO(Caitlyn): maybe this should be a hard error here - print( - f"Warning: Class '{node.struct.type}' is polymorphic, " - f"but its non-polymorphic base class " - f"'{primary_base.base.struct.type}' is placed at offset zero" - ) - - # normalize VFTs into indexable lists - normalized_vfts: set[str] = set() - - def normalize_vft(node: Node) -> None: - if node.struct.type in normalized_vfts: - return - - primary_base = node.primary_base() - if primary_base is not None: - normalize_vft(primary_base.base) - - if not node.is_virtual(): - normalized_vfts.add(node.struct.type) - return - - vfuncs = node.struct.virtual_functions or [] - - last_vfunc = max( - (vf for vf in vfuncs if vf is not None), - key=lambda vf: vf.offset, - default=None, - ) - - needed_size = ( - last_vfunc.offset + 8 - if last_vfunc is not None - else 0 - ) - vft_size = needed_size - - if node.struct.vtable_size and node.struct.vtable_size > vft_size: - vft_size = node.struct.vtable_size - - needed_base_size = 0 - if ( - primary_base is not None - and primary_base.base.is_virtual() - and (primary_base.base.struct.vtable_size or 0) > vft_size - ): - needed_base_size = primary_base.base.struct.vtable_size - vft_size = needed_base_size - - defined_size = node.declared_vtable_size - if ( - defined_size is not None - and vft_size > defined_size - ): - if needed_base_size > needed_size: - reason = f"baseclass '{primary_base.base.struct.type}'" - else: - reason = ( - f"virtual function '{last_vfunc.name}' " - f"at offset {last_vfunc.offset:#x}" - ) - - print( - f"Warning: Expanding VFT for '{node.struct.type}' from " - f"{defined_size:#x} to {vft_size:#x} to fit {reason}" - ) - - new_vft = [None] * (vft_size // 8) - for vf in vfuncs: - if vf is not None: - new_vft[vf.offset // 8] = vf - - node.struct.virtual_functions = new_vft - node.struct.vtable_size = vft_size - normalized_vfts.add(node.struct.type) - - for node in nodes.values(): - normalize_vft(node) - - def primary_children(node: Node): - for edge in node.children: - if edge.derived.primary_base() is edge: - yield edge.derived - - def copy_vfunc_for_type(vf: DefinedStructVFunc, struct_type: str): - new_vf = copy.deepcopy(vf) - new_vf.inherited_from_preprocess = True - if hasattr(new_vf, 'parameters') and new_vf.parameters: - new_vf.parameters[0].type = struct_type + "*" - return new_vf - - # propagate known VFs down into subclasses - def propagate_vft_down(node: Node, visited=None) -> None: - if visited is None: - visited = set() - - if node.struct.type in visited: - return - - visited.add(node.struct.type) - - if not node.is_virtual(): - return - - parent_vfs = node.struct.virtual_functions or [] - - for child in primary_children(node): - if parent_vfs: - if not child.is_virtual(): - child.struct.virtual_functions = [None] * len(parent_vfs) - child.struct.vtable_size = node.struct.vtable_size - elif len(child.struct.virtual_functions) < len(parent_vfs): - extra_slots = len(parent_vfs) - len(child.struct.virtual_functions) - child.struct.virtual_functions += [None] * extra_slots - child.struct.vtable_size = max(child.struct.vtable_size or 0, node.struct.vtable_size) - - for i, vf in enumerate(parent_vfs): - if vf is None or child.struct.virtual_functions[i] is not None: - continue - - child.struct.virtual_functions[i] = copy_vfunc_for_type(vf, child.struct.type) - - propagate_vft_down(child, visited) - - visited_nodes: set[str] = set() - def propagate_vft_up(node: Node) -> None: - if node.struct.type in visited_nodes: - return - - visited_nodes.add(node.struct.type) - - if not node.is_virtual(): - return - - for child in primary_children(node): - # propagate from bottom up - propagate_vft_up(child) - - child_vfs = child.struct.virtual_functions - parent_vfs = node.struct.virtual_functions - if not child_vfs or not parent_vfs: - continue - - for i, vf in enumerate(child_vfs): - if i >= len(parent_vfs): - break - - if (vf is None or parent_vfs[i] is not None): - continue - - parent_vfs[i] = copy_vfunc_for_type(vf, node.struct.type) - - roots = [ - node - for node in nodes.values() - if node.is_virtual() - and ( - node.primary_base() is None - or not node.primary_base().base.is_virtual() - ) - ] - - for root in roots: - propagate_vft_down(root) - - for root in roots: - propagate_vft_up(root) - - for root in roots: - propagate_vft_down(root) - - full_padding = False - srclang_importer = False - if idaapi.IDA_SDK_VERSION >= 900: - srclang_importer = ( - ida_kernwin.ask_buttons( - "SrcLang Importer", - "Legacy Importer", - "", - ida_kernwin.ASKBTN_YES, - "HIDECANCEL\nWhich importer should be used?\n\nSrcLang Importer: Experimental - faster importer with full padding, and improved support for inheritance and virtual function calls.\n\nLegacy Importer: The original, battle-tested importer. Full Padding on the Legacy Importer on IDA 9+ can take 8 hours or longer.", - ) - == ida_kernwin.ASKBTN_YES - ) - - if srclang_importer: - full_padding = True - - if not srclang_importer: - full_padding = ( - ida_kernwin.ask_buttons( - "Full Padding", - "Array Padding", - "", - ida_kernwin.ASKBTN_YES, - "HIDECANCEL\nWhat padding style to use?\n\nFull Padding: Adds padding based on allignment of 1,2,4,8\nArray Padding: Adds padding based on the size between fields with byte arrays\n\nFull Padding will take longer to add padding between fields but is recommended for quick struct modifications.", - ) - == ida_kernwin.ASKBTN_YES - ) - - api = IdaApi(full_padding, srclang_importer) - if not api.validate_name_cfg(): - ida_kernwin.warning( - "Type name validation failed.\n" - "\n" - "Your IDA.cfg is missing necessary NameChars and TypeNameChars.\n" - "\n" - "Please see https://github.com/aers/FFXIVClientStructs/blob/main/ida/idauser.cfg to update your config accordingly.") - exit() - -if api is None: - try: - import ghidra - import re - - try: - from ghidra.ghidra_builtins import * - except ImportError: - pass - - from yaml import SafeLoader as Loader - - from ghidra.program.model.data import * - from ghidra.program.model.listing import * - from ghidra.program.model.symbol import SourceType - from ghidra.app.util import SymbolPathParser - from java.util import ArrayList - - except ImportError: - print("Warning: Unable to load Ghidra") - else: - # noinspection PyUnresolvedReferences - - class GhidraApi(BaseApi): - def can_run(self): - return True - - def get_size_from_type(self, name): - # type: (str) -> int - dt = self.get_datatype(name) - if dt is not None: - return dt.getLength() - return 0 - - def fix_generic_name(self, name): - # type: (str) -> str - if "<" not in name: - return name - for match in re.finditer(r"unsigned _*[\w*]{3,}|[:\w*]{3,}", name): - tn = self.get_ghidra_type( - SymbolPathParser.parse(match.group(0)).getLast() - ) - name = name.replace(match.group(0), tn) - return name - - def get_ghidra_type(self, name): - # type: (str) -> str - if name == "__int8": - return "char" - elif name == "__int16": - return "short" - elif name == "__int64": - return "longlong" - elif name == "unsigned __int16": - return "ushort" - elif name == "unsigned int": - return "uint" - elif name == "unsigned __int64": - return "ulonglong" - elif name == "__int8*": - return "char*" - elif name == "__int16*": - return "short*" - elif name == "__int64*": - return "longlong*" - elif name == "unsigned __int16*": - return "ushort*" - elif name == "unsigned int*": - return "uint*" - elif name == "unsigned __int64*": - return "ulonglong*" - elif name == "__fastcall": - return "void*" - return name - - def get_category_path(self, typename): - # type: (str) -> CategoryPath - syms = SymbolPathParser.parse(typename) - return CategoryPath("/" + "/".join(syms.subList(0, syms.size() - 1))) - - def get_datatype(self, typename): - # type: (str) -> DataType - raw_type = self.get_ghidra_type(typename) - if not raw_type: - return raw_type - typename = raw_type.rstrip("*") - pointer_count = len(raw_type) - len(typename) - - syms = SymbolPathParser.parse(typename) - syms[-1] = self.fix_generic_name(syms.getLast()) - - dtm = currentProgram.getDataTypeManager() - dt = dtm.getDataType("/" + "/".join(syms)) - for i in range(pointer_count): - dt = dtm.getPointer(dt) - return dt - - def create_datatype(self, datatype): - # type: (DataType) -> DataType - dtm = currentProgram.getDataTypeManager() - old = dtm.getDataType(datatype.getDataTypePath()) - if old is not None: - old.replaceWith(datatype) - return old - else: - return dtm.addDataType(datatype, None) - - def create_function_def(self, func): - # type: (DefinedStructVFunc) -> FunctionDefinitionDataType - fd = FunctionDefinitionDataType(func.name) - return_type = self.get_datatype(func.return_type) - fd.setReturnType(return_type) - args = [] - for arg in func.parameters: - arg_type = self.get_datatype(arg.type) - ad = ParameterDefinitionImpl(arg.name, arg_type, None) - args.append(ad) - fd.setArguments(args) - return fd - - def get_func_by_name(self, name): - # type: (str) -> Function - funcs = getGlobalFunctions(name) - return funcs.first if not funcs.size() == 0 else None - - def create_memberfunc_args(self, member_func): - # type: (DefinedStructMemFunc) -> ArrayList - arg_vars = ArrayList() - for param in member_func.parameters: - dt = self.get_datatype(param.type) - if not dt: - return ArrayList() - arg_vars.add(ParameterImpl(param.name, dt, currentProgram)) - return arg_vars - - @property - def get_file_path(self): - return os.path.join( - os.path.dirname(str(sourceFile)), "ffxiv_structs.yml" - ) - - def create_enum_struct(self, enum): - # type: (DefinedStructEnum) -> None - if monitor.isCancelled(): - return - enum_size = self.get_size_from_type(enum.underlying) or 4 - dt = EnumDataType(enum.name, enum_size) - dt.setCategoryPath(self.get_category_path(enum.type)) - for value in enum.values: - if not dt.contains(enum.values[value]): - dt.add(value, enum.values[value]) - self.create_datatype(dt) - - def delete_enum(self, enum): - # type: (DefinedStructEnum) -> None - pass - - def delete_struct(self, struct): - # type: (DefinedStruct) -> None - pass - - def create_struct(self, struct): - # type: (DefinedStruct) -> None - if monitor.isCancelled(): - return - - name = struct.name - syms = SymbolPathParser.parse(struct.type) - if syms.size() > 0: - name = syms.getLast() - - name = self.fix_generic_name(name) - if struct.union: - dt = UnionDataType(name) - else: - dt = StructureDataType(name, struct.size or 0) - dt.setCategoryPath(self.get_category_path(struct.type)) - self.create_datatype(dt) - - def create_struct_members(self, struct): - # type: (DefinedStruct) -> None - dt = self.get_datatype(struct.type) - if dt is None: - return - - struct.fields.sort(key=lambda fld: fld.offset) - dtsize = dt.getLength() if not dt.isZeroLength() else 0 - if ( - dtsize == 0 - and struct.virtual_functions is not None - and not struct.union - ): - dt.growStructure(8) - - for field in struct.fields: - if monitor.isCancelled(): - return - - offset = field.offset - dtsize = dt.getLength() if not dt.isZeroLength() else 0 - - ft = self.get_datatype(field.type) - if ft is None: - continue - - if isinstance(field, DefinedStructFixedField): - ft = ArrayDataType(ft, int(field.size), ft.getLength() or -1) - - if not struct.union: - if dtsize <= offset and not struct.size: - dt.growStructure(((offset - dtsize) or 0) + ft.getLength()) - - if ( - dt.getLength() <= offset - or dt.getLength() < offset + ft.getLength() - ): - print( - f"Field {field.name} (off=0x{offset:X} size=0x{ft.getLength():X}) not within Struct {dt.getDataTypePath()} (size=0x{dt.getLength():X})" - ) - break - - dt.replaceAtOffset(offset, ft, -1, field.name, "") - else: - dt.add(ft, ft.getLength(), field.name, "") - - def create_vtable(self, struct): - # type: (DefinedStruct) -> None - if monitor.isCancelled(): - return - dtm = currentProgram.getDataTypeManager() - dt = self.get_datatype(struct.type) - - struct.virtual_functions.sort(key=lambda fn: fn.offset) - vt_type = StructureDataType("VTable", 0) - vt_type.setCategoryPath( - CategoryPath(dt.getCategoryPath(), [dt.getName()]) - ) - vt_type = self.create_datatype(vt_type) - if struct.fields != [] and struct.fields[0].offset == 0: - u_type = UnionDataType("Union") - u_type.setCategoryPath( - CategoryPath(dt.getCategoryPath(), [dt.getName()]) - ) - u_type.add(dtm.getPointer(vt_type), -1, "VTable", "") - comp = dt.getComponentContaining(0) - if comp and not Undefined.isUndefined(comp.getDataType()): - u_type.add( - comp.getDataType(), -1, comp.getFieldName(), "parent class" - ) - self.create_datatype(u_type) - - void_ptr = dtm.getPointer(VoidDataType.dataType) - for func in struct.virtual_functions: - if func.return_type and func.parameters: - func_def = self.create_function_def(func) - func_def.setCategoryPath( - CategoryPath(vt_type.getCategoryPath(), [vt_type.getName()]) - ) - vt_type.insertAtOffset( - func.offset, - dtm.getPointer(func_def), - -1, - func.name, - f"vf{int(func.offset / 8)}", - ) - else: - dtc = vt_type.getComponentAt(func.offset) - if dtc and Undefined.isUndefined(dtc.getDataType()): - vt_type.replaceAtOffset( - func.offset, - void_ptr, - -1, - func.name, - f"vf{int(func.offset / 8)}", - ) - else: - vt_type.insertAtOffset( - func.offset, - void_ptr, - -1, - func.name, - f"vf{int(func.offset / 8)}", - ) - - if struct.vtable_size: - vt_size = struct.vtable_size - vt_type.setLength(vt_size) - else: - vt_size = struct.virtual_functions[-1].offset - for offset in range(0, vt_size, 8): - dtc = vt_type.getComponentContaining(offset) - if not dtc or Undefined.isUndefined(dtc.getDataType()): - vt_type.replaceAtOffset( - offset, void_ptr, -1, f"vf{int(offset / 8)}", None - ) - - def finalise_struct(self, struct): - return - - def create_union(self, struct): - # type: (DefinedStruct) -> None - if monitor.isCancelled() or not struct.virtual_functions: - return - - dtm = currentProgram.getDataTypeManager() - void_ptr = dtm.getPointer(VoidDataType.dataType) - dt = self.get_datatype(struct.type) - u_type = self.get_datatype(struct.type + "::Union") - vt_type = self.get_datatype(struct.type + "::VTable") - - if vt_type: - dtc = dt.getComponentContaining(0) - while dtc and not Undefined.isUndefined(dtc.getDataType()): - if monitor.isCancelled(): - return - parent = dtc.getDataType() - parent_vt = dtm.getDataType( - CategoryPath(parent.getCategoryPath(), [parent.getName()]), - "VTable", - ) - if parent_vt: - if parent_vt.getLength() > vt_type.getLength(): - vt_type.replaceWith(parent_vt) - else: - for c in parent_vt.getComponents(): - if ( - vt_type.getComponentContaining(c.getOffset()) - .getDataType() - .equals(void_ptr) - ): - vt_type.replaceAtOffset( - c.getOffset(), - c.getDataType(), - -1, - c.getFieldName(), - c.getComment(), - ) - dtc = parent_vt.getComponentContaining(0) - else: - break - - if u_type and struct.fields != [] and struct.fields[0].offset == 0: - dt.replaceAtOffset( - 0, u_type, -1, "Union", "vtable and parent union" - ) - elif vt_type: - dt.replaceAtOffset(0, dtm.getPointer(vt_type), -1, "VTable", "") - - def update_member_func(self, member_func, struct): - # type: (DefinedStructMemFunc, DefinedStruct) -> None - if monitor.isCancelled(): - return - if not member_func.parameters: - return - func_name = f"{struct.type}.{member_func.name}" - func = self.get_func_by_name(func_name) - if not func: - return - arg_vars = self.create_memberfunc_args(member_func) - return_type = self.get_datatype(member_func.return_type) - if not return_type: - return - return_var = ReturnParameterImpl(return_type, currentProgram) - update_type = Function.FunctionUpdateType.DYNAMIC_STORAGE_ALL_PARAMS - func.updateFunction( - "__fastcall", - return_var, - arg_vars, - update_type, - False, - SourceType.USER_DEFINED, - ) - - def update_virt_func(self, virt_func, struct): - # type: (DefinedStructVFunc, DefinedStruct) -> None - if monitor.isCancelled(): - return - func_name = f"{struct.type}.{virt_func.name}" - func = self.get_func_by_name(func_name) - if not func: - return - arg_vars = self.create_memberfunc_args(virt_func) - return_type = self.get_datatype(virt_func.return_type) - if not return_type: - return - return_var = ReturnParameterImpl(return_type, currentProgram) - update_type = Function.FunctionUpdateType.DYNAMIC_STORAGE_ALL_PARAMS - func.updateFunction( - "__fastcall", - return_var, - arg_vars, - update_type, - False, - SourceType.USER_DEFINED, - ) - - def update_static_member(self, static_member, struct): - # type: (DefinedStructStaticMember, DefinedStruct) -> None - pass - - def should_update_member_func(self): - # type: () -> bool - return askYesNo("ffxiv_structimporter", "Update member function types?") - - def should_update_virt_func(self): - # type: () -> bool - return askYesNo( - "ffxiv_structimporter", "Update virtual function types?" - ) - - def preprocess_yaml(self, yaml: DefinedStructExport): - return - - api = GhidraApi() - -if api is None: - try: - import binaryninja - import struct - except ImportError: - print("Warning: Unable to load Binary Ninja") - else: - # TODO: VTables, Unions - class BinjaApi(BaseApi): - def can_run(self): - return True - - def get_binja_type(self, name): - # type: (str) -> str - lookup = { - "__int8": "int8_t", - "__int16": "int16_t", - "__int32": "int32_t", - "__int64": "int64_t", - "unsigned __int8": "uint8_t", - "unsigned __int16": "uint16_t", - "unsigned __int32": "uint32_t", - "unsigned __int64": "uint64_t", - "unsigned int": "uint32_t", - "_DWORD": "uint32_t", - "float": "float", - } - if name in lookup: - return lookup[name] - else: - return name - - def get_type(self, name): - # type: (str) -> binaryninja.Type - if name == "__fastcall": - return None - - fixed_name = self.get_binja_type(name) - - pointer_count = 0 - while fixed_name.endswith("*"): - fixed_name = fixed_name[:-1] - pointer_count += 1 - - type = None - - try: - type = bv.parse_type_string(fixed_name)[0] - except: - # Sometimes it just throws. Dunno why - type = bv.types[fixed_name] - - if pointer_count > 0: - type = binaryninja.Type.pointer(type=type, arch=bv.arch) - - return type - - def get_size_from_type(self, name): - # type: (str) -> int - type_obj = self.get_type(name) - if type_obj is None: - return 4 - return type_obj.width - - @property - def get_file_path(self): - # type: () -> str - return os.path.join(os.path.dirname(__file__), "ffxiv_structs.yml") - - def create_enum_struct(self, enum): - # type: (DefinedStructEnum) -> None - members = [] - for value in enum.values: - members.append((value, enum.values[value])) - enum_type = binaryninja.Type.enumeration( - members=members, width=self.get_size_from_type(enum.underlying) - ) - bv.define_user_type(enum.type, enum_type) - - def create_struct(self, struct): - # type: (DefinedStruct) -> None - struct_type = binaryninja.Type.structure( - type=binaryninja.StructureVariant.ClassStructureType - ) - bv.define_user_type(struct.type, struct_type) - - def create_struct_members(self, struct): - # type: (DefinedStruct) -> None - struct_type = bv.types[ - struct.type - ].mutable_copy() # type: binaryninja.StructureBuilder - - for field in struct.fields: - field_type = self.get_type(field.type) - if field_type is None: - continue - - if isinstance(field, DefinedStructFixedField): - field_type = binaryninja.Type.array(field_type, int(field.size)) - struct_type.add_member_at_offset( - field.name, - field_type, - int(field.offset), - ) - - bv.define_user_type(struct.type, struct_type) - - def get_func_ea_by_sig(self, pattern): - # type: (str) -> int - regex = "" - for part in pattern.split(" "): - if part == "??": - regex = regex + "." - else: - regex = regex + "\\x" + part - compiled = re.compile(regex.encode("utf-8")) - - for segment in bv.segments: - data = bv.read(segment.start, segment.end - segment.start) - match = compiled.search(data) - if match: - match_start = match.start() - addr = segment.start + match_start - if data[match_start] == 0xE8 or data[match_start] == 0xE9: - addr += 5 - addr += struct.unpack( - " None - func_name = "{0}.{1}".format(struct.type, member_func.name) - - func = None - symbol = bv.get_symbol_by_raw_name(func_name) - if symbol: - func = bv.get_function_at(symbol.address) - - if not func: - func_addr = self.get_func_ea_by_sig(member_func.signature) - if func_addr is not None: - func = bv.get_function_at(func_addr) - - if not func: - return - - if member_func.return_type != "void": - new_return_type = self.get_type(member_func.return_type) - if new_return_type is not None: - func.return_type = new_return_type - - for i, param in enumerate(member_func.parameters): - if i < len(func.parameter_vars): - param_var = func.parameter_vars[i] - if param_var is None: - continue - - new_param_type = self.get_type(param.type) - if new_param_type is not None: - param_var.type = new_param_type - param_var.name = param.name - - def update_virt_func(self, virt_func, struct): - # type: (DefinedStructVFunc, DefinedStruct) -> None - func_name = "{0}.{1}".format(struct.type, virt_func.name) - - func = None - symbol = bv.get_symbol_by_raw_name(func_name) - if symbol: - func = bv.get_function_at(symbol.address) - - if not func: - return - - if virt_func.return_type != "void": - new_return_type = self.get_type(virt_func.return_type) - if new_return_type is not None: - func.return_type = new_return_type - - for i, param in enumerate(virt_func.parameters): - if i < len(func.parameter_vars): - param_var = func.parameter_vars[i] - if param_var is None: - continue - - new_param_type = self.get_type(param.type) - if new_param_type is not None: - param_var.type = new_param_type - param_var.name = param.name - - def update_static_member(self, static_member, struct): - # type: (DefinedStructStaticMember, DefinedStruct) -> None - pass - - def should_update_member_func(self): - # type: () -> bool - return ( - binaryninja.get_choice_input( - "Update member function types?", - "ffxiv_structimporter", - ["Yes", "No"], - ) - == 0 - ) - - def should_update_virt_func(self): - # type: () -> bool - return ( - binaryninja.get_choice_input( - "Update virtual function types?", - "ffxiv_structimporter", - ["Yes", "No"], - ) - == 0 - ) - - def preprocess_yaml(self, yaml: DefinedStructExport): - return - - api = BinjaApi() - - -if api is None: - raise Exception("Unable to load API (supported: IDA, Ghidra, Binary Ninja)") - -start_time = time() - - -def get_time(): - val = round(time() - start_time, 6).__str__() - while val.split(".")[-1].__len__() < 6: - val += "0" - return val - -def run(): - if not api.can_run(): - raise RuntimeError("This script depends on exdgetters. Run that script before retrying") - - update_virt_func = api.should_update_virt_func() - update_member_func = api.should_update_member_func() - - print("{0} Loading yaml".format(get_time())) - yaml = api.get_yaml() - - print("{0} Loading data yaml".format(get_time())) - api.data_yaml = api.load_data_yaml() - - print("{0} Preprocessing yaml".format(get_time())) - api.preprocess_yaml(yaml) - - print("{0} Deleting old structs".format(get_time())) - for struct in yaml.structs[::-1]: - api.delete_struct(struct) - - print("{0} Deleting old enums and creating new ones".format(get_time())) - for enum in yaml.enums: - api.delete_enum(enum) - - for enum in yaml.enums: - api.create_enum_struct(enum) - - print("{0} Creating new structs".format(get_time())) - for struct in yaml.structs: - api.create_struct(struct) - - print("{0} Creating members for structs".format(get_time())) - for struct in yaml.structs: - api.create_struct_members(struct) - - print("{0} Finalising structs".format(get_time())) - for struct in yaml.structs: - api.finalise_struct(struct) - - print("{0} Creating vtables for structs".format(get_time())) - for struct in yaml.structs: - if struct.virtual_functions: - api.create_vtable(struct) - - print("{0} Mapping unions/vtables for structs".format(get_time())) - for struct in yaml.structs: - api.create_union(struct) - - if update_virt_func: - for struct in yaml.structs: - if struct.virtual_functions: - print( - "{0} Updating virtual functions for {1}".format( - get_time(), struct.type - ) - ) - for virt_func in struct.virtual_functions: - if virt_func is None: - continue - - if virt_func.return_type != None and virt_func.parameters != None: - api.update_virt_func(virt_func, struct) - - if update_member_func: - for struct in yaml.structs: - if struct.member_functions != []: - print( - "{0} Updating member functions for {1}".format( - get_time(), struct.type - ) - ) - for member_func in struct.member_functions: - api.update_member_func(member_func, struct) - - if struct.static_member_functions: - print( - "{0} Updating static member functions for {1}".format( - get_time(), struct.type - ) - ) - for member_func in struct.static_member_functions: - api.update_member_func(member_func, struct) - - if struct.static_members: - print( - "{0} Updating static members for {1}".format( - get_time(), struct.type - ) - ) - for member in struct.static_members: - api.update_static_member(member, struct) - -run() diff --git a/ida/ida_wrapper.py b/ida/ida_wrapper.py deleted file mode 100644 index b1f0795eb3..0000000000 --- a/ida/ida_wrapper.py +++ /dev/null @@ -1,1277 +0,0 @@ -import idaapi -import idc -import ida_bytes -import ida_nalt -import ida_kernwin -import ida_search -import ida_ida -import ida_typeinf -import ida_hexrays -import ida_name -import ida_funcs -from structs_schema import * - -from abc import abstractmethod - -# For more information about TERR_ constants, see: -# https://docs.hex-rays.com/developer-guide/idapython/idapython-porting-guide-ida-9#type-information-error-codes - -class BaseIdaInterface(object): - @abstractmethod - def get_struct_id(self, name): - pass - - @abstractmethod - def get_enum_id(self, name): - pass - - @abstractmethod - def delete_enum_members(self, eid: int): - """Remove all enum members - - Args: - eid (int): The id of the enum - """ - pass - - def enum_exists(self, name: str): - return self.get_enum_id(name) != idaapi.BADADDR - - def get_idc_type_from_ida_type(self, type: str): - """Retrieve the idc type from the ida type. - - Args: - type (str): Name of the type. - """ - if ( - type == "unsigned byte" - or type == "unsigned char" - or type == "unsigned __int8" - or type == "ubyte" - or type == "byte" - or type == "bool" - or type == "char" - or type == "uchar" - or type == "__int8" - ): - return ida_bytes.byte_flag() - elif ( - type == "unsigned short" - or type == "ushort" - or type == "unsigned __int16" - or type == "short" - or type == "__int16" - or type == "_WORD" - or type == "wchar_t" - ): - return ida_bytes.word_flag() - elif ( - type == "unsigned int" - or type == "unsigned __int32" - or type == "uint" - or type == "int" - or type == "__int32" - or type == "_DWORD" - ): - return ida_bytes.dword_flag() - elif ( - type == "unsigned long" - or type == "unsigned __int64" - or type == "ulong" - or type == "long" - or type == "__int64" - or type == "__fastcall" - or type.endswith("*") - ): - return ida_bytes.qword_flag() - elif type == "float": - return ida_bytes.float_flag() - elif type == "double": - return ida_bytes.double_flag() - elif self.get_enum_id(type) != idaapi.BADADDR: - return ida_bytes.enum_flag() - elif self.get_struct_id(type) != idaapi.BADADDR: - return ida_bytes.stru_flag() - else: - return ida_bytes.stru_flag() - - def get_idc_type_from_size(self, size: int, offset=0): - if offset == 0: - offset = size - if offset % 8 == 0 and size >= 8: - return ida_bytes.qword_flag() - elif offset % 4 == 0 and size >= 4: - return ida_bytes.dword_flag() - elif offset % 2 == 0 and size >= 2: - return ida_bytes.word_flag() - else: - return ida_bytes.byte_flag() - - def get_size_from_idc_type(self, type: int): - if type == ida_bytes.byte_flag(): - return 1 - elif type == ida_bytes.word_flag(): - return 2 - elif type == ida_bytes.dword_flag(): - return 4 - elif type == ida_bytes.qword_flag(): - return 8 - elif type == ida_bytes.float_flag(): - return 4 - elif type == ida_bytes.double_flag(): - return 8 - else: - return 0 - - def is_signed(self, type: str): - if ( - type == "__int8" - or type == "__int16" - or type == "__int32" - or type == "__int64" - or type == "int" - ): - return True - else: - return False - - def get_size_from_ida_type(self, type: str): - return self.get_size_from_idc_type(self.get_idc_type_from_ida_type(type)) - - def clean_name(self, name: str): - """Clean a name - - Args: - name (str): The name - - Returns: - str: The cleaned name - """ - return name - - def clean_struct_name(self, name: str): - """Clean a struct name - - Args: - name (str): The struct name - - Returns: - str: The cleaned struct name - """ - - if name == "Tm": - return "tm" # tm is a keyword in IDA for the time struct but C# exports it as Tm - return ( - name.replace(" ", "") - .replace("unsigned", "u") - .replace("__int64", "long") - .replace("__int32", "int") - .replace("__int16", "short") - .replace("__int8", "char") - ) - - def get_named_type(self, name: str): - """Retrieve a tinfo_t from the named type. - - Args: - name (str): Name of the type. - """ - - tinfo = ida_typeinf.tinfo_t() - clean_name = self.clean_struct_name(name) - if ( - self.get_struct_id(clean_name) != idaapi.BADADDR - or self.get_enum_id(clean_name) != idaapi.BADADDR - ): - if not tinfo.get_named_type(idaapi.get_idati(), clean_name): - raise ValueError("{0} not found in IDA database".format(clean_name)) - return tinfo - - if name == "void": - idaapi.parse_decl( - tinfo, idaapi.get_idati(), "void (__fastcall)();", idaapi.PT_SIL - ) - return tinfo.get_rettype() - - terminated = name + ";" - idaapi.parse_decl(tinfo, idaapi.get_idati(), terminated, idaapi.PT_SIL) - - tinfo_str = tinfo.dstr() - if tinfo_str == name or tinfo_str == clean_name: - return tinfo - - terminated = clean_name + ";" - idaapi.parse_decl(tinfo, idaapi.get_idati(), terminated, idaapi.PT_SIL) - - return tinfo - - def get_tinfo_from_type(self, raw_type: str, array_size=0): - """Retrieve a tinfo_t from a raw type string. - - Args: - raw_type (str): Raw type string. - array_size (int, optional): Size of the array. Defaults to 0. - """ - - type = raw_type.rstrip("*") - ptr_count = len(raw_type) - len(type) - - type_tinfo = self.get_named_type(type) - - ptr_tinfo = None - if ptr_count > 0: - for i in range(ptr_count): - ptr_tinfo = idaapi.tinfo_t() - if not ptr_tinfo.create_ptr(type_tinfo): - print("! failed to create pointer") - return None - type_tinfo = ptr_tinfo - else: - ptr_tinfo = type_tinfo - - if array_size > 0: - array_tinfo = idaapi.tinfo_t() - if not array_tinfo.create_array(ptr_tinfo, array_size): - print("! failed to create array") - return None - - ptr_tinfo = array_tinfo - - return ptr_tinfo - - def rename_struct(self, sid: int, new_name: str): - """Rename a struct in IDA. - - Args: - sid (int): The struct ID. - new_name (str): The new struct name. - """ - idc.set_struc_name(sid, new_name) - - -if idaapi.IDA_SDK_VERSION < 900: - import ida_struct # pyright: ignore[reportMissingImports] - import ida_enum # pyright: ignore[reportMissingImports] -else: - print("Using IDA 9+ API") - -class IdaInterface(BaseIdaInterface): - # This is only for IDA 7 and 8 due to a change in the API for IDA 9 - if idaapi.IDA_SDK_VERSION < 900 and idaapi.IDA_SDK_VERSION >= 700: - - def get_tinfo_from_func_data(self, data: DefinedStructFuncField): - """Retrieve a tinfo_t from a raw function data. - - Args: - data (DefinedStructFuncField): Function data. - - Returns: - idaapi.tinfo_t: tinfo_t created from the function data. - """ - tinfo = ida_typeinf.tinfo_t() - func_data = ida_typeinf.func_type_data_t() - func_data.cc = ida_typeinf.CM_CC_FASTCALL - func_data.rettype = self.get_tinfo_from_type(data.return_type) - for param in data.parameters: - arg = ida_typeinf.funcarg_t() - arg.type = self.get_tinfo_from_type(param.type) - arg.name = param.name - func_data.push_back(arg) - - tinfo.create_func(func_data) - return tinfo - - def get_struct_opinfo_from_type(self, raw_type: str): - """Retrieve an opinfo_t from a raw structure type string. - - Args: - raw_type (str): Raw structure type string. - - Returns: - ida_nalt.opinfo_t: opinfo_t created from the structure type string. - """ - opinf = ida_nalt.opinfo_t() - opinf.tid = ida_struct.get_struc_id(raw_type) - return opinf - - def get_enum_opinfo_from_type(self, raw_type: str): - """Retrieve an opinfo_t from a raw enum type string. - - Args: - raw_type (str): Raw enum type string. - - Returns: - ida_nalt.opinfo_t: opinfo_t created from the enum type string. - """ - opinf = ida_nalt.opinfo_t() - opinf.ec.tid = ida_enum.get_enum(raw_type) - return opinf - - def search_binary(self, ea: int, pattern: str, flag: int): - """Search for a pattern in a binary - - Args: - ea (int): The address to start searching from - pattern (str): The pattern to search for - flag (int): The search flags - - Returns: - int: The start address of the pattern - """ - return ida_search.find_binary( - ea, - flag & 1 and ida_ida.cvar.inf.max_ea or ida_ida.cvar.inf.min_ea, - pattern, - 16, - flag, - ) - - def get_dword(self, ea: int): - """Retrieve a dword (32-bit value) from the specified address. - - Args: - ea (int): The effective address to read the dword from. - - Returns: - int: The dword value read from the given address. - """ - return ida_bytes.get_original_dword(ea) - - def get_func_ea_by_name(self, name: str): - """Retrieve the effective address of a function by its name. - - Args: - name (str): The name of the function. - - Returns: - int: The effective address of the function. - """ - return ida_name.get_name_ea(0, name) - - def get_func_ea_by_sig(self, pattern: str): - """Retrieve the effective address of a function by its signature. - - Args: - pattern (str): The signature of the function. - - Returns: - int: The effective address of the function. - """ - ea = self.search_binary(0, pattern, ida_search.SEARCH_DOWN) - - if ida_funcs.get_func(ea) is None: - finf = ida_funcs.func_t() - finf.start_ea = ea - finf.end_ea = idc.BADADDR - ida_funcs.add_func_ex(finf) - - if ida_funcs.get_func(ea) is None: - return idc.BADADDR - - if ida_funcs.get_func(ea).start_ea == ea: - return ea - mnem = idc.print_insn_mnem(ea) - if not mnem: - return idc.BADADDR - - opType0 = idc.get_operand_type(ea, 0) - if mnem == "jmp" or mnem == "call" or mnem[0] == "j": - if opType0 != idc.o_near and opType0 != idc.o_mem: - print( - "Error: Can't follow opType0 {0}".format( - self.opTypeAsName(opType0) - ) - ) - return idc.BADADDR - return idc.get_operand_value(ea, 0) - - if idc.next_head(ea) == ea + idc.get_item_size(ea) and idc.is_flow( - idc.get_full_flags(idc.next_head(ea)) - ): - return idc.next_head(ea) - - return idc.BADADDR - - def opTypeAsName(self, n: int): - """ - Retrieve the name of the operand type constant. - - Args: - n (int): The integer value of the operand type. - - Returns: - str: The name of the operand type constant if found; otherwise, None. - """ - - for item in [x for x in dir(idc) if x.startswith("o_")]: - if getattr(idc, item) == n: - return item - - def create_struct_type(self, name: str, union: bool = False) -> int: - """Create a struct type - - Args: - name (str): The name of the struct - union (bool, optional): Whether the struct is a union. Defaults to False. - - Returns: - int: The struct id - """ - return ida_struct.add_struc(idc.BADADDR, name, union) - - def get_struct_id(self, name: str) -> int: - """Get the struct id - - Args: - name (str): The name of the struct - - Returns: - int: The struct id - """ - return ida_struct.get_struc_id(name) - - def get_struct(self, sid: int) -> ida_struct.struc_t: - """Get the struct - - Args: - sid (int): The struct id - - Returns: - ida_struct.struc_t: The struct - """ - return ida_struct.get_struc(sid) - - def get_struct_size(self, sid: ida_struct.struc_t) -> int: - """Get the struct size - - Args: - sid (ida_struct.struc_t): The struct - - Returns: - int: The struct size - """ - return ida_struct.get_struc_size(sid) - - def create_struct_member( - self, - sid: ida_struct.struc_t, - name: str, - offset: int = -1, - flag: int = idc.FF_DATA | idc.FF_QWORD, - typeid: int | None = None, - nbytes: int = 8, - ): - """Create a struct member - - Args: - sid (ida_struct.struc_t): The struct to add the member to - name (str): The name of the member - offset (int, optional): The offset of the member. Defaults to -1. - flag (int, optional): The type flag of the member. Defaults to idc.FF_DATA | idc.FF_QWORD. - typeid (int | None, optional): The type id of the member. Defaults to None. - nbytes (int, optional): The number of bytes of the member. Defaults to 8. - - Returns: - struc_error_t: Unknown IDA documentation for this - """ - return ida_struct.add_struc_member(sid, name, offset, flag, typeid, nbytes) - - def remove_struct_member(self, sid: int, name: str) -> bool: - """Remove a struct member - - Args: - sid (int): The struct id - name (str): The name of the member - - Returns: - bool: True if successful - """ - return ida_struct.del_struc_member(sid, name) - - def remove_struct_members(self, sid: int) -> int: - """Remove all struct members - - Args: - sid (int): The struct id - - Returns: - int: The number of members removed or -1 if failed - """ - return ida_struct.del_struc_members( - ida_struct.get_struc(sid), - 0, - ida_struct.get_struc_size(sid) + 1, - ) - - def get_struct_member( - self, sid: ida_struct.struc_t, offset: int - ) -> ida_struct.member_t: - """Get a struct member - - Args: - sid (ida_struct.struc_t): The struct - offset (int): The offset of the member - - Returns: - ida_struct.member_t: The member data - """ - return ida_struct.get_member(sid, offset) - - def get_struct_member_by_name( - self, sid: ida_struct.struc_t, name: str - ) -> ida_struct.member_t: - """Get a struct member - - Args: - sid (ida_struct.struc_t): The struct - name (str): The name of the member - - Returns: - ida_struct.member_t: The member data - """ - return ida_struct.get_member_by_name(sid, name) - - def set_struct_member_info( - self, - sid: ida_struct.struc_t, - member: ida_struct.member_t, - offset: int, - tif: ida_typeinf.tinfo_t, - flag: int = 0, - is_string: bool = False - ): - """Set the info of a struct member - - Args: - sid (ida_struct.struc_t): The struct - member (ida_struct.member_t): The member - offset (int): The offset in the member - tif (ida_typeinf.tinfo_t): The type info - flag (int, optional): The flag of the member type. Defaults to 0. - - Returns: - smt_code_t: Unknown IDA documentation - """ - set_mem_ret = ida_struct.set_member_tinfo(sid, member, offset, tif, flag) - if is_string: - member.flag = member.flag | ida_bytes.FF_STRLIT - return set_mem_ret - - def get_struct_member_id(self, sid: ida_struct.struc_t, offset: int) -> int: - """Get the member id - - Args: - sid (ida_struct.struc_t): The struct - offset (int): The offset of the member in the struct - - Returns: - int: The member id inside of the struct - """ - return ida_struct.get_member_id(sid, offset) - - def get_base_class_flag(self): - """Get the base class flag - - Returns: - int: The flag for the base class type - """ - return ida_struct.MF_BASECLASS - - def get_enum_id(self, name: str): - """Get the id of an enum by its name - - Args: - name (str): The name of the enum - - Returns: - int: The id of the enum - """ - return ida_enum.get_enum(name) - - def remove_enum_member(self, eid: int, value: str, name: str): - """Remove an enum member by its value and name - - Args: - eid (int): The id of the enum - value (str): The value of the enum member - name (str): The name of the enum member - """ - mem = ida_enum.get_enum_member_by_name("{0}.{1}".format(name, value)) - ida_enum.del_enum_member( - eid, - ida_enum.get_enum_member_value(mem), - ida_enum.get_enum_member_serial(mem), - ida_enum.get_enum_member_bmask(mem), - ) - - def delete_enum_members(self, eid: int): - """Remove all enum members - - Args: - eid (int): The id of the enum - """ - - masks = [ -1 ] - if idc.is_bf(eid): - mask = idc.get_first_bmask(eid) - while mask != idaapi.DEFMASK: - masks.append(mask) - mask = idc.get_next_bmask(eid, mask) - - members = [] - for mask in masks: - mem_val = idc.get_first_enum_member(eid, mask) - while mem_val != idaapi.BADNODE: - cid = ida_enum.get_first_serial_enum_member(eid, mem_val, mask) - members.append(cid[0]) - mem_val = idc.get_next_enum_member(eid, mem_val, mask) - - for cid in members: - ida_enum.del_enum_member( - eid, - ida_enum.get_enum_member_value(cid), - ida_enum.get_enum_member_serial(cid), - ida_enum.get_enum_member_bmask(cid) - ) - - def create_enum(self, name: str) -> int: - """Create an enum by its name - - Args: - name (str): The name of the enum - - Returns: - int: The id of the added enum - """ - return ida_enum.add_enum(idc.BADADDR, name, 0) - - def set_enum_width(self, eid: int, width: int): - """Set the width of an enum by its id - - Args: - eid (int): The id of the enum - width (int): The width of the enum - """ - ida_enum.set_enum_width(eid, width) - - def set_enum_flag(self, eid: int, flag: int): - """Set a flag on an enum by its id - - Args: - eid (int): The id of the enum - flag (int): The flag to set - """ - ida_enum.set_enum_flag(eid, flag) - - def get_enum_default_mask(self, eid: int): - """Get the default bitmask for an enum - - Args: - eid (int): The id of the enum - - Returns: - int: The default bitmask for the enum - """ - width = idc.get_enum_width(eid) - mask = (1 << (width * 8)) - 1 - - if mask == ida_enum.DEFMASK: - mask >>= 1 - - return mask - - def get_enum_name(self, eid: int): - return idc.get_enum_name(eid) - - def get_enum_bf(self, eid: int): - return ida_enum.is_bf(eid) - - def set_enum_as_bf(self, eid: int): - ida_enum.set_enum_bf(eid, True) - - name = self.get_enum_name(eid) - bmask = self.get_enum_default_mask(eid) - - # this shouldn't happen under normal circumstances - if ida_enum.get_bmask_name(eid, bmask) == f"{name}_Mask": - return - - ida_enum.set_bmask_name(eid, bmask, f"{name}_Mask") - - def add_enum_member(self, eid: int, name: str, value: int, mask: int = -1): - """Add an enum member to an enum by its id - Args: - eid (int): The id of the enum - name (str): The name of the enum member - value (int): The value of the enum member - mask (int): The bitmask of the enum member, or -1 to use the default mask - """ - if mask == -1 and self.get_enum_bf(eid): - mask = self.get_enum_default_mask(eid) - - # print(f"Adding value: {value} with name: {name} and mask: {mask}") - ec = ida_enum.add_enum_member(eid, name, value, mask) - # print(f"Got error code: {ec}") - - if ec == ida_enum.ENUM_MEMBER_ERROR_MASK: - ida_enum.add_enum_member(eid, name, value) - - def get_struct_flag(self): - """Get the flag for a struct data type - - Returns: - int: The flag for a struct data type - """ - return ida_bytes.stru_flag() - - def get_enum_flag(self): - """Get the flag for an enum data type - - Returns: - int: The flag for an enum data type - """ - return ida_bytes.enum_flag() - - elif idaapi.IDA_SDK_VERSION >= 900: - - def get_tinfo_from_func_data(self, data: DefinedStructFuncField): - """Retrieve a tinfo_t from a raw function data. - - Args: - data (DefinedStructFuncField): Function data. - - Returns: - idaapi.tinfo_t: tinfo_t created from the function data. - """ - tinfo = ida_typeinf.tinfo_t() - func_data = ida_typeinf.func_type_data_t() - func_data.cc = ida_typeinf.CM_CC_FASTCALL - func_data.rettype = self.get_tinfo_from_type(data.return_type) - for param in data.parameters: - arg = ida_typeinf.funcarg_t() - arg.type = self.get_tinfo_from_type(param.type) - arg.name = param.name - func_data.push_back(arg) - - tinfo.create_func(func_data) - return tinfo - - def get_struct_opinfo_from_type(self, raw_type: str): - """Retrieve an opinfo_t from a raw structure type string. - - Args: - raw_type (str): Raw structure type string. - - Returns: - ida_nalt.opinfo_t: opinfo_t created from the structure type string. - """ - opinf = ida_nalt.opinfo_t() - opinf.tid = ida_typeinf.get_named_type_tid(raw_type) - - return opinf - - def get_enum_opinfo_from_type(self, raw_type: str): - """Retrieve an opinfo_t from a raw enum type string. - - Args: - raw_type (str): Raw enum type string. - - Returns: - ida_nalt.opinfo_t: opinfo_t created from the enum type string. - """ - opinf = ida_nalt.opinfo_t() - opinf.ec.tid = idc.get_enum(raw_type) - - return opinf - - def search_binary(self, ea: int, pattern: str, flag: int): - """Search for a pattern in a binary - - Args: - ea (int): The address to start searching from - pattern (str): The pattern to search for - flag (int): The search flags - - Returns: - int: The start address of the pattern - """ - return ida_bytes.find_bytes( - pattern, ea, None, ida_ida.inf_get_max_ea(), None, flag - ) - - def get_dword(self, ea: int): - """Retrieve a dword (32-bit value) from the specified address. - - Args: - ea (int): The effective address to read the dword from. - - Returns: - int: The dword value read from the given address. - """ - return ida_bytes.get_original_dword(ea) - - def get_func_ea_by_name(self, name: str): - """Retrieve the effective address of a function by its name. - - Args: - name (str): The name of the function. - - Returns: - int: The effective address of the function. - """ - return ida_name.get_name_ea(0, name) - - def get_func_ea_by_sig(self, pattern: str): - """Retrieve the effective address of a function by its signature. - - Args: - pattern (str): The signature of the function. - - Returns: - int: The effective address of the function. - """ - ea = self.search_binary(0, pattern, ida_search.SEARCH_DOWN) - - if ida_funcs.get_func(ea) is None: - finf = ida_funcs.func_t() - finf.start_ea = ea - finf.end_ea = idc.BADADDR - ida_funcs.add_func_ex(finf) - - if ida_funcs.get_func(ea) is None: - return idc.BADADDR - - if ida_funcs.get_func(ea).start_ea == ea: - return ea - mnem = idc.print_insn_mnem(ea) - if not mnem: - return idc.BADADDR - - opType0 = idc.get_operand_type(ea, 0) - if mnem == "jmp" or mnem == "call" or mnem[0] == "j": - if opType0 != idc.o_near and opType0 != idc.o_mem: - print( - "Error: Can't follow opType0 {0}".format( - self.opTypeAsName(opType0) - ) - ) - return idc.BADADDR - return idc.get_operand_value(ea, 0) - - if idc.next_head(ea) == ea + idc.get_item_size(ea) and idc.is_flow( - idc.get_full_flags(idc.next_head(ea)) - ): - return idc.next_head(ea) - - return idc.BADADDR - - def opTypeAsName(self, n: int): - """ - Retrieve the name of the operand type constant. - - Args: - n (int): The integer value of the operand type. - - Returns: - str: The name of the operand type constant if found; otherwise, None. - """ - - for item in [x for x in dir(idc) if x.startswith("o_")]: - if getattr(idc, item) == n: - return item - - def create_struct_type(self, name: str, union: bool = False) -> int: - """Create a struct type - - Args: - name (str): The name of the struct - union (bool, optional): Whether the struct is a union. Defaults to False. - - Returns: - int: The struct id - """ - - udt = ida_typeinf.udt_type_data_t() - udt.is_union = union - - tinfo = ida_typeinf.tinfo_t() - if not tinfo.create_udt(udt): - return idc.BADADDR - - if tinfo.set_named_type(None, name) != ida_typeinf.TERR_OK: - return idc.BADADDR - - return tinfo.get_tid() - - def get_struct_id(self, name: str) -> int: - """Get the struct id - - Args: - name (str): The name of the struct - - Returns: - int: The struct id - """ - return ida_typeinf.get_named_type_tid(name) - - def get_struct(self, sid: int) -> ida_typeinf.tinfo_t: - """Get the struct - - Args: - sid (int): The struct id - - Returns: - ida_typeinf.tinfo_t: The struct's parent tinfo_t - """ - return ida_typeinf.tinfo_t(tid=sid) - - def get_struct_size(self, sid: ida_typeinf.tinfo_t | int) -> int: - """Get the struct size - - Args: - sid (ida_typeinf.tinfo_t): The struct - - Returns: - int: The struct size - """ - if isinstance(sid, int): - sid = self.get_struct(sid) - return sid.get_size() - - def create_struct_member( - self, - sid: ida_typeinf.tinfo_t, - name: str, - offset: int = -1, - flag: int = idc.FF_DATA | idc.FF_QWORD, - typeid: int | idaapi.opinfo_t | None = None, - nbytes: int = 8, - ): - """Create a struct member - - Args: - sid (ida_typeinf.tinfo_t): The struct to add the member to - name (str): The name of the member - offset (int, optional): The offset of the member. Defaults to -1. - flag (int, optional): The type flag of the member. Defaults to idc.FF_DATA | idc.FF_QWORD. - typeid (int | None, optional): The type id of the member. Defaults to None. - nbytes (int, optional): The number of bytes of the member. Defaults to 8. - - Returns: - bool: True if successful - """ - - if typeid is None: - typeid = -1 - - if isinstance(typeid, idaapi.opinfo_t): - tinfo = ida_typeinf.tinfo_t() - if tinfo.get_type_by_tid(typeid.tid): - nbytes = tinfo.get_size() - else: - raise ValueError("Cannot find type with tid {0}".format(typeid.tid)) - - typeid = typeid.tid - - ec = idc.add_struc_member(sid.get_tid(), name, offset, flag, typeid, nbytes) - if ec != ida_typeinf.TERR_OK: - return False - - return True - - def remove_struct_member(self, sid: int, name: str) -> bool: - """Remove a struct member - - Args: - sid (int): The struct id - name (str): The name of the member - - Returns: - tinfo_code_t: See ida_typeinfo.TERR_ constants - """ - try: - tinfo = ida_typeinf.tinfo_t(tid=sid) - idx = tinfo.find_udm(name=name) - if idx == -1: - return True - - return tinfo.del_udm(idx) - except: - return ida_typeinf.TERR_BAD_ARG - - def remove_struct_members(self, sid: int) -> int: - """Remove all struct members - - Args: - sid (int): The struct id - - Returns: - int: The number of members removed or -1 if failed - """ - try: - tinfo = ida_typeinf.tinfo_t() - if not tinfo.get_type_by_tid(sid): - return -1 - - size = tinfo.get_udt_nmembers() - tinfo.del_udms(0, size) - return size - except: - return -1 - - def get_struct_member( - self, sid: ida_typeinf.tinfo_t, offset: int - ) -> ida_typeinf.udm_t: - """Get a struct member - - Args: - sid (ida_typeinf.tinfo_t): The struct - offset (int): The offset of the member - - Returns: - ida_typeinf.udm_t: The member data - """ - _, udm = sid.get_udm_by_offset(offset) - return udm - - def get_struct_member_by_name( - self, sid: ida_typeinf.tinfo_t, name: str - ) -> ida_typeinf.udm_t: - """Get a struct member - - Args: - sid (ida_typeinf.tinfo_t): The struct - name (str): The name of the member - - Returns: - ida_typeinf.udm_t: The member data - """ - _, udm = sid.get_udm(name) - return udm - - def set_struct_member_info( - self, - sid: ida_typeinf.tinfo_t, - member: ida_typeinf.udm_t, - offset: int, - tif: ida_typeinf.tinfo_t, - flag: int = 0, - is_string: bool = False - ): - """Set the info of a struct member - - Args: - sid (ida_typeinf.tinfo_t): The struct - member (ida_typeinf.udm_t): The member - offset (int): The offset in the member - tif (ida_typeinf.tinfo_t): The type info - flag (int, optional): The flag of the member type. Defaults to 0. - - Returns: - tinfo_code_t: See ida_typeinfo.TERR_ constants - """ - if offset != 0: - raise ValueError("IDA 9+ does not support offset != 0 in set_struct_member_info") - - memberIdx, _ = sid.get_udm_by_offset(member.offset) - return sid.set_udm_type(index=memberIdx, tif=tif) - - def get_struct_member_id(self, sid: ida_typeinf.tinfo_t, offset: int) -> int: - """Get the member id - - Args: - sid (ida_typeinf.tinfo_t): The struct - offset (int): The offset of the member in the struct - - Returns: - int: The member id inside of the struct or idc.BADADDR if not found - """ - idx, _ = sid.get_udm_by_offset(offset * 8) - return idc.BADADDR if idx == -1 else idx - - def get_enum_id(self, name: str): - """Get the id of an enum by its name - - Args: - name (str): The name of the enum - - Returns: - int: The id of the enum - """ - - return idc.get_enum(name) - - def remove_enum_member(self, eid: int, value: str, name: str): - """Remove an enum member by its value and name - - Args: - eid (int): The id of the enum - value (str): The value of the enum member - name (str): The name of the enum member - """ - - mem = idc.get_enum_member_by_name("{0}.{1}".format(name, value)) - if mem != idc.BADADDR and idc.get_enum_member_value(mem) != None: - idc.del_enum_member( - eid, - idc.get_enum_member_value(mem), - 0, - idc.get_enum_member_bmask(mem) or -1, - ) - - def delete_enum_members(self, eid: int): - """Remove all enum members - - Args: - eid (int): The id of the enum - """ - tif = ida_typeinf.tinfo_t() - if not tif.get_type_by_tid(eid): - raise RuntimeError(f'Failed to get tinfo_t for enum {eid}') - - eti = ida_typeinf.enum_type_data_t() - if not tif.get_enum_details(eti): - raise RuntimeError(f'Failed to get enum details for enum id {eid}') - - # must clear values before groups - - values = [] - for (idx, _, _) in eti.all_constants(): - values.append(eti[idx].name) - - for name in values: - tif.del_edm(name) - - groups = [] - for (idx, _) in eti.all_groups(): - groups.append(eti[idx].name) - - for name in groups: - tif.del_edm(name) - - def create_enum(self, name: str) -> int: - """Create an enum by its name - - Args: - name (str): The name of the enum - - Returns: - int: The id of the added enum - """ - return idc.add_enum(0, name, 0) - - def set_enum_width(self, eid: int, width: int): - """Set the width of an enum by its id - - Args: - eid (int): The id of the enum - width (int): The width of the enum - """ - # TODO(caitlyn): handle changing the width of a bitfield enum - - idc.set_enum_width(eid, width) - - def get_enum_default_mask(self, eid: int): - """Get the default bitmask for an enum - - Args: - eid (int): The id of the enum - - Returns: - int: The default bitmask for the enum - """ - width = idc.get_enum_width(eid) - mask = (1 << (width * 8)) - 1 - - # IDA 9+ has an issue wherein the default mask for 64-bit - # enums is equal to DEFMASK64. This causes issues when - # creating and assigning members to the default group. - # - # To work around this, we drop the highest order bit - # from the default group for 64-bit enums. - if mask == ida_typeinf.DEFMASK64: - mask >>= 1 - - return mask - - def set_enum_flag(self, eid: int, flag: int): - """Set a flag on an enum by its id - - Args: - eid (int): The id of the enum - flag (int): The flag to set - """ - idc.set_enum_flag(eid, flag) - - def get_enum_bf(self, eid: int): - return idc.is_bf(eid) - - def get_enum_name(self, eid: int): - return idc.get_enum_name(eid) - - def get_enum_bitmask_field(self, eid: int): - name = self.get_enum_name(eid) - bmask = self.get_enum_default_mask(eid) - - return (f"{name}_Mask", bmask) - - def set_enum_as_bf(self, eid: int): - if idc.is_bf(eid): - return - - idc.set_enum_bf(eid, True) - - (name, bmask) = self.get_enum_bitmask_field(eid) - - # this shouldn't happen under normal circumstances - if idc.get_enum_member_by_name(f"{name}_Mask") != idaapi.BADADDR: - return - - idc.add_enum_member(eid, f"{name}_Mask", bmask, bmask) - - def add_enum_member(self, eid: int, name: str, value: int, mask: int = -1): - """Add an enum member to an enum by its id - Args: - eid (int): The id of the enum - name (str): The name of the enum member - value (int): The value of the enum member - mask (int): The bitmask of the enum member, or -1 to use the default mask - """ - - if mask == -1 and idc.is_bf(eid): - mask = self.get_enum_default_mask(eid) - - # IDA 9+ requires all enum member names be unique across the DB. - # We attempt to avoid collisions by appending up to 3 underscores to the name. - retries = 0 - processedName = name - while idc.get_enum_member_by_name(processedName) != idaapi.BADADDR: - if retries >= 3: - raise RuntimeError(f"Error: too many duplicate enum member names for '{name}'") - retries += 1 - processedName += "_" - - # IDA 9.0 - 9.2 vary in how they handle errors. - # IDA 9.0 will return a TERR_ code, while IDA 9.2 will raise an exception. - # This will effectively convert the TERR_ to an equivalent ValueError for 9.0. - ec = idc.add_enum_member(eid, processedName, value, mask) - if ec != ida_typeinf.TERR_OK: - raise ValueError(ida_typeinf.tinfo_errstr(ec)) - - def get_struct_flag(self): - """Get the flag for a struct data type - - Returns: - int: The flag for a struct data type - """ - return ida_bytes.stru_flag() - - def get_enum_flag(self): - """Get the flag for an enum data type - - Returns: - int: The flag for an enum data type - """ - return ida_bytes.enum_flag() - else: - raise RuntimeError("Unsupported IDA version") diff --git a/ida/luminapie/definitions.py b/ida/luminapie/definitions.py deleted file mode 100644 index a2f9342da5..0000000000 --- a/ida/luminapie/definitions.py +++ /dev/null @@ -1,110 +0,0 @@ -def get_definition(schema): - # type: (dict[str, str]) -> Definition - if "type" in schema: - if schema["type"] == "array": - return RepeatDefinition(schema) - return Definition(schema) - - -class Definition: - def __init__(self, obj): - # type: (dict[str, str]) -> None - self.name = obj["name"] - if "pendingName" in obj: - self.pendingName = obj["pendingName"] - - def get_name(self): - # type: () -> str - if hasattr(self, "pendingName"): - return self.pendingName - else: - return self.name - - def __repr__(self): - # type: () -> str - return self.name - - -class RepeatDefinition(Definition): - def __init__(self, obj): - # type: (dict[str, str]) -> None - super().__init__(obj) - self.obj = obj - self.count = obj["count"] - self.inner_defs = [] - self.process_inner() - - def process_inner(self): - # type: () -> None - if "fields" in self.obj: - for field in self.obj["fields"]: - if "name" in field: - self.inner_defs.append(get_definition(field)) - else: - self.inner_defs.append(Definition({"name": ""})) - if self.inner_defs == []: - self.inner_defs.append(Definition({"name": ""})) - - def flatten(self, extern): - # type: (str) -> list[Definition] - defs = [] - extern = extern + self.name - for i in range(0, int(self.count)): - for inner in self.inner_defs: - if isinstance(inner, RepeatDefinition): - defs.extend(inner.flatten(extern + i.__str__())) - else: - defs.append(Definition({"name": extern + i.__str__() + inner.name})) - return defs - - def __repr__(self): - # type: () -> str - return self.flatten("") - - -class SemanticVersion: - """Represents a semantic version string that can compare versions""" - - def __init__(self, year, month, date, patch, build=0): - # type: (int, int, int, int, int) -> None - self.year = year - self.month = month - self.date = date - self.patch = patch - self.build = build - - def __lt__(self, other): - # type: (SemanticVersion) -> bool - return ( - self.year < other.year - or self.month < other.month - or self.date < other.date - or self.patch < other.patch - or self.build < other.build - ) - - def __repr__(self): - # type: () -> str - return "{0}.{1}.{2}.{3}.{4}".format( - self.year, - self.month.__str__().rjust(2, "0"), - self.date.__str__().rjust(2, "0"), - self.patch.__str__().rjust(4, "0"), - self.build.__str__().rjust(4, "0"), - ) - - def __eq__(self, __value): - # type: (object) -> bool - if not isinstance(__value, SemanticVersion): - return False - return ( - self.year == __value.year - and self.month == __value.month - and self.date == __value.date - and self.patch == __value.patch - and self.build == __value.build - ) - - def __hash__(self): - # type: () -> int - return hash(repr(self)) diff --git a/ida/luminapie/enums.py b/ida/luminapie/enums.py deleted file mode 100644 index 44b90a4ed8..0000000000 --- a/ida/luminapie/enums.py +++ /dev/null @@ -1,64 +0,0 @@ -import enum - - -class SqPackCatergories(enum.IntEnum): - COMMON = 0x0 - BGCOMMON = 0x1 - BG = 0x2 - CUT = 0x3 - CHARA = 0x4 - SHADER = 0x5 - UI = 0x6 - SOUND = 0x7 - VFX = 0x8 - UI_SCRIPT = 0x9 - EXD = 0xA - GAME_SCRIPT = 0xB - MUSIC = 0xC - SQPACK_TEST = 0x12 - DEBUG = 0x13 - - -class SqPackPlatformId(enum.IntEnum): - Win32 = 0x0 - PS3 = 0x1 - PS4 = 0x2 - - -class SqPackFileType(enum.IntEnum): - Empty = 1 - Standard = 2 - Model = 3 - Texture = 4 - - -class DatBlockType(enum.IntEnum): - Compressed = 4713 - Uncompressed = 32000 - - -class ExcelColumnDataType(enum.IntEnum): - String = 0x0 - Bool = 0x1 - Int8 = 0x2 - UInt8 = 0x3 - Int16 = 0x4 - UInt16 = 0x5 - Int32 = 0x6 - UInt32 = 0x7 - # unused? - Unk = 0x8 - Float32 = 0x9 - Int64 = 0xA - UInt64 = 0xB - # unused? - Unk2 = 0xC - # 0 is read like data & 1, 1 is like data & 2, 2 = data & 4, etc... - PackedBool0 = 0x19 - PackedBool1 = 0x1A - PackedBool2 = 0x1B - PackedBool3 = 0x1C - PackedBool4 = 0x1D - PackedBool5 = 0x1E - PackedBool6 = 0x1F - PackedBool7 = 0x20 diff --git a/ida/luminapie/excel.py b/ida/luminapie/excel.py deleted file mode 100644 index 9e77754f92..0000000000 --- a/ida/luminapie/excel.py +++ /dev/null @@ -1,305 +0,0 @@ -from luminapie.enums import ExcelColumnDataType -from luminapie.definitions import Definition - - -class ExcelListFile: - def __init__(self, data): - # type: (list[bytes]) -> None - self.data = b"".join(data).split("\r\n".encode("utf-8")) - self.parse() - - def parse(self): - # type: () -> None - self.header = self.data[0].decode("utf-8").split(",") - self.version = int(self.header[1]) - self.data = self.data[1:] - self.dict: dict[int, str] = {} - for line in [x.decode("utf-8") for x in self.data]: - if line == "": - continue - linearr = line.split(",") - if linearr[1] == "-1": - continue - self.dict[int(linearr[1])] = linearr[0] - - def __repr__(self): - # type: () -> str - return "ExcelListFile: {0}, {1}".format(self.header, self.dict) - - -class ExcelHeader: - def __init__(self, data): - # type: (bytes) -> None - self.data = data - self.parse() - - def parse(self): - # type: () -> None - self.magic = self.data[0:4] - self.version = int.from_bytes(self.data[4:6], "big") - self.data_offset = int.from_bytes(self.data[6:8], "big") - self.column_count = int.from_bytes(self.data[8:10], "big") - self.page_count = int.from_bytes(self.data[10:12], "big") - self.language_count = int.from_bytes(self.data[12:14], "big") - self.unknown1 = int.from_bytes(self.data[14:16], "big") - self.unknown2 = self.data[17] - self.variant = self.data[18] - self.unknown3 = int.from_bytes(self.data[19:20], "big") - self.row_count = int.from_bytes(self.data[20:24], "big") - self.unknown4 = [ - int.from_bytes(self.data[24:28], "big"), - int.from_bytes(self.data[28:32], "big"), - ] - - def __repr__(self): - # type: () -> str - return "Header: {0}, version: {1}, data_offset: {2}, column_count: {3}, page_count: {4}, language_count: {5}, unknown1: {6}, unknown2: {7}, variant: {8}, unknown3: {9}, row_count: {10}, unknown4: {11}".format( - self.magic, - self.version, - self.data_offset, - self.column_count, - self.page_count, - self.language_count, - self.unknown1, - self.unknown2, - self.variant, - self.unknown3, - self.row_count, - self.unknown4, - ) - - -class ExcelColumnDefinition: - def __init__(self, data): - # type: (bytes) -> None - self.data = data - self.parse() - - def parse(self): - # type: () -> None - self.type = ExcelColumnDataType(int.from_bytes(self.data[0:2], "big")) - self.offset = int.from_bytes(self.data[2:4], "big") - - def __lt__(self, other): - # type: (ExcelColumnDefinition) -> bool - return self.offset < other.offset - - def __eq__(self, other): - return self.offset == other.offset and self.type == other.offset - - def __repr__(self): - # type: () -> str - return "Column: {0}, offset: {1}".format(self.type.name, self.offset) - - -class ExcelDataPagination: - def __init__(self, data): - # type: (bytes) -> None - self.data = data - self.parse() - - def parse(self): - # type: () -> None - self.start_id = int.from_bytes(self.data[0:2], "big") - self.row_count = int.from_bytes(self.data[2:4], "big") - - def __repr__(self): - # type: () -> str - return "Pagination: {0:x}, count: {1}".format(self.start_id, self.row_count) - - -class ExcelHeaderFile: - def __init__(self, data, name): - # type: (list[bytes], str) -> None - self.data = data[0] - self.column_definitions: list[ExcelColumnDefinition] = [] - self.pagination: list[ExcelDataPagination] = [] - self.languages: list[int] = [] - self.header: ExcelHeader = None - self.name = name - self.parse() - - def parse(self): - # type: () -> None - self.header = ExcelHeader(self.data[0:32]) - if self.header.magic != b"EXHF": - raise Exception("Invalid EXHF header") - self.column_definitions: list[ExcelColumnDefinition] = [] - for i in range(self.header.column_count): - self.column_definitions.append( - ExcelColumnDefinition(self.data[32 + (i * 4) : 32 + ((i + 1) * 4)]) - ) - self.column_definitions = sorted(self.column_definitions) - self.pagination: list[ExcelDataPagination] = [] - for i in range(self.header.page_count): - self.pagination.append( - ExcelDataPagination( - self.data[ - 32 - + (self.header.column_count * 4) - + (i * 4) : 32 - + (self.header.column_count * 4) - + ((i + 1) * 4) - ] - ) - ) - self.languages: list[int] = [] - for i in range(self.header.language_count): - self.languages.append( - self.data[ - 32 - + (self.header.column_count * 4) - + (self.header.page_count * 4) - + i - ] - ) - - def map_names(self, names: list[Definition]) -> tuple[dict[int, tuple[str, str]], dict[str, dict[int, str]], int]: - """ - Map the column definitions to the names in the header. - - Args: - names: A list of names of the columns in the header. - - Returns: - A tuple of a dictionary mapping column offsets to tuples of the column type - and name, and the size of the structure. - """ - mapped: dict[int, tuple[str, str]] = {} - enumMapped: dict[str, dict[int, str]] = {} - largest_offset_index: int = 0 - for i in range(self.header.column_count): - if ( - self.column_definitions[i].offset - > self.column_definitions[largest_offset_index].offset - ): - largest_offset_index = i - - size = self.column_definitions[ - largest_offset_index - ].offset + column_data_type_to_size( - self.column_definitions[largest_offset_index].type - ) - - if self.header.column_count != len(names): - for i in range(self.header.column_count): - mapped[self.column_definitions[i].offset] = ( - column_data_type_to_c_type(self.column_definitions[i].type), - "field_{0}".format(i), - ) - else: - for i in range(self.header.column_count): - col_def = self.column_definitions[i] - if ( - col_def.offset in mapped - and mapped[col_def.offset] is not None - ): - if (col_def.type in {ExcelColumnDataType.PackedBool0, ExcelColumnDataType.PackedBool1, ExcelColumnDataType.PackedBool2, ExcelColumnDataType.PackedBool3, ExcelColumnDataType.PackedBool4, ExcelColumnDataType.PackedBool5, ExcelColumnDataType.PackedBool6, ExcelColumnDataType.PackedBool7}): - name = "PackedBool{0:X}".format(col_def.offset) - if (name in enumMapped): - enumMapped["PackedBool{0:X}".format(col_def.offset)][(1 << col_def.type - ExcelColumnDataType.PackedBool0)] = "{0}_{1}".format(name, names[i].get_name()) - else: - # this should never be hit but just to be safe - enumMapped[name] = {(1 << col_def.type - ExcelColumnDataType.PackedBool0): "{0}_{1}".format(name, names[i].get_name())} - else: - [_, name] = mapped[col_def.offset] - if name.split("_")[0] == "Unknown": - continue - if ( - column_data_type_to_c_type(col_def.type) - != "unsigned __int8" - ): - continue - else: - mapped[col_def.offset] = ( - column_data_type_to_c_type(col_def.type), - names[i].get_name(), - ) - else: - if(col_def.type in {ExcelColumnDataType.PackedBool0, ExcelColumnDataType.PackedBool1, ExcelColumnDataType.PackedBool2, ExcelColumnDataType.PackedBool3, ExcelColumnDataType.PackedBool4, ExcelColumnDataType.PackedBool5, ExcelColumnDataType.PackedBool6, ExcelColumnDataType.PackedBool7}): - name = "PackedBool{0:X}".format(col_def.offset) - mapped[col_def.offset] = ( - f"Component::Exd::Sheets::{self.name}::{name}", - name - ) - enumMapped[name] = {(1 << col_def.type - ExcelColumnDataType.PackedBool0): "{0}_{1}".format(name, names[i].get_name())} - else: - mapped[self.column_definitions[i].offset] = ( - column_data_type_to_c_type(self.column_definitions[i].type), - names[i].get_name(), - ) - mapped = dict(sorted(mapped.items())) - enumMapped = dict(sorted(enumMapped.items())) - return [mapped, enumMapped, size] - - -def column_data_type_to_c_type(column_data_type): - # type: (ExcelColumnDataType) -> str - if column_data_type == ExcelColumnDataType.Bool: - return "bool" - elif column_data_type == ExcelColumnDataType.Int8: - return "__int8" - elif column_data_type == ExcelColumnDataType.UInt8: - return "unsigned __int8" - elif column_data_type == ExcelColumnDataType.Int16: - return "__int16" - elif column_data_type == ExcelColumnDataType.UInt16: - return "unsigned __int16" - elif column_data_type == ExcelColumnDataType.Int32: - return "__int32" - elif column_data_type == ExcelColumnDataType.UInt32: - return "unsigned __int32" - elif column_data_type == ExcelColumnDataType.Float32: - return "float" - elif column_data_type == ExcelColumnDataType.Int64: - return "__int64" - elif column_data_type == ExcelColumnDataType.UInt64: - return "unsigned __int64" - elif ( - column_data_type == ExcelColumnDataType.PackedBool0 - or column_data_type == ExcelColumnDataType.PackedBool1 - or column_data_type == ExcelColumnDataType.PackedBool2 - or column_data_type == ExcelColumnDataType.PackedBool3 - or column_data_type == ExcelColumnDataType.PackedBool4 - or column_data_type == ExcelColumnDataType.PackedBool5 - or column_data_type == ExcelColumnDataType.PackedBool6 - or column_data_type == ExcelColumnDataType.PackedBool7 - ): - return "unsigned __int8" - elif column_data_type == ExcelColumnDataType.String: - return "unsigned __int32" - - -def column_data_type_to_size(column_data_type): - # type: (ExcelColumnDataType) -> int - if ( - column_data_type == ExcelColumnDataType.Bool - or column_data_type == ExcelColumnDataType.Int8 - or column_data_type == ExcelColumnDataType.UInt8 - or column_data_type == ExcelColumnDataType.PackedBool0 - or column_data_type == ExcelColumnDataType.PackedBool1 - or column_data_type == ExcelColumnDataType.PackedBool2 - or column_data_type == ExcelColumnDataType.PackedBool3 - or column_data_type == ExcelColumnDataType.PackedBool4 - or column_data_type == ExcelColumnDataType.PackedBool5 - or column_data_type == ExcelColumnDataType.PackedBool6 - or column_data_type == ExcelColumnDataType.PackedBool7 - ): - return 1 - elif ( - column_data_type == ExcelColumnDataType.Int16 - or column_data_type == ExcelColumnDataType.UInt16 - ): - return 2 - elif ( - column_data_type == ExcelColumnDataType.Int32 - or column_data_type == ExcelColumnDataType.UInt32 - or column_data_type == ExcelColumnDataType.Float32 - or column_data_type == ExcelColumnDataType.String - ): - return 4 - elif ( - column_data_type == ExcelColumnDataType.Int64 - or column_data_type == ExcelColumnDataType.UInt64 - ): - return 8 diff --git a/ida/luminapie/exdschema.py b/ida/luminapie/exdschema.py deleted file mode 100644 index fc2e798b7c..0000000000 --- a/ida/luminapie/exdschema.py +++ /dev/null @@ -1,66 +0,0 @@ -from urllib.request import urlopen, Request -from urllib.error import HTTPError, URLError -from luminapie.definitions import ( - Definition, - RepeatDefinition, - get_definition, - SemanticVersion, -) -from yaml import load, Loader -from zipfile import ZipFile -from tempfile import TemporaryFile - - -def get_url(url, supress=False): - # type: (str, bool) -> bytes | None - req = Request(url) - try: - resp = urlopen(req) - return resp.read() - except HTTPError as e: - if not supress: - print("HTTP Error code: ", e.code, " for url: ", url) - return None - except URLError as e: - if not supress: - print("HTTP Reason: ", e.reason, " for url: ", url) - return None - - -# TODO: Add the ability to use previous version schemas as well -def get_definitions(schema): - # type: (SemanticVersion) -> dict[str, list[Definition]] - exd_schema_map = {} - with TemporaryFile() as f: - f.write( - get_url( - "https://github.com/xivdev/EXDSchema/archive/refs/heads/latest.zip", - True, - ) - ) - f.seek(0) - schema_zip = ZipFile(f) - - for file in schema_zip.namelist(): - if file.endswith(".yml") and ".github" not in file: - schema_yml = load(schema_zip.read(file), Loader=Loader) - if "pendingFields" in schema_yml: - exd_schema_map[file.rsplit(".", 1)[0].rsplit("/")[1]] = schema_yml[ - "pendingFields" - ] - else: - exd_schema_map[file.rsplit(".", 1)[0].rsplit("/")[1]] = schema_yml[ - "fields" - ] - - defs_map = {} - for exd in exd_schema_map: - defs = [] - for field in exd_schema_map[exd]: - defin = get_definition(field) - if isinstance(defin, RepeatDefinition): - defs.extend(defin.flatten("")) - else: - defs.append(defin) - defs_map[exd] = defs - return defs_map diff --git a/ida/luminapie/file_handlers.py b/ida/luminapie/file_handlers.py deleted file mode 100644 index 27a2421018..0000000000 --- a/ida/luminapie/file_handlers.py +++ /dev/null @@ -1,39 +0,0 @@ -import os - - -def get_game_data_folders(root): - # type: (str) -> str - for folder in os.listdir(os.path.join(root, "sqpack")): - if os.path.isdir(os.path.join(root, "sqpack", folder)): - yield folder - - -def get_files(path): - # type: (str) -> list[bytes] - files: list[bytes] = [] - for dir_path, dir_names, file_names in os.walk(path): - files.extend(os.path.join(dir_path, file) for file in file_names) - - return files - - -def get_sqpack_files(root, path): - # type: (str, str) -> str - for file in get_files(os.path.join(root, "sqpack", path)): - ext = file.split(".")[-1] - if ext.startswith("dat"): - yield file - - -def get_sqpack_index(root, path): - # type: (str, str) -> str - for file in get_files(os.path.join(root, "sqpack", path)): - if file.endswith(".index"): - yield file - - -def get_sqpack_index2(root, path): - # type: (str, str) -> str - for file in get_files(os.path.join(root, "sqpack", path)): - if file.endswith(".index2"): - yield file diff --git a/ida/luminapie/game_data.py b/ida/luminapie/game_data.py deleted file mode 100644 index 05ca000be9..0000000000 --- a/ida/luminapie/game_data.py +++ /dev/null @@ -1,132 +0,0 @@ -from luminapie.sqpack import SqPack, SqPackIndexHashTable -from luminapie.file_handlers import get_game_data_folders, get_sqpack_index -from luminapie.se_crc import Crc32 -from luminapie.exdschema import get_definitions -from luminapie.definitions import SemanticVersion -import os - -crc = Crc32() - - -class Repository: - def __init__(self, name, root): - # type: (str, str) -> None - self.root = root - self.name = name - self.sqpacks: list[SqPack] = [] - self.index: dict[int, tuple[SqPackIndexHashTable, SqPack]] = {} - self.expansion_id = 0 - self.get_expansion_id() - - def get_expansion_id(self): - # type: () -> None - if self.name.startswith("ex"): - self.expansion_id = int(self.name.removeprefix("ex")) - - def parse_version(self): - # type: () -> None - versionPath = "" - if self.name == "ffxiv": - versionPath = os.path.join(self.root, "ffxivgame.ver") - else: - versionPath = os.path.join( - self.root, "sqpack", self.name, self.name + ".ver" - ) - if os.path.exists(versionPath): - with open(versionPath, "r") as f: - self.version = SemanticVersion( - *(int(v) for v in f.read().strip().split(".")) - ) - else: - self.version = SemanticVersion(0, 0, 0, 0) - - def setup_indexes(self): - # type: () -> None - for file in get_sqpack_index(self.root, self.name): - self.sqpacks.append(SqPack(self.root, file)) - - for sqpack in self.sqpacks: - sqpack.discover_data_files() - for indexes in sqpack.hash_table: - self.index[indexes.hash] = [indexes, sqpack] - - def get_index(self, hash): - # type: (int) -> tuple[SqPackIndexHashTable, SqPack] - return self.index[hash] - - def get_file(self, hash): - # type: (int) -> bytes - index, sqpack = self.get_index(hash) - id = index.data_file_id() - offset = index.data_file_offset() - return SqPack(self.root, sqpack.data_files[id]).read_file(offset) - - def __repr__(self): - # type: () -> str - return "Repository: {0} ({1}) - {2}".format( - self.name, self.version, self.expansion_id - ) - - -class GameData: - def __init__(self, root, load_schema=True): - # type: (str, bool) -> None - self.root = root - self.repositories: dict[int, Repository] = {} - self.load_schema = load_schema - self.setup() - - def get_repo_index(self, folder): - # type: (str) -> int - if folder == "ffxiv": - return 0 - else: - return int(folder.removeprefix("ex")) - - def setup(self): - # type: () -> None - for folder in get_game_data_folders(self.root): - self.repositories[self.get_repo_index(folder)] = Repository( - folder, self.root - ) - - for folder in self.repositories: - repo = self.repositories[folder] - repo.parse_version() - repo.setup_indexes() - - if self.load_schema: - self.schema = get_definitions(self.repositories[0].version) - - def get_file(self, file): - # type: (ParsedFileName) -> bytes - return self.repositories[self.get_repo_index(file.repo)].get_file(file.index) - - def get_exd_schema(self, key): - # type: (str) -> list[Definition] - if key not in self.schema: - return [] - return self.schema[key] - - def __repr__(self): - # type: () -> str - return "Repositories: {0}".format(self.repositories) - - -class ParsedFileName: - def __init__(self, path): - # type: (str) -> None - self.path = path.lower().strip() - parts = self.path.split("/") - self.category = parts[0] - self.index = crc.calc_index(self.path) - self.index2 = crc.calc_index2(self.path) - self.repo = parts[1] - if self.repo[0] != "e" or self.repo[1] != "x" or not self.repo[2].isdigit(): - self.repo = "ffxiv" - - def __repr__(self): - # type: () -> str - return "ParsedFileName: {0}, category: {1}, index: {2:X}, index2: {3:X}, repo: {4}".format( - self.path, self.category, self.index, self.index2, self.repo - ) diff --git a/ida/luminapie/se_crc.py b/ida/luminapie/se_crc.py deleted file mode 100644 index 6d3729222d..0000000000 --- a/ida/luminapie/se_crc.py +++ /dev/null @@ -1,74 +0,0 @@ -class Crc32: - def __init__(self): - # type: () -> None - self.poly = 0xEDB88320 - self.table = [0] * 256 * 16 - for i in range(256): - res = i - for j in range(16): - for k in range(8): - if res & 1 == 1: - res = self.poly ^ (res >> 1) - else: - res = res >> 1 - self.table[i + j * 256] = res - - def calc(self, value): - # type: (bytes) -> int - start = 0 - size = len(value) - crc_local = 4294967295 ^ 0 - while size >= 16: - a = ( - self.table[(3 * 256) + value[start + 12]] - ^ self.table[(2 * 256) + value[start + 13]] - ^ self.table[(1 * 256) + value[start + 14]] - ^ self.table[(0 * 256) + value[start + 15]] - ) - b = ( - self.table[(7 * 256) + value[start + 8]] - ^ self.table[(6 * 256) + value[start + 9]] - ^ self.table[(5 * 256) + value[start + 10]] - ^ self.table[(4 * 256) + value[start + 11]] - ) - c = ( - self.table[(11 * 256) + value[start + 4]] - ^ self.table[(10 * 256) + value[start + 5]] - ^ self.table[(9 * 256) + value[start + 6]] - ^ self.table[(8 * 256) + value[start + 7]] - ) - d = ( - self.table[(15 * 256) + (self.byte(crc_local) ^ value[start])] - ^ self.table[(14 * 256) + (self.byte(crc_local, 1) ^ value[start + 1])] - ^ self.table[(13 * 256) + (self.byte(crc_local, 2) ^ value[start + 2])] - ^ self.table[(12 * 256) + (self.byte(crc_local, 3) ^ value[start + 3])] - ) - crc_local = d ^ c ^ b ^ a - start += 16 - size -= 16 - - while size > 0: - crc_local = self.table[(crc_local ^ value[start]) & 0xFF] ^ (crc_local >> 8) - start += 1 - size -= 1 - - return ~(crc_local ^ 4294967295) % (1 << 32) - - def byte(self, number, i=0): - # type: (int, int) -> int - return (number & (0xFF << (i * 8))) >> (i * 8) - - def calc_index(self, path): - # type: (str) -> int - path_parts = path.split("/") - filename = path_parts[-1] - folder = path.rstrip(filename).rstrip("/") - - foldercrc = self.calc(folder.encode("utf-8")) - filecrc = self.calc(filename.encode("utf-8")) - - return foldercrc << 32 | filecrc - - def calc_index2(self, path): - # type: (str) -> int - return self.calc(path.encode("utf-8")) diff --git a/ida/luminapie/sqpack.py b/ida/luminapie/sqpack.py deleted file mode 100644 index ef6be0ad5a..0000000000 --- a/ida/luminapie/sqpack.py +++ /dev/null @@ -1,225 +0,0 @@ -from luminapie.enums import SqPackFileType, SqPackPlatformId -from io import BufferedReader -import os -import zlib -from luminapie.file_handlers import get_sqpack_files - - -class SqPackFileInfo: - def __init__(self, bytes, offset): - # type: (bytes, int) -> None - self.header_size = int.from_bytes(bytes[0:4], byteorder="little") - self.type = SqPackFileType(int.from_bytes(bytes[4:8], byteorder="little")) - self.raw_file_size = int.from_bytes(bytes[8:12], byteorder="little") - self.unknown = [ - int.from_bytes(bytes[12:16], byteorder="little"), - int.from_bytes(bytes[16:20], byteorder="little"), - ] - self.number_of_blocks = int.from_bytes(bytes[20:24], byteorder="little") - self.offset = offset - - -class DatStdFileBlockInfos: - def __init__(self, bytes): - # type: (bytes) -> None - self.offset = int.from_bytes(bytes[0:4], byteorder="little") - self.compressed_size = int.from_bytes(bytes[4:6], byteorder="little") - self.uncompressed_size = int.from_bytes(bytes[6:8], byteorder="little") - - -class DatBlockHeader: - def __init__(self, bytes): - # type: (bytes) -> None - self.size = int.from_bytes(bytes[0:4], byteorder="little") - self.unknown1 = int.from_bytes(bytes[4:8], byteorder="little") - self.block_data_size = int.from_bytes(bytes[8:12], byteorder="little") - self.dat_block_type = int.from_bytes(bytes[12:16], byteorder="little") - - def __repr__(self): - # type: () -> str - return "Size: {0} Unknown1: {1} DatBlockType: {2} BlockDataSize: {3}".format( - self.size, self.unknown1, self.dat_block_type, self.block_data_size - ) - - -class SqPackHeader: - def __init__(self, file): - # type: (BufferedReader) -> None - self.magic = file.read(8) - self.platform_id = SqPackPlatformId( - int.from_bytes(file.read(1), byteorder="little") - ) - self.unknown = file.read(3) - if self.platform_id != SqPackPlatformId.PS3: - self.size = int.from_bytes(file.read(4), byteorder="little") - self.version = int.from_bytes(file.read(4), byteorder="little") - self.type = int.from_bytes(file.read(4), byteorder="little") - else: - raise Exception("PS3 is not supported") - - def __repr__(self): - # type: () -> str - return "Magic: {0} Platform: {1} Size: {2} Version: {3} Type: {4}".format( - self.magic, self.platform_id, self.size, self.version, self.type - ) - - -class SqPackIndexHeader: - def __init__(self, bytes: bytes): - # type: (bytes) -> None - self.size = int.from_bytes(bytes[0:4], byteorder="little") - self.version = int.from_bytes(bytes[4:8], byteorder="little") - self.index_data_offset = int.from_bytes(bytes[8:12], byteorder="little") - self.index_data_size = int.from_bytes(bytes[12:16], byteorder="little") - self.index_data_hash = bytes[16:80] - self.number_of_data_file = int.from_bytes(bytes[80:84], byteorder="little") - self.synonym_data_offset = int.from_bytes(bytes[84:88], byteorder="little") - self.synonym_data_size = int.from_bytes(bytes[88:92], byteorder="little") - self.synonym_data_hash = bytes[92:156] - self.empty_block_data_offset = int.from_bytes( - bytes[156:160], byteorder="little" - ) - self.empty_block_data_size = int.from_bytes(bytes[160:164], byteorder="little") - self.empty_block_data_hash = bytes[164:228] - self.dir_index_data_offset = int.from_bytes(bytes[228:232], byteorder="little") - self.dir_index_data_size = int.from_bytes(bytes[232:236], byteorder="little") - self.dir_index_data_hash = bytes[236:300] - self.index_type = int.from_bytes(bytes[300:304], byteorder="little") - self.reserved = bytes[304:960] - self.hash = bytes[960:1024] - - def __repr__(self): - # type: () -> str - return "Size: {0} Version: {1} Index Data Offset: {2} Index Data Size: {3} Index Data Hash: {4} Number Of Data File: {5} Synonym Data Offset: {6} Synonym Data Size: {7} Synonym Data Hash: {8} Empty Block Data Offset: {9} Empty Block Data Size: {10} Empty Block Data Hash: {11} Dir Index Data Offset: {12} Dir Index Data Size: {13} Dir Index Data Hash: {14} Index Type: {15} Reserved: {16} Hash: {17}".format( - self.size, - self.version, - self.index_data_offset, - self.index_data_size, - self.index_data_hash, - self.number_of_data_file, - self.synonym_data_offset, - self.synonym_data_size, - self.synonym_data_hash, - self.empty_block_data_offset, - self.empty_block_data_size, - self.empty_block_data_hash, - self.dir_index_data_offset, - self.dir_index_data_size, - self.dir_index_data_hash, - self.index_type, - self.reserved, - self.hash, - ) - - -class SqPackIndexHashTable: - def __init__(self, bytes): - # type: (bytes) -> None - self.hash = int.from_bytes(bytes[0:8], byteorder="little") - self.data = int.from_bytes(bytes[8:12], byteorder="little") - self.padding = int.from_bytes(bytes[12:16], byteorder="little") - - def is_synonym(self): - # type: () -> bool - return (self.data & 0b1) == 0b1 - - def data_file_id(self): - # type: () -> int - return (self.data & 0b1110) >> 1 - - def data_file_offset(self): - # type: () -> int - return (self.data & ~0xF) * 0x08 - - def __repr__(self): - # type: () -> str - return "Hash: {0} Data: {1} Padding: {2} Is Synonym: {3} Data File ID: {4} Data File Offset: {5}".format( - self.hash, - self.data, - self.padding, - self.is_synonym(), - self.data_file_id(), - self.data_file_offset(), - ) - - -class SqPack: - def __init__(self, root, path): - # type: (str, str) -> None - self.root = root - self.path = path - self.file = open(path, "rb") - self.header = SqPackHeader(self.file) - - def get_index_header(self): - # type: () -> SqPackIndexHeader - self.file.seek(self.header.size) - return SqPackIndexHeader(self.file.read(1024)) - - def get_index_hash_table(self, index_header): - # type: (SqPackIndexHeader) -> list[SqPackIndexHashTable] - self.file.seek(index_header.index_data_offset) - entry_count = index_header.index_data_size // 16 - return [SqPackIndexHashTable(self.file.read(16)) for _ in range(entry_count)] - - def load_index_header(self): - # type: () -> None - self.index_header = self.get_index_header() - - def load_hash_table(self): - # type: () -> None - self.hash_table = self.get_index_hash_table(self.index_header) - - def discover_data_files(self): - # type: () -> None - self.load_index_header() - self.load_hash_table() - self.data_files: list[str] = [] - for file in get_sqpack_files( - self.root, self.path.rsplit(os.sep, 1)[0].split(os.sep)[-1] - ): - for i in range(0, self.index_header.number_of_data_file): - name = self.path.rsplit(".", 1)[0] + ".dat" + str(i) - if file == name: - self.data_files.append(file) - - def read_file(self, offset): - # type: (int) -> list[bytes] - if self.path.rsplit(".", 1)[1][0:3] != "dat": - raise Exception("Not a data file") - self.file.seek(offset) - file_info_bytes = self.file.read(24) - file_info = SqPackFileInfo(file_info_bytes, offset) - data: list[bytes] = [] - if file_info.type == SqPackFileType.Empty: - raise Exception(f"File located at 0x{hex(offset)} is empty.") - elif file_info.type == SqPackFileType.Standard: - data = self.read_standard_file(file_info) - else: - raise Exception("Type: " + str(file_info.type) + " not implemented.") - return data - - def read_standard_file(self, file_info): - # type: (SqPackFileInfo) -> list[bytes] - block_bytes = self.file.read(file_info.number_of_blocks * 8) - data: list[bytes] = [] - for i in range(file_info.number_of_blocks): - block = DatStdFileBlockInfos(block_bytes[i * 8 : i * 8 + 8]) - self.file.seek(file_info.offset + file_info.header_size + block.offset) - block_header = DatBlockHeader(self.file.read(16)) - if block_header.dat_block_type == 32000: - data.append(self.file.read(block_header.block_data_size)) - else: - data.append( - zlib.decompress( - self.file.read(block_header.block_data_size), wbits=-15 - ) - ) - - return data - - def __repr__(self): - # type: () -> str - return "Path: {0} Header: {1}".format( - os.path.join(self.root, "sqpack", self.path), self.header - ) diff --git a/ida/structs_schema.py b/ida/structs_schema.py deleted file mode 100644 index 828f1f3103..0000000000 --- a/ida/structs_schema.py +++ /dev/null @@ -1,116 +0,0 @@ -class DefinedStructBase: - def __init__(self, name, type, namespace): - # type: (str, str, str) -> None - self.name = name - self.type = type - self.namespace = namespace - - -class DefinedStructEnum(DefinedStructBase, object): - def __init__(self, name, type, underlying, namespace, flags, values): - # type: (str, str, str, str, bool, dict[str, int]) -> None - super(DefinedStructEnum, self).__init__(name, type, namespace) - self.name = name - self.type = type - self.values = values - self.flags = flags - self.underlying = underlying - - -class DefinedStructFuncParam: - def __init__(self, name, type): - # type: (str, str) -> None - self.name = name - if type == "__fastcall": - self.type = "__int64" - else: - self.type = type - - -class DefinedStructVFunc: - def __init__(self, name, return_type, offset, parameters): - # type: (str, str, int, list[DefinedStructFuncParam]) -> None - self.name = name - self.return_type = return_type - self.offset = offset - self.parameters = parameters - self.inherited_from_preprocess = False - - -class DefinedStructMemFunc: - def __init__(self, signature, return_type, parameters, name): - # type: (str, str, list[DefinedStructFuncParam], str) -> None - self.signature = signature - self.return_type = return_type - self.parameters = parameters - self.name = name - - -class DefinedStructField(DefinedStructFuncParam, object): - def __init__(self, name, type, offset, base): - # type: (str, str, int, bool) -> None - super(DefinedStructField, self).__init__(name, type) - self.offset = offset - self.base = base - self.srclang_field_name = name - self.srclang_is_baseclass = False - - -class DefinedStructFuncField(DefinedStructField, object): - def __init__(self, name, type, offset, base, return_type, params): - # type: (str, str, int, bool, str | None, list[DefinedStructFuncParam] | None) -> None - super(DefinedStructFuncField, self).__init__(name, type, offset, base) - self.return_type = return_type - self.parameters = params - - -class DefinedStructStaticMember: - def __init__(self, name, signature, relative_offsets, return_type, is_pointer): - # type: (str, str, list[int], str, bool) -> None - self.name = name - self.signature = signature - self.relative_offsets = relative_offsets - self.return_type = return_type - self.is_pointer = is_pointer - - -class DefinedStructFixedField(DefinedStructField, object): - def __init__(self, name, type, offset, base, size, is_string): - # type: (str, str, int, bool, str | None, bool) -> None - super(DefinedStructFixedField, self).__init__(name, type, offset, base) - self.size = size - self.is_string = is_string - -class DefinedStruct(DefinedStructBase, object): - def __init__( - self, - name, - type, - namespace, - fields, - size, - vtable_size, - virtual_functions, - member_functions, - union, - static_member_functions, - static_members, - ): - # type: (str, str, str, list[DefinedStructField], int | None, int | None, list[DefinedStructVFunc] | None, list[DefinedStructMemFunc], str, list[DefinedStructMemFunc] | None, list[DefinedStructStaticMember] | None) -> None - super(DefinedStruct, self).__init__(name, type, namespace) - self.fields = fields - self.size = size - self.vtable_size = vtable_size - self.virtual_functions = virtual_functions - self.member_functions = member_functions - self.union = bool(union) - self.static_member_functions = static_member_functions - self.static_members = static_members - self.srclang_needs_vfptr = len(fields) == 0 or fields[0].offset != 0 - - -class DefinedStructExport: - def __init__(self, enums, structs): - # type: (list[DefinedStructEnum], list[DefinedStruct]) -> None - self.enums = enums - self.structs = structs diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000000..e206831281 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,47 @@ +## ida/ and ghidra/ +Both folders hold the same scripts for their disassembler, and share the files in `shared/`. + +### rename.py +This script ingests the `data.yml` file in the scripts directory, and renames various offsets as globals, functions, vtables and such. + +### struct_import.py +This script imports the Excel sheets and `structs.yml` as types, names the Excel getter functions and applies the function types. +Both generate the same C++ header and let clang work out the layout of every struct, template instances included. IDA uses its built in clang parser. Ghidra has no C++ parser, so `ghidra/clang_wrapper.py` reads the layouts back from libclang and `ghidra_wrapper.py` creates the data types from that. + +### full_run.py +This script runs `rename.py` and `struct_import.py` in sequence + +#### Shared prerequisites: +Both IDA and Ghidra need these for `struct_import.py`: +- Python, any version still listed on the [Python versions status page](https://devguide.python.org/versions/) is supported. +- The [.NET 10 runtime](https://dotnet.microsoft.com/download/dotnet/10.0), the same version FFXIVClientStructs uses. `pythonnet` loads [Lumina](https://github.com/NotAdam/Lumina) with it to read the Excel headers. +- The game installed. Its folder is read from the best query holding both `game` and `boot`. + +#### IDA dependency installation: +`pip install pyyaml anytree pythonnet` or `python -m pip install pyyaml anytree pythonnet` in whichever version of python you are currently using with IDA Pro. +If you are familiar with Poetry, the `idarename` extras package will do the same. + +Additionally, you will need to place "idauser.cfg" from this repository in "%AppData%\Hex-Rays\IDA Pro\". IDA doesn't allow you to use certain characters in names and this config changes that. + +#### Ghidra dependency installation: +See [Getting Started](../Ghidra/Getting%20Started.md) for information on how to set up PyGhidra. The same packages as IDA are needed in the PyGhidra environment, plus `libclang`. +Outside of Windows the MSVC and Windows SDK headers are needed as well, `struct_import.py` asks for the folder [mstorsjo/msvc-wine](https://github.com/mstorsjo/msvc-wine)'s `vsdownload.py` downloaded to and `install.sh` was then run on, it picks the newest MSVC and Windows SDK version in it. +The `libclang` python package ships libclang 18 while newer MSVC headers want clang 20 or newer, the Ghidra scripts define `_ALLOW_COMPILER_AND_STL_VERSION_MISMATCH` and `_MSVC_STL_USE_ABORT_AS_DOOM_FUNCTION` so they parse anyway. +The `LIBCLANG_LIBRARY_FILE` environment variable can point to another libclang, ex. `/usr/lib/llvm-20/lib/libclang-20.so.1`. + +## ffxiv_sigmaker.py +> [!WARNING] +> Script file is no longer maintained by anyone. Most people use most people use this [sigmaker](https://github.com/caraxi/sigmaker-x64) instead + +This script ingests the data.yml file in the same directory, and generates automatic signatures for consumption by the source-gen modules. + +This support **Python3** and **IDA** only. + +#### Dependency installation: +`pip install dacite pyyaml ruamel-yaml` or `python -m pip install dacite pyyaml ruamel-yaml` in whichever version of python you are currently using with IDA Pro. +If you are familiar with Poetry, the `sigmaker` extras package will do the same. + +## classinformer.csv +Once upon a time, someone did a bad thing and released FFXIV with the RTTI data intact. +This is an export of that RTTI data using the ClassInformer IDA Pro plugin. +It is useful as a baseline for many hierarchies and class names, be aware however it is several years old at this point. diff --git a/ida/classinformer.csv b/scripts/classinformer.csv similarity index 100% rename from ida/classinformer.csv rename to scripts/classinformer.csv diff --git a/ida/classinformer_lysander.csv b/scripts/classinformer_lysander.csv similarity index 100% rename from ida/classinformer_lysander.csv rename to scripts/classinformer_lysander.csv diff --git a/ida/data-validator.js b/scripts/data-validator.js similarity index 98% rename from ida/data-validator.js rename to scripts/data-validator.js index 02ee3b0c1f..6c4e884960 100644 --- a/ida/data-validator.js +++ b/scripts/data-validator.js @@ -15,7 +15,7 @@ function toHex(str) { return !isNaN(str) ? toHexStringFromNumber(parseInt(str)) : str; } try { - const doc = yaml.load(fs.readFileSync('./ida/data.yml', 'utf8')); + const doc = yaml.load(fs.readFileSync('./scripts/data.yml', 'utf8')); if (versionRegex.test(doc.version) === false) errors.push('Invalid version format'); if (typeof doc.globals == "object") diff --git a/ida/data.yml b/scripts/data.yml similarity index 100% rename from ida/data.yml rename to scripts/data.yml diff --git a/ida/luminapie/__init__.py b/scripts/errors.txt similarity index 100% rename from ida/luminapie/__init__.py rename to scripts/errors.txt diff --git a/ida/ffxiv_structs.yml b/scripts/ffxiv_structs.yml similarity index 100% rename from ida/ffxiv_structs.yml rename to scripts/ffxiv_structs.yml diff --git a/scripts/ghidra/clang_wrapper.py b/scripts/ghidra/clang_wrapper.py new file mode 100644 index 0000000000..e91d4900e4 --- /dev/null +++ b/scripts/ghidra/clang_wrapper.py @@ -0,0 +1,390 @@ +import os +import re +from dataclasses import replace + +# the crash recovery of libclang takes the SIGSEGV the JVM raises on purpose and kills Ghidra, read when the index is created +os.environ.setdefault("LIBCLANG_DISABLE_CRASH_RECOVERY", "1") + +from clang.cindex import Config, Cursor, CursorKind, Index, Type, TypeKind +from src_wrapper import layout_probe_prefix +from structs_schema import ( + DefinedStruct, + DefinedStructEnum, + DefinedStructExport, + DefinedStructField, + DefinedStructFixedField, + DefinedStructFuncField, + DefinedStructFuncParam, +) + +main_file = "ffxiv_structs.hpp" +# an instantiation can use templates that are only pointed to themselves +max_instantiation_passes = 8 + +# named like structs.yml does, sized for x86_64-pc-windows-msvc +primitive_types = { + TypeKind.VOID: "void", + TypeKind.BOOL: "bool", + TypeKind.CHAR_S: "char", + TypeKind.CHAR_U: "byte", + TypeKind.SCHAR: "__int8", + TypeKind.UCHAR: "byte", + TypeKind.WCHAR: "wchar_t", + TypeKind.CHAR16: "unsigned __int16", + TypeKind.CHAR32: "unsigned int", + TypeKind.SHORT: "__int16", + TypeKind.USHORT: "unsigned __int16", + TypeKind.INT: "int", + TypeKind.UINT: "unsigned int", + TypeKind.LONG: "int", + TypeKind.ULONG: "unsigned int", + TypeKind.LONGLONG: "__int64", + TypeKind.ULONGLONG: "unsigned __int64", + TypeKind.FLOAT: "float", + TypeKind.DOUBLE: "double", + TypeKind.LONGDOUBLE: "double", + TypeKind.NULLPTR: "void*", +} + +record_kinds = (CursorKind.STRUCT_DECL, CursorKind.CLASS_DECL, CursorKind.UNION_DECL) +pointer_kinds = (TypeKind.POINTER, TypeKind.LVALUEREFERENCE, TypeKind.RVALUEREFERENCE) +function_kinds = (TypeKind.FUNCTIONPROTO, TypeKind.FUNCTIONNOPROTO) + + +def split_type_path(name: str) -> list[str]: + """Split a type on its namespaces, not the ones in template arguments.""" + parts, depth, start, index = [], 0, 0, 0 + while index < len(name): + if name[index] == "<": + depth += 1 + elif name[index] == ">": + depth -= 1 + elif depth == 0 and name[index : index + 2] == "::": + parts.append(name[start:index]) + start = index + 2 + index += 1 + index += 1 + parts.append(name[start:]) + return parts + + +class RecordCollector(object): + """Turns the records clang laid out into the same structs structs.yml is read as.""" + + def __init__(self): + self.enums: dict[str, DefinedStructEnum] = {} + # in dependency order, a struct comes after everything it holds by value + self.structs: dict[str, DefinedStruct] = {} + # only ever pointed to, so they are imported without their members + self.opaque: dict[str, bool] = {} + # typedef name to the type behind it, see get_template_alias in structs_schema + self.aliases: dict[str, str] = {} + # template instances that are only pointed to, clang did not lay those out + self.incomplete: set[str] = set() + # records that are pointed to, laid out once the struct pointing to them is done to keep the dependency order + self._deferred: list[tuple[Type, Cursor, str]] = [] + + def collect(self, cursor: Cursor): + """Everything declared in the string and what it needs from the headers.""" + self._collect_declarations(cursor) + while self._deferred: + type, decl, name = self._deferred.pop() + if name not in self.structs: + self.add_record(type, decl, name) + + def _collect_declarations(self, cursor: Cursor): + for child in cursor.get_children(): + if child.location.file is None or child.location.file.name != main_file: + continue + if child.kind in (CursorKind.NAMESPACE, CursorKind.LINKAGE_SPEC): + self._collect_declarations(child) + elif child.kind in record_kinds and child.is_definition(): + if "(" not in child.type.spelling: + self.get_type_string(child.type, True, True) + self._collect_declarations(child) + elif child.kind == CursorKind.ENUM_DECL and child.is_definition(): + self.get_enum(child) + elif child.kind == CursorKind.TYPEDEF_DECL and child.spelling.startswith("__cs_"): + type_string = self.get_type_string(child.underlying_typedef_type, True, False) + if type_string is not None: + self.aliases[child.spelling] = type_string + + def get_export(self) -> DefinedStructExport: + self.wrap_primary_bases() + opaque = [ + self.new_struct(name, [], None, union) + for name, union in self.opaque.items() + if name not in self.structs + ] + return DefinedStructExport(list(self.enums.values()), [*opaque, *self.structs.values()]) + + def get_primary_base(self, struct: DefinedStruct) -> DefinedStruct | None: + """The base at offset 0, it shares its vtable pointer with the struct inheriting it.""" + if struct.union: + return None + base_field = next((field for field in struct.fields if field.base and field.offset == 0), None) + if base_field is None: + return None + base = self.structs.get(base_field.type) + return None if base is None or base.union else base + + def has_vtable_pointer(self, struct: DefinedStruct) -> bool: + """The struct or a base at offset 0 starts with __vtable.""" + if any(field.offset == 0 and field.name == "__vtable" for field in struct.fields): + return True + base = self.get_primary_base(struct) + return base is not None and self.has_vtable_pointer(base) + + def wrap_primary_bases(self): + """Ghidra can not retype the vtable pointer inside an embedded base, so a struct with a vtable of its own + holds its offset 0 base in a union with __vtable pointing to its own VTable, the base keeps its layout.""" + wrappers: dict[str, DefinedStruct] = {} + for name, struct in self.structs.items(): + vtable_name = f"{name}::VTable" + base = self.get_primary_base(struct) + if vtable_name not in self.structs or base is None or not self.has_vtable_pointer(base): + continue + base_field = next(field for field in struct.fields if field.base and field.offset == 0) + # ghidra settles a tie between union members on the first, so a vtable call goes through __vtable + wrapper = self.new_struct( + f"{name}::__primary_base", + [ + DefinedStructField("__vtable", f"{vtable_name}*", 0, False, [], []), + replace(base_field, name=re.sub(r"\W", "_", split_type_path(base.type)[-1]), base=False), + ], + base.size, + True, + ) + wrappers[name] = wrapper + + # replaced after the loop, so every base is looked at as clang laid it out + # every union comes right before the struct holding it, after the base and vtable it is made of + structs: dict[str, DefinedStruct] = {} + for name, struct in self.structs.items(): + if name in wrappers: + wrapper = wrappers[name] + structs[wrapper.type] = wrapper + struct = replace(struct, fields=[ + replace(field, type=wrapper.type) if field.base and field.offset == 0 else field + for field in struct.fields + ]) + structs[name] = struct + self.structs = structs + + def new_struct(self, type_name: str, fields: list, size: int | None, union: bool) -> DefinedStruct: + parts = split_type_path(type_name) + return DefinedStruct( + parts[-1], type_name, ".".join(parts[:-1]), fields, size, + None, [], [], union, None, None, [], [], [], False, + ) + + def get_decl_name(self, decl: Cursor) -> str: + name = re.sub(r"\s+([*&])", r"\1", decl.type.spelling) + if "(" in name: + # ex. Foo::(unnamed union at file:9:175) + name = f"{name[:name.index('(')]}anon_{decl.location.line}_{decl.location.column}" + return name + + def get_type_string(self, type: Type, follow: bool, by_value: bool) -> str | None: + """The type as structs.yml spells it, None if it can not be described.""" + type = type.get_canonical() + if type.kind in primitive_types: + return primitive_types[type.kind] + + if type.kind in pointer_kinds: + pointee = type.get_pointee().get_canonical() + if pointee.kind in function_kinds: + return "__fastcall" + return (self.get_type_string(pointee, follow, False) or "void") + "*" + + if type.kind in (TypeKind.CONSTANTARRAY, TypeKind.INCOMPLETEARRAY) and not by_value: + # a pointer to an array points at its first element + return self.get_type_string(type.element_type, follow, False) + + if type.kind == TypeKind.ENUM: + return self.get_enum(type.get_declaration()) + + if type.kind == TypeKind.RECORD: + return self.get_record(type, follow, by_value) + + return None + + def get_enum(self, decl: Cursor) -> str: + decl = decl.get_definition() or decl + name = self.get_decl_name(decl) + if name not in self.enums: + short_name = split_type_path(name)[-1] + values = { + child.spelling.removeprefix(f"{short_name}_"): child.enum_value + for child in decl.get_children() + if child.kind == CursorKind.ENUM_CONSTANT_DECL + } + underlying = primitive_types.get(decl.enum_type.get_canonical().kind, "int") + self.enums[name] = DefinedStructEnum( + short_name, name, ".".join(split_type_path(name)[:-1]), underlying, False, values + ) + return name + + def get_record(self, type: Type, follow: bool, by_value: bool) -> str: + decl = type.get_declaration() + decl = decl.get_definition() or decl + name = self.get_decl_name(decl) + if name in self.structs: + return name + + if type.get_size() >= 0 and by_value: + self.add_record(type, decl, name) + elif type.get_size() >= 0 and follow: + self._deferred.append((type, decl, name)) + else: + if follow and "<" in name: + self.incomplete.add(name) + self.opaque[name] = decl.kind == CursorKind.UNION_DECL + return name + + def add_record(self, type: Type, decl: Cursor, name: str): + # what a header of the compiler or windows points to is not followed, that would pull in most of it + follow = decl.location.file is not None and decl.location.file.name == main_file + fields = [] + + # clang does not report where a base is, they are laid out in order before the fields + offset = 0 + for child in decl.get_children(): + if child.kind != CursorKind.CXX_BASE_SPECIFIER: + continue + base = child.type.get_canonical() + if base.get_size() < 0 or self.is_empty(base): + continue + offset = -(-offset // base.get_align()) * base.get_align() + fields.append( + DefinedStructField(f"baseclass_{offset:X}", self.get_type_string(base, follow, True), offset, True, [], []) + ) + offset += base.get_size() + + union = decl.kind == CursorKind.UNION_DECL + # libclang validates the whole record for every offset asked for, so it is only asked where alignment moved a field + max_field_align = type.get_align() + end = offset + ask_clang = False + for index, field in enumerate(type.get_fields()): + if field.is_bitfield(): + ask_clang = True + continue + field_type = field.type.get_canonical() + field_size, field_align = field_type.get_size(), field_type.get_align() + if union: + field_offset = 0 + elif ask_clang or index == 0 or field_size < 0 or field_align <= 0: + bit_offset = field.get_field_offsetof() + if bit_offset < 0 or bit_offset % 8 != 0: + continue + field_offset = bit_offset // 8 + else: + field_align = min(field_align, max_field_align) + field_offset = -(-end // field_align) * field_align + if field_offset != end: + bit_offset = field.get_field_offsetof() + if bit_offset != field_offset * 8: + ask_clang = True + if bit_offset < 0 or bit_offset % 8 != 0: + continue + field_offset = bit_offset // 8 + end = field_offset + max(field_size, 0) + defined_field = self.get_field(field, field_offset, follow) + if defined_field is not None: + fields.append(defined_field) + + self.structs[name] = self.new_struct(name, fields, type.get_size(), union) + + def is_empty(self, type: Type) -> bool: + """An empty base takes up no room in the struct inheriting it.""" + if next(iter(type.get_fields()), None) is not None: + return False + return all( + self.is_empty(child.type.get_canonical()) + for child in type.get_declaration().get_children() + if child.kind == CursorKind.CXX_BASE_SPECIFIER + ) + + def get_field(self, field: Cursor, offset: int, follow: bool): + name = field.spelling or f"anon_{offset:X}" + type = field.type.get_canonical() + count = None + while type.kind == TypeKind.CONSTANTARRAY: + # ghidra gets the flattened array, ex. int[2][3] as int[6] + count = (count or 1) * type.element_count + type = type.element_type.get_canonical() + if count == 0 or type.kind == TypeKind.INCOMPLETEARRAY: + return None + + if count is None and type.kind == TypeKind.POINTER and type.get_pointee().get_canonical().kind == TypeKind.FUNCTIONPROTO: + function = type.get_pointee().get_canonical() + names = [child.spelling for child in field.get_children() if child.kind == CursorKind.PARM_DECL] + parameters = [ + DefinedStructFuncParam( + names[index] if index < len(names) and names[index] else f"a{index + 1}", + self.get_type_string(argument, follow, False) or "__int64", + ) + for index, argument in enumerate(function.argument_types()) + ] + return_type = self.get_type_string(function.get_result(), follow, False) or "__int64" + return DefinedStructFuncField(name, "__fastcall", offset, False, [], [], return_type, parameters) + + type_string = self.get_type_string(type, follow, True) + if type_string is None: + # ex. a member pointer, keep its room in the struct + type_string, count = "byte", field.type.get_size() + if count <= 0: + return None + + if count is not None: + return DefinedStructFixedField(name, type_string, offset, False, [], [], count, False) + return DefinedStructField(name, type_string, offset, False, [], []) + + +class ClangInterface(object): + def __init__(self, args: list[str], library_file: str | None = None): + """Uses the libclang of the python package unless library_file is given.""" + if library_file is not None and not Config.loaded: + Config.set_library_file(library_file) + self.args = args + + def parse_string(self, decl: str) -> tuple[DefinedStructExport, dict[str, str], int]: + """The layout of every struct, the template typedefs and the error count.""" + index = Index.create() + forced: list[str] = [] + for _ in range(max_instantiation_passes): + # sizeof needs the complete type, that makes clang instantiate the templates only used behind a pointer + source = decl + "".join( + f"\nstatic const auto __cs_instantiate_{number} = sizeof(::{name});" + for number, name in enumerate(forced) + ) + unit = index.parse(main_file, args=self.args, unsaved_files=[(main_file, source)]) + collector = RecordCollector() + collector.collect(unit.cursor) + missing = sorted(collector.incomplete.difference(forced)) + if not missing: + break + forced.extend(missing) + + errors = [diagnostic for diagnostic in unit.diagnostics if diagnostic.severity >= 3] + for diagnostic in errors[:50]: + location = diagnostic.location + print(f"Error: clang: {location.file}:{location.line}:{location.column}: {diagnostic.spelling}") + if len(errors) > 50: + print(f"Error: clang: {len(errors) - 50} more error(s) not shown") + + return collector.get_export(), collector.aliases, len(errors) + + def parse_layout_probe(self, source: str) -> dict[str, tuple[int, int]]: + """Size and offset of value in each probe struct of SrcInterface.get_layout_probe.""" + unit = Index.create().parse(main_file, args=self.args, unsaved_files=[(main_file, source)]) + layouts: dict[str, tuple[int, int]] = {} + for child in unit.cursor.get_children(): + if child.kind != CursorKind.STRUCT_DECL or not child.spelling.startswith(layout_probe_prefix): + continue + for field in child.type.get_fields(): + size, bit_offset = field.type.get_size(), field.get_field_offsetof() + if field.spelling == "value" and size > 0 and bit_offset >= 0: + layouts[child.spelling] = (size, bit_offset // 8) + return layouts diff --git a/scripts/ghidra/full_run.py b/scripts/ghidra/full_run.py new file mode 100644 index 0000000000..50404ce24a --- /dev/null +++ b/scripts/ghidra/full_run.py @@ -0,0 +1,28 @@ +# Runs rename.py and struct_import.py in sequence +# @category __UserScripts +# @menupath Tools.Scripts.ffxiv_full_run +# @runtime PyGhidra + +import os +import sys + +try: + working_directory = os.path.dirname(os.path.realpath(__file__)) +except NameError: + working_directory = os.path.dirname(str(sourceFile)) # noqa + +if working_directory not in sys.path: + sys.path.append(working_directory) + +from ghidra_logging import start_logging, stop_logging + +start_logging("full_run.py", monitor, isRunningHeadless()) # noqa + +print("Loading rename.py") +exec(open(os.path.join(working_directory, "rename.py")).read()) + +print("Loading struct_import.py") +exec(open(os.path.join(working_directory, "struct_import.py")).read()) + +analyzeChanges(currentProgram) # noqa +stop_logging() diff --git a/scripts/ghidra/ghidra_logging.py b/scripts/ghidra/ghidra_logging.py new file mode 100644 index 0000000000..76abc4a367 --- /dev/null +++ b/scripts/ghidra/ghidra_logging.py @@ -0,0 +1,110 @@ +"""Forwards stdout and stderr of a script run to Ghidra's log, which saves it to application.log.""" + +import sys + +from ghidra.app.script import GhidraScript +from ghidra.util import Msg + +try: + # what GhidraScript.println wraps its lines in, lets the log filter on script messages + from ghidra.app.script import ScriptMessage +except ImportError: + ScriptMessage = str + +ORIGINATOR = GhidraScript.class_ + + +class GhidraLogStream(object): + """Text stream that passes writes on to the stream it replaces and logs each full line.""" + + def __init__(self, target, script_name: str, is_stderr: bool, echo: bool): + self.target = target + self.script_name = script_name + self.is_stderr = is_stderr + self.echo = echo + self.buffer = "" + + def write(self, text: str) -> int: + if self.echo: + self.target.write(text) + self.buffer += text + *lines, self.buffer = self.buffer.split("\n") + for line in lines: + self.log(line) + return len(text) + + def flush(self): + if self.buffer: + self.log(self.buffer) + self.buffer = "" + if self.echo: + self.target.flush() + + def log(self, line: str): + line = line.rstrip("\r") + if not line.strip(): + return + message = ScriptMessage(f"{self.script_name}> {line}") + stripped = line.lstrip() + if self.is_stderr or stripped.startswith("Error"): + Msg.error(ORIGINATOR, message) + elif stripped.startswith("Warning"): + Msg.warn(ORIGINATOR, message) + else: + Msg.info(ORIGINATOR, message) + + def isatty(self) -> bool: + return False + + def __getattr__(self, name): + return getattr(self.target, name) + + +class _Redirect(object): + def __init__(self, monitor, stdout, stderr): + self.monitor = monitor + self.stdout = stdout + self.stderr = stderr + self.depth = 1 + + +_active: _Redirect | None = None + + +def start_logging(script_name: str, monitor, headless: bool): + """Sends stdout and stderr to Ghidra's log until stop_logging. + + Scripts exec'd by another script share its monitor and nest into its redirect. + """ + global _active + if _active is not None: + if _active.monitor is monitor: + _active.depth += 1 + return + _restore() + + stdout = GhidraLogStream(sys.stdout, script_name, False, not headless) + stderr = GhidraLogStream(sys.stderr, script_name, True, not headless) + _active = _Redirect(monitor, stdout, stderr) + sys.stdout = stdout + sys.stderr = stderr + + +def stop_logging(): + if _active is None: + return + _active.depth -= 1 + if _active.depth <= 0: + _restore() + + +def _restore(): + global _active + for stream in (_active.stdout, _active.stderr): + stream.flush() + # the script runner may have swapped the streams back already + if sys.stdout is _active.stdout: + sys.stdout = _active.stdout.target + if sys.stderr is _active.stderr: + sys.stderr = _active.stderr.target + _active = None diff --git a/scripts/ghidra/ghidra_wrapper.py b/scripts/ghidra/ghidra_wrapper.py new file mode 100644 index 0000000000..00f008d1dd --- /dev/null +++ b/scripts/ghidra/ghidra_wrapper.py @@ -0,0 +1,760 @@ +from ghidra.program.flatapi import FlatProgramAPI +from ghidra.program.model.data import ( + ArrayDataType, + BooleanDataType, + ByteDataType, + CategoryPath, + CharDataType, + DataType, + DataUtilities, + DoubleDataType, + EnumDataType, + FloatDataType, + FunctionDefinitionDataType, + IntegerDataType, + LongLongDataType, + ParameterDefinition, + ParameterDefinitionImpl, + ShortDataType, + SignedByteDataType, + StructureDataType, + UnionDataType, + UnsignedIntegerDataType, + UnsignedLongLongDataType, + UnsignedShortDataType, + VoidDataType, + WideCharDataType, +) +from ghidra.program.model.listing import Function, ParameterImpl, ReturnParameterImpl +from ghidra.program.model.symbol import SourceType +from java.util import ArrayList +from jpype import JArray, JByte, JLong + +from structs_schema import ( + DefinedStruct, + DefinedStructEnum, + DefinedStructExport, + DefinedStructFixedField, + DefinedStructFuncField, + DefinedStructFuncParam, + DefinedStructMemFunc, + DefinedStructStaticMember, + DefinedStructVFunc, + get_template_alias, +) +from data_schema import get_yaml, DefinedData, DefinedDataClassVtable +from clang_wrapper import ClangInterface, split_type_path + +from platform import system +from pathlib import Path +import os +import re + +DEFAULT_NAME_PREFIXES = ("FUN_", "LAB_", "SUB_", "LOC_", "DAT_") +# the fields SrcInterface.get_struct_pad fills the unknown parts of a struct with +PAD_FIELD_NAME = re.compile(r"field_[0-9A-F]+") + +def newest_version_folder(base: Path) -> Path: + """The highest version folder in base.""" + if not base.is_dir(): + raise RuntimeError(f"{base} is missing, select the folder msvc-wine's install.sh was run on") + versions = [p for p in base.iterdir() if p.is_dir() and re.fullmatch(r"[\d.]+", p.name)] + if not versions: + raise RuntimeError(f"No version folder in {base}") + return max(versions, key=lambda p: tuple(int(part) for part in p.name.split(".") if part)) + + +class GhidraInterface(object): + BADADDR = 0xFFFFFFFFFFFFFFFF + + data_yaml: DefinedData + parser_args: list[str] + # typedef name to the type behind it, see get_template_alias + template_aliases: dict[str, str] = {} + + def __init__(self, program, monitor): + """Takes currentProgram and monitor of the running script.""" + # the script globals (currentProgram, getFunctionAt, ...) only exist in the script itself + self.program = program + self.monitor = monitor + self.flat = FlatProgramAPI(program, monitor) + self.dtm = program.getDataTypeManager() + self.listing = program.getListing() + self.memory = program.getMemory() + self.symbols = program.getSymbolTable() + void_ptr = self.dtm.getPointer(VoidDataType.dataType) + self.primitives = { + "void": VoidDataType.dataType, + "bool": BooleanDataType.dataType, + "char": CharDataType.dataType, + "wchar_t": WideCharDataType.dataType, + "byte": ByteDataType.dataType, + "float": FloatDataType.dataType, + "double": DoubleDataType.dataType, + "int": IntegerDataType.dataType, + "unsigned int": UnsignedIntegerDataType.dataType, + "__int8": SignedByteDataType.dataType, + "__int16": ShortDataType.dataType, + "__int32": IntegerDataType.dataType, + "__int64": LongLongDataType.dataType, + "unsigned __int8": ByteDataType.dataType, + "unsigned __int16": UnsignedShortDataType.dataType, + "unsigned __int32": UnsignedIntegerDataType.dataType, + "unsigned __int64": UnsignedLongLongDataType.dataType, + "int8_t": SignedByteDataType.dataType, + "int16_t": ShortDataType.dataType, + "int32_t": IntegerDataType.dataType, + "int64_t": LongLongDataType.dataType, + "uint8_t": ByteDataType.dataType, + "uint16_t": UnsignedShortDataType.dataType, + "uint32_t": UnsignedIntegerDataType.dataType, + "uint64_t": UnsignedLongLongDataType.dataType, + # function pointers are exported without their signature + "__fastcall": void_ptr, + } + + def get_base_path(self): + return os.path.join( + os.path.dirname(os.path.realpath(__file__)), ".." + ) + + def get_data_path(self): + return os.path.join(self.get_base_path(), "data.yml") + + def load_data_yml(self): + with open(self.get_data_path()) as f: + self.data_yaml = get_yaml(f) + + def get_struct_path(self): + return os.path.join(self.get_base_path(), "structs.yml") + + def to_addr(self, ea: int): + return self.flat.toAddr(JLong(ea)) + + def check_cancelled(self): + if self.monitor.isCancelled(): + raise RuntimeError("Cancelled by user") + + # region types + + def get_category_path(self, type_name: str): + """The category a type lives in, ex. /Component/GUI.""" + category = CategoryPath.ROOT + for part in split_type_path(type_name)[:-1]: + category = CategoryPath(category, part) + return category + + def get_type_name(self, type_name: str) -> str: + return split_type_path(type_name)[-1] + + def get_named_type(self, name: str): + """Returns None if Ghidra does not know the type.""" + if name in self.primitives: + return self.primitives[name] + + if "<" in name and get_template_alias(name) in self.template_aliases: + # clang spells template names differently than the yml + name = self.template_aliases[get_template_alias(name)] + + dt = self.dtm.getDataType(self.get_category_path(name), self.get_type_name(name)) + if dt is None and "::" not in name: + # ex. windows types that came from an applied archive + found = ArrayList() + self.dtm.findDataTypes(name, found) + if found.size() > 0: + dt = found.get(0) + return dt + + def get_tinfo_from_type(self, raw_type: str, array_size=0): + """Returns None if the type could not be built.""" + type = raw_type.strip().rstrip("*").strip() + ptr_count = raw_type.count("*", len(type)) + + dt = self.get_named_type(type) + if dt is None: + if ptr_count == 0: + return None + # a pointer to something unknown is still a pointer + dt = VoidDataType.dataType + + for _ in range(ptr_count): + dt = self.dtm.getPointer(dt) + + if array_size > 0: + dt = ArrayDataType(dt, array_size, dt.getLength()) + + return dt + + def get_enum_id(self, name: str): + return self.get_named_type(name) + + def get_struct_id(self, name: str): + return self.get_named_type(name) + + def create_datatype(self, datatype): + """Add the type, replacing the one with the same path.""" + old = self.dtm.getDataType(datatype.getDataTypePath()) + if old is not None: + old.replaceWith(datatype) + return old + return self.dtm.addDataType(datatype, None) + + def create_enum(self, enum: DefinedStructEnum): + underlying = self.get_named_type(enum.underlying) + size = underlying.getLength() if underlying is not None else 4 + dt = EnumDataType(self.get_category_path(enum.type), self.get_type_name(enum.type), size) + for name, value in enum.values.items(): + if value >= 1 << 63: + # java longs are signed + value -= 1 << 64 + try: + dt.add(name, value) + except Exception as e: + print(f"Error: enum {enum.type} can not hold {name} = {value}: {e}") + self.create_datatype(dt) + + def create_struct(self, struct: DefinedStruct): + """The struct without its members, so it can be pointed to.""" + category, name = self.get_category_path(struct.type), self.get_type_name(struct.type) + if struct.union: + dt = UnionDataType(category, name) + else: + dt = StructureDataType(category, name, struct.size or 0) + self.create_datatype(dt) + + def create_struct_members(self, struct: DefinedStruct): + dt = self.get_named_type(struct.type) + if dt is None: + return + + for field in sorted(struct.fields, key=lambda fld: fld.offset): + if PAD_FIELD_NAME.fullmatch(field.name): + # undefined bytes are of more use in ghidra than a filler field + continue + + if isinstance(field, DefinedStructFuncField): + ft = self.dtm.getPointer(self.create_function_def(field, struct)) + else: + ft = self.get_tinfo_from_type(field.type) + if ft is None: + print(f"Error: field {field.name} of {struct.type} has unknown type {field.type!r}") + continue + + if isinstance(field, DefinedStructFixedField): + if int(field.size) < 1: + continue + ft = ArrayDataType(ft, int(field.size), ft.getLength() or -1) + + if struct.union: + dt.add(ft, ft.getLength(), field.name, "") + continue + + size = 0 if dt.isZeroLength() else dt.getLength() + end = field.offset + ft.getLength() + if size < end: + if struct.size: + print( + f"Error: field {field.name} (off=0x{field.offset:X} size=0x{ft.getLength():X}) not within struct {struct.type} (size=0x{size:X})" + ) + break + dt.growStructure(end - size) + + dt.replaceAtOffset(field.offset, ft, -1, field.name, "") + + def create_function_def(self, func: DefinedStructFuncField, struct: DefinedStruct): + """Signature of a function pointer field.""" + fd = FunctionDefinitionDataType( + CategoryPath(self.get_category_path(struct.type), self.get_type_name(struct.type)), func.name + ) + fd.setReturnType(self.get_tinfo_from_type(func.return_type) or DataType.DEFAULT) + args = [ + ParameterDefinitionImpl(arg.name, self.get_tinfo_from_type(arg.type) or DataType.DEFAULT, None) + for arg in func.parameters + ] + fd.setArguments(JArray(ParameterDefinition)(args)) + return fd + + def select_parser(self, msvc_path: str | None = None): + """msvc_path is where msvc-wine was installed, not needed on windows.""" + includes = [] + if system() != "Windows": + if msvc_path is None: + raise RuntimeError("Has to have msvc includes for the parsers to work with the project") + path = Path(msvc_path) + vc_path = newest_version_folder(path / "vc" / "tools" / "msvc") / "include" + window_kit = newest_version_folder(path / "kits" / "10" / "include") + # as system headers like clang adds them on windows, otherwise the warnings it treats as errors in them count + for folder in (vc_path, window_kit / "ucrt", window_kit / "um", window_kit / "shared"): + includes += ["-isystem", folder.as_posix()] + + self.parser_args = [ + "-x", "c++", "-target", "x86_64-pc-windows-msvc", "-fms-compatibility", "-fms-extensions", + "-U__int64", "-std=c++17", "-ferror-limit=0", + # the libclang package is older than what newer msvc headers expect + "-D_ALLOW_COMPILER_AND_STL_VERSION_MISMATCH", "-D_MSVC_STL_USE_ABORT_AS_DOOM_FUNCTION", + *includes, + ] + + def get_probe_layouts(self, source: str) -> dict[str, tuple[int, int]]: + """Size and offset of value in each probe struct of SrcInterface.get_layout_probe.""" + return ClangInterface( + self.parser_args, os.environ.get("LIBCLANG_LIBRARY_FILE") + ).parse_layout_probe(source) + + def parse_string(self, decl: str, vtables: list[str]) -> int: + """Import the declarations as clang lays them out, returns the error count.""" + export, self.template_aliases, error_count = ClangInterface( + self.parser_args, os.environ.get("LIBCLANG_LIBRARY_FILE") + ).parse_string(decl) + if error_count > 0: + print(f"Error: clang parser reported {error_count} error(s) while importing declarations") + else: + print("Clang parser imported declarations without errors") + + self.import_structs(export) + + missing = [vtable for vtable in vtables if self.get_named_type(vtable) is None] + for vtable in missing: + print(f"Error: vtable '{vtable}' not found in the data types") + print(f"Imported {len(vtables) - len(missing)}/{len(vtables)} vtables ({len(missing)} failed)") + + return error_count + + def import_structs(self, export: DefinedStructExport) -> int: + """Import the structs in dependency order, returns the failure count.""" + steps = [ + ("Creating enums", export.enums, self.create_enum), + ("Creating structs", export.structs, self.create_struct), + ("Creating members for structs", export.structs, self.create_struct_members), + ] + error_count = 0 + for message, items, step in steps: + print(message) + self.monitor.setMessage(message) + for item in items: + self.check_cancelled() + try: + step(item) + except Exception as e: + error_count += 1 + print(f"Error: {message.lower()} failed on {item.type}: {e}") + + if error_count > 0: + print(f"Error: {error_count} error(s) while importing declarations") + else: + print("Imported declarations without errors") + return error_count + + # endregion + + # region functions + + def get_func(self, ea: int, create: bool = False): + """Function starting at ea.""" + addr = self.to_addr(ea) + func = self.flat.getFunctionAt(addr) + if func is None and create: + if self.flat.getInstructionAt(addr) is None: + self.flat.disassemble(addr) + func = self.flat.createFunction(addr, None) + return func + + def has_default_name(self, ea: int) -> bool: + """Only true for functions Ghidra named itself.""" + func = self.get_func(ea, True) + return func is not None and func.getSymbol().getSource() == SourceType.DEFAULT + + def apply_func_type(self, ea: int, func_name: str, return_type: str, parameters: list[DefinedStructFuncParam]) -> bool: + """Returns False if one of the types is not known to Ghidra.""" + func = self.get_func(ea, True) + if func is None: + print(f"Error: {func_name} @ {ea:#x} is not a function") + return False + + rettype = self.get_tinfo_from_type(return_type) + if rettype is None: + print(f"Error: {func_name} @ {ea:#x} has unknown return type {return_type!r}") + return False + + args = ArrayList() + for param in parameters: + arg_type = self.get_tinfo_from_type(param.type) + if arg_type is None: + print(f"Error: {func_name} @ {ea:#x} has unknown type {param.type!r} on parameter {param.name!r}") + return False + args.add(ParameterImpl(param.name, arg_type, self.program)) + + try: + func.updateFunction( + "__fastcall", + ReturnParameterImpl(rettype, self.program), + args, + Function.FunctionUpdateType.DYNAMIC_STORAGE_ALL_PARAMS, + False, + SourceType.USER_DEFINED, + ) + except Exception as e: + print(f"Error: {func_name} @ {ea:#x} failed to apply its function type: {e}") + return False + return True + + def get_vft_from_data(self, class_name: str) -> DefinedDataClassVtable | None: + class_data = self.data_yaml.classes.get(class_name) + if class_data is None or not class_data.vtbls: + return None + + return class_data.vtbls[0] + + def get_vfunc_ea_from_data(self, class_name: str, offset: int) -> int: + vtable = self.get_vft_from_data(class_name) + if vtable is None: + return self.BADADDR + + vtable_ea = vtable.ea + self.get_image_base() - 0x140000000 + try: + return self.get_qword(vtable_ea + offset) + except Exception: + return self.BADADDR + + def set_func_name(self, ea, name, cmt=None, unique=False): + """With unique only default names are replaced, a taken name gets a number.""" + if unique: + if not self.has_default_name(ea): + return + base_name, count = name, 1 + while self.get_func_ea_by_name(name) not in (self.BADADDR, ea): + count += 1 + name = f"{base_name}{count}" + print(name) + + func = self.get_func(ea, True) + if func is None: + self.set_addr_name(ea, name) + return + func.setName(name, SourceType.USER_DEFINED) + if cmt is not None: + func.setComment(cmt) + + def get_func_name(self, ea: int) -> str: + """Name of the function containing ea.""" + func = self.flat.getFunctionContaining(self.to_addr(ea)) + return func.getName() if func is not None else "" + + def get_func_start(self, ea: int) -> int: + """Start of the function containing ea.""" + return self.flat.getFunctionContaining(self.to_addr(ea)).getEntryPoint().getOffset() + + def get_func_ea_by_name(self, name: str) -> int: + for symbol in self.symbols.getGlobalSymbols(name): + return symbol.getAddress().getOffset() + return self.BADADDR + + def get_func_xrefs(self, ea: int) -> list[int]: + """The addresses calling ea.""" + return [ + xref.getFromAddress().getOffset() + for xref in self.flat.getReferencesTo(self.to_addr(ea)) + if xref.getReferenceType().isCall() + ] + + def get_func_ea_by_sig(self, pattern: str) -> int: + ea = self.search_binary(0, pattern) + if ea == self.BADADDR: + return ea + + if self.get_func(ea) is not None: + return ea + + insn = self.flat.getInstructionAt(self.to_addr(ea)) + if insn is None: + return self.BADADDR + + flow = insn.getFlowType() + if flow.isCall() or flow.isJump(): + flows = insn.getFlows() + if len(flows) != 1: + print(f"Error: Can't follow {insn} @ {ea:#x}") + return self.BADADDR + return flows[0].getOffset() + + return self.BADADDR + + def update_member_func(self, member_func: DefinedStructMemFunc, struct: DefinedStruct) -> None: + func_name = f"{struct.type}.{member_func.name}" + ea = self.get_func_ea_by_name(func_name) + if ea == self.BADADDR: + ea = self.get_func_ea_by_sig(member_func.signature) + if ea == self.BADADDR: + print( + f"Error: {func_name} not found bad sig? {member_func.signature}" + ) + return + if self.has_default_name(ea): + self.set_func_name(ea, func_name) + self.apply_func_type(ea, func_name, member_func.return_type, member_func.parameters) + + def update_virt_func(self, virt_func: DefinedStructVFunc, struct: DefinedStruct): + func_name = f"{struct.type}.{virt_func.name}" + + ea = self.get_func_ea_by_name(func_name) + if ea == self.BADADDR: + if self.get_vft_from_data(struct.type) is None: + return + ea = self.get_vfunc_ea_from_data(struct.type, virt_func.offset) + if ea in (0, self.BADADDR): + print(f"Error: {func_name} not found and its VFT slot could not be resolved") + return + + normalized_name = self.get_addr_name(ea).removeprefix("thunk_") + + if normalized_name == "_purecall": + return + + primary_base = self.get_vft_from_data(struct.type).base + if primary_base: + base_ea = self.get_vfunc_ea_from_data(primary_base, virt_func.offset) + if base_ea == ea: + return + + if ( + normalized_name.endswith("." + virt_func.name) + and not normalized_name.startswith(struct.type + ".") + ): + return + self.apply_func_type(ea, func_name, virt_func.return_type, virt_func.parameters) + + def get_static_member_ea(self, static_member: DefinedStructStaticMember) -> int: + """Where the static member's signature resolves to, BADADDR if not found.""" + ea = self.search_binary(0, static_member.signature) + if ea == self.BADADDR: + return ea + for follows in static_member.relative_offsets: + ea = ea + follows + ea = ea + 4 + self.memory.getInt(self.to_addr(ea)) + return ea + + def update_static_member(self, static_member: DefinedStructStaticMember, struct: DefinedStruct): + ea = self.get_static_member_ea(static_member) + if ea == self.BADADDR: + print(f"Error: {static_member.signature} not found something is wrong") + return + addr = self.to_addr(ea) + data = self.listing.getDefinedDataContaining(addr) + if data is not None and data.getAddress().getOffset() != ea and static_member.return_type != struct.type: + # points into an already typed item (ex. a field of the struct instance) + return + return_type = static_member.return_type + if static_member.is_pointer: + return_type = return_type + "*" + dt = self.get_tinfo_from_type(return_type) + if dt is None: + print(f"Error: static member of {struct.type} @ {ea:#x} has unknown type {return_type!r}") + else: + try: + DataUtilities.createData( + self.program, addr, dt, -1, DataUtilities.ClearDataMode.CLEAR_ALL_CONFLICT_DATA + ) + except Exception as e: + print(f"Error: static member of {struct.type} @ {ea:#x} failed to apply {return_type!r}: {e}") + self.set_addr_name(ea, f"g_{struct.type}_{static_member.name}") + + # endregion + + # region addresses + + def get_image_base(self) -> int: + return self.program.getImageBase().getOffset() + + def is_offset(self, ea: int) -> bool: + data = self.flat.getDataAt(self.to_addr(ea)) + return bool(data) and data.isPointer() + + def xrefs_to(self, ea: int) -> list[int]: + return [xref.getFromAddress().getOffset() for xref in self.flat.getReferencesTo(self.to_addr(ea))] + + def get_qword(self, ea: int) -> int: + return self.memory.getLong(self.to_addr(ea)) & 0xFFFFFFFFFFFFFFFF + + def get_dword(self, ea: int) -> int: + return self.memory.getInt(self.to_addr(ea)) & 0xFFFFFFFF + + def get_addr_name(self, ea: int) -> str: + addr = self.to_addr(ea) + sym = self.flat.getSymbolAt(addr) + if not sym: + # code only reachable through a vtable is not always disassembled, it does follow the padding of the previous function + try: + is_padded = self.memory.getByte(addr.subtract(1)) & 0xFF == 0xCC + except Exception: + return "" + if is_padded: + self.get_func(ea, True) + sym = self.flat.getSymbolAt(addr) + if not sym: + return "" + return sym.getName(True) + + def set_addr_name(self, ea: int, name: str) -> bool: + self.flat.createLabel(self.to_addr(ea), name, True, SourceType.ANALYSIS) + return True + + def get_comment(self, ea: int): + return self.flat.getEOLComment(self.to_addr(ea)) + + def set_comment(self, ea: int, comment: str): + # the callers append to what is already there, keeping the first one makes a rerun not stack them + if self.flat.getEOLComment(self.to_addr(ea)) is None: + self.flat.setEOLComment(self.to_addr(ea), comment) + + def get_string_from_ptr(self, ea: int) -> str: + val = self.get_string(ea) + if val is None: + val = self.get_string(self.get_qword(ea)) + return val + + def get_string(self, ea: int) -> str | None: + """Returns None if there is no printable string at ea.""" + chars = bytearray() + try: + while len(chars) < 256: + char = self.memory.getByte(self.to_addr(ea + len(chars))) & 0xFF + if char == 0: + break + if not 0x20 <= char < 0x7F: + return None + chars.append(char) + except Exception: + return None + return chars.decode("ascii") if chars else None + + # endregion + + # region instructions + + def heads_before(self, ea: int, count: int): + """The instructions before ea, closest first.""" + insn = self.listing.getInstructionContaining(self.to_addr(ea)) + for _ in range(count): + insn = insn.getPrevious() + if insn is None: + return + yield insn.getAddress().getOffset() + + def heads_until_call(self, ea: int, func_name: str): + """The instructions from ea up to the call of the named function.""" + target = self.get_func_ea_by_name(func_name) + insn = self.listing.getInstructionAt(self.to_addr(ea)) + while insn is not None: + if insn.getFlowType().isCall() and any(flow.getOffset() == target for flow in insn.getFlows()): + return + yield insn.getAddress().getOffset() + insn = insn.getNext() + + def get_registers(self, eas, mnemonics=None) -> dict[str, dict[str, object]]: + """What the instructions write to each register, later ones win.""" + regs = {} + for ea in eas: + insn = self.listing.getInstructionAt(self.to_addr(ea)) + if insn is None or insn.getNumOperands() < 2: + continue + if mnemonics is not None and str(insn.getMnemonicString()).lower() not in mnemonics: + continue + val_text = str(insn.getDefaultOperandRepresentation(1)).lower() + val = 0 + refs = insn.getOperandReferences(1) + if len(refs) > 0: + val = refs[0].getToAddress().getOffset() + if refs[0].getReferenceType().isRead(): + # the register is loaded from memory, ex. a pointer to the class name + val_text = self.get_string_from_ptr(val) + elif insn.getScalar(1) is not None: + val = insn.getScalar(1).getUnsignedValue() + regs[str(insn.getDefaultOperandRepresentation(0)).lower()] = {'val_text': val_text, 'val': val} + return regs + + # endregion + + # region naming + + def format_class_name_for_vtbl(self, class_name: str) -> str: + return f"vtbl_{class_name}" + + def format_class_name_for_secondary_vtbl(self, class_name: str, secondary_name: str) -> str: + return f"vtbl_{class_name}___{secondary_name}" + + def format_class_name(self, class_name: str) -> str: + return class_name + + def format_vfunc_name(self, ea: int, current_func_name: str, proposed_func_name: str, class_name: str, base_class_names: list[str]) -> str | None: + current_func_name = current_func_name.removeprefix("thunk_") + + # Previously renamed as a vfunc + if current_func_name.startswith(class_name): + # return the proposed func name in case it was updated since last run + current_class_name = current_func_name.rsplit(".", 1)[0] + return f"{current_class_name}.{proposed_func_name}" + + # This should have been handled in the base class + if any(current_func_name.startswith(name) for name in base_class_names): + return "" + + if current_func_name.startswith(DEFAULT_NAME_PREFIXES): + return f"{class_name}.{proposed_func_name}" + + # Name it later in a child class when it gets overridden + if current_func_name == "_purecall": + return "" + + return None + + def format_func_name(self, ea: int, current_func_name: str, proposed_func_name: str, class_name: str) -> str | None: + # override default thunk names and thunks of already named funcs with the default + func = self.get_func(ea) + if func and func.isThunk(): + current_func_name = f"FUN_{ea:x}" + + proposed_qualified_func_name = f"{class_name}.{proposed_func_name}" + if current_func_name == proposed_qualified_func_name: + return "" + + if current_func_name.startswith(DEFAULT_NAME_PREFIXES): + return proposed_qualified_func_name + + return None + + # endregion + + # region search + + def get_search_range(self): + """Only .text if the program has one, the signatures are all code.""" + block = self.memory.getBlock(".text") + if block is not None: + return block.getStart(), block.getEnd() + return self.memory.getMinAddress(), self.memory.getMaxAddress() + + def search_binary(self, ea: int, pattern: str) -> int: + """The pattern is space separated hex bytes, ? or ?? for a wildcard.""" + start, end = self.get_search_range() + if ea > end.getOffset(): + return self.BADADDR + if ea > start.getOffset(): + start = self.to_addr(ea) + + tokens = pattern.split() + values = bytes(0 if "?" in token else int(token, 16) for token in tokens) + masks = bytes(0 if "?" in token else 0xFF for token in tokens) + found = self.memory.findBytes( + start, end, JArray(JByte)(values), JArray(JByte)(masks), True, self.monitor + ) + return found.getOffset() if found is not None else self.BADADDR + + def search_binary_all(self, pattern: str, ea: int = 0): + """Every address matching the pattern, starting from ea.""" + ea = self.search_binary(ea, pattern) + while ea != self.BADADDR: + self.check_cancelled() + yield ea + ea = self.search_binary(ea + 1, pattern) + + # endregion diff --git a/scripts/ghidra/rename.py b/scripts/ghidra/rename.py new file mode 100644 index 0000000000..80d49f99dd --- /dev/null +++ b/scripts/ghidra/rename.py @@ -0,0 +1,74 @@ +# Names the globals, functions, vtables and such of data.yml +# @category __UserScripts +# @menupath Tools.Scripts.ffxiv_rename +# @runtime PyGhidra + +from os.path import join, dirname +import sys + +try: + script_dir = dirname(__file__) +except NameError: + script_dir = dirname(str(sourceFile)) # noqa + +for import_path in (script_dir, join(script_dir, "..", "shared")): + if import_path not in sys.path: + sys.path.append(import_path) + +from ghidra_wrapper import GhidraInterface +from ghidra_logging import start_logging, stop_logging +from class_factory import apply_data + +lua_namespace = "Client::Game::Event::Lua" + +def apply_lua_funcs(): + """Needs the LuaState names, so apply_data has to run first.""" + string_eas = ghidra.get_func_xrefs(ghidra.get_func_ea_by_name('Common::Lua::LuaState.SetStringField')) + func_eas = ghidra.get_func_xrefs(ghidra.get_func_ea_by_name('Common::Lua::LuaState.SetFunctionField')) + + string_eas = [ghidra.get_func_start(x) for x in string_eas] + + eas = {} + for x in func_eas: + regs = ghidra.get_registers(ghidra.heads_before(x, 4)) + regs['r8'] = ghidra.get_string_from_ptr(regs['r8']['val']) + eas.setdefault(ghidra.get_func_start(x), {'calls': []})['calls'].append({'off': x, 'regs': regs}) + + eas = {k: v for k,v in eas.items() if k in string_eas} + + for func_ea in eas: + eas[func_ea]['regs'] = ghidra.get_registers(ghidra.heads_until_call(func_ea, 'Common::Lua::LuaState.SetStringField'), ('mov',)) + + for func_ea in eas: + eas[func_ea]['name'] = eas[func_ea]['regs'][eas[func_ea]['regs']['r9']['val_text']]['val_text'] + eas[func_ea]['name_val'] = eas[func_ea]['regs'][eas[func_ea]['regs']['r9']['val_text']]['val'] + + for func_ea in eas: + if eas[func_ea]['name'] == 'r8': + eas[func_ea]['name'] = ghidra.get_func_name(func_ea).split('.')[0].split(':')[-1] + eas[func_ea]['name_val'] = func_ea + if eas[func_ea]['name'] == 'rax': + eas[func_ea]['name_val'] = ghidra.get_func_xrefs(func_ea)[0] + eas[func_ea]['name'] = ghidra.get_func_name(eas[func_ea]['name_val']).split('.')[0].split(':')[-1] + + for func_ea in eas: + name = f"{lua_namespace}::{eas[func_ea]['name']}" + ghidra.set_func_name(func_ea, f"{name}.SetupLuaClass") + for call in eas[func_ea]['calls']: + regs = call['regs'] + func_name = regs['r8'] + ghidra.set_func_name(regs['r9']['val'], f"{name}.{func_name}", unique=True) + +start_logging("rename.py", monitor, isRunningHeadless()) # noqa + +ghidra = GhidraInterface(currentProgram, monitor) # noqa +print("Loading data.yml") +ghidra.load_data_yml() + +print("Executing") +apply_data(ghidra, ghidra.data_yaml) +print("Naming lua functions") +apply_lua_funcs() +print("Done") + +stop_logging() diff --git a/scripts/ghidra/struct_import.py b/scripts/ghidra/struct_import.py new file mode 100644 index 0000000000..14c1e19a31 --- /dev/null +++ b/scripts/ghidra/struct_import.py @@ -0,0 +1,220 @@ +# Imports the Excel sheets and structs.yml as data types, and names and types the functions using them +# @category __UserScripts +# @menupath Tools.Scripts.ffxiv_struct_import +# @runtime PyGhidra + +from os.path import join, dirname +import sys + +try: + script_dir = dirname(__file__) +except NameError: + script_dir = dirname(str(sourceFile)) # noqa + +for import_path in (script_dir, join(script_dir, "..", "shared")): + if import_path not in sys.path: + sys.path.append(import_path) + +from java.util import ArrayList +from ghidra.util.exception import CancelledException + +from lumina_wrapper import get_excel_header_files, find_game_path, is_game_path +from exdschema_wrapper import ( + create_struct_from_header_and_schema, + get_exdschema_data, + exd_comment_patterns, + exd_func_patterns, +) +from src_wrapper import SrcInterface +from ghidra_wrapper import GhidraInterface +from ghidra_logging import start_logging, stop_logging +from structs_schema import get_yaml, DefinedStructFuncParam + +start_logging("struct_import.py", monitor, isRunningHeadless()) # noqa + +src = SrcInterface() +# one filler field per gap, clang gets slow on a struct with a field for every unknown qword +src.full_padding = False +ghidra = GhidraInterface(currentProgram, monitor) # noqa +print("Loading data.yml") +ghidra.load_data_yml() + +if not askYesNo( # noqa + "Check version match.", + f"Loaded with ver: {ghidra.data_yaml.version}\n" + "Check if this matches with what is reported in the version files of the game.", +): + raise RuntimeError("Version is not correct please update repo") + +msvc_path = None +if sys.platform != "win32": + msvc_path = str(askDirectory( # noqa + "Select msvc, the folder mstorsjo/msvc-wine install.sh was run on", "Select" + )) +ghidra.select_parser(msvc_path) + +def ask_game_path() -> str: + game_path = find_game_path() + if game_path is None: + game_path = str(askDirectory( # noqa + "Select the game install, the folder holding both game and boot", "Select" + )) + if not is_game_path(game_path): + raise RuntimeError(f"No game or boot folder in {game_path}, select the folder holding both") + print(f"Using game install at {game_path}") + return game_path + +function_types = {"virtual": "Virtual functions", "member": "Member functions"} +try: + _selected = askChoices( # noqa + "Select function types to update", + "Types to update", + ArrayList(list(function_types.keys())), + ArrayList(list(function_types.values())), + ) + _function_update_options = {key: key in _selected for key in function_types} +except CancelledException: + _function_update_options = { + "virtual": False, + "member": False, + } + +print("Grabbing ExdSchema and Lumina") + +excel_header_files = get_excel_header_files(ghidra.data_yaml.version, ask_game_path) +exdschema = get_exdschema_data("latest") + +excel_structs, excel_map = create_struct_from_header_and_schema( + excel_header_files, exdschema +) + +exd_map = {excel_map[k]: k for k in excel_map} + +excel_header, excel_vtables = src.build_export_string(excel_structs, 4, ghidra.get_probe_layouts) + + +print("Parsing structs.yml") + +with open(ghidra.get_struct_path()) as f: + import_structs = get_yaml(f) + +struct_header, struct_vtables = src.build_export_string(import_structs, 1, ghidra.get_probe_layouts, excel_header) + +print("Parsing combined Excel and FFXIV declarations") +ghidra.parse_string( + f"""{src.get_predef_header()} +{excel_header} +{struct_header} +""", + [*excel_vtables, *struct_vtables], +) + +def do_pattern(pattern, suffix, sheetSearchPattern, sheetSearchOffset): + for ea in ghidra.search_binary_all(pattern, 1): + if sheetSearchPattern == None: + sheetEa = ea + sheetSearchOffset + else: + sheetEa = ( + ghidra.search_binary(ea, sheetSearchPattern) + + sheetSearchOffset + ) + sheetIdx = ghidra.get_dword(sheetEa) + + if ghidra.has_default_name(ea): + if exd_map.get(sheetIdx) == None: + print( + f"Func @ 0x{ea:X} references unknown sheet {sheetIdx}! sheet dword offset = 0x{sheetEa:X}" + ) + continue + + sheetName = exd_map[sheetIdx] + + if suffix == None: + suffix = "" + + fnName = "Component::Exd::ExdModule.Get%s%s" % ( + exd_map[sheetIdx], + suffix, + ) + + uniquifier = 0 + while True: + uniqueName = fnName + (f"_{uniquifier}" if uniquifier > 0 else "") + + if ghidra.get_func_ea_by_name(uniqueName) == ghidra.BADADDR and uniquifier > 0: + fnName = uniqueName + break + + uniquifier += 1 + + ghidra.set_func_name( + ea, fnName, "Sheet: {0} ({1})".format(sheetName, sheetIdx) + ) + + if suffix == "SheetIndex": + continue + + sheetType = f"Component::Exd::Sheets::{exd_map[sheetIdx]}*" + if suffix == "RowAndSubRowId": + parameters = [ + DefinedStructFuncParam("rowId", "unsigned int"), + DefinedStructFuncParam("subRowId", "unsigned int"), + ] + elif suffix == "RowCount": + parameters = [] + else: + parameters = [DefinedStructFuncParam("rowIdOrIndex", "unsigned int")] + + ghidra.apply_func_type(ea, fnName, sheetType, parameters) + + +print("Fixing up Excel functions") + +for pattern in exd_func_patterns: + suffix, search = exd_func_patterns[pattern] + if suffix is not None: + print(f"Finding exd funcs of {suffix}... please wait.") + searchPattern, searchOffset = search + do_pattern(pattern, suffix, searchPattern, searchOffset) + + +for pattern_key in exd_comment_patterns: + for ea in list(ghidra.search_binary_all(pattern_key)): + sheetIdx = ghidra.get_dword(ea + exd_comment_patterns[pattern_key]) + sheetName = exd_map.get(sheetIdx) + if sheetName is None: + print(f"Sheet getter @ 0x{ea:X} references unknown sheet {sheetIdx}!") + continue + ghidra.set_comment(ea, "Sheet: {0} ({1})".format(sheetName, sheetIdx)) + +if _function_update_options["virtual"]: + for struct in import_structs.structs: + if struct.virtual_functions: + print(f"Updating virtual functions for {struct.type}") + for virt_func in struct.virtual_functions: + if virt_func is None: + continue + + if virt_func.return_type is not None and virt_func.parameters is not None: + ghidra.update_virt_func(virt_func, struct) + +print("Fixing up CS defined functions") + +if _function_update_options["member"]: + for struct in import_structs.structs: + if struct.member_functions != []: + print(f"Updating member functions for {struct.type}") + for member_func in struct.member_functions: + ghidra.update_member_func(member_func, struct) + + if struct.static_member_functions: + print(f"Updating static member functions for {struct.type}") + for member_func in struct.static_member_functions: + ghidra.update_member_func(member_func, struct) + + if struct.static_members: + print(f"Updating static members for {struct.type}") + for member in struct.static_members: + ghidra.update_static_member(member, struct) + +stop_logging() diff --git a/scripts/ida/data_struct_matcher.py b/scripts/ida/data_struct_matcher.py new file mode 100644 index 0000000000..dbb66a3ab5 --- /dev/null +++ b/scripts/ida/data_struct_matcher.py @@ -0,0 +1,28 @@ +from os.path import join, dirname +import sys + +sys.path.append(join(dirname(__file__), "..", "shared")) + +from ida_wrapper import IdaInterface +from structs_schema import get_yaml + +ida = IdaInterface() +print("Loading data.yml") +ida.load_data_yml() + +print("Parsing structs.yml") +with open(ida.get_struct_path()) as f: + structs = {struct.type: struct for struct in get_yaml(f).structs} + +# data.yml addresses are relative to the standard image base +rebase_offset = ida.get_image_base() - 0x140000000 + +for class_name, class_data in ida.data_yaml.classes.items(): + struct = structs.get(class_name) + if struct is None or struct.static_members is None: + continue + for static_member in struct.static_members: + ea = ida.get_static_member_ea(static_member) + instance = next((inst for inst in class_data.instances if inst.ea + rebase_offset == ea), None) + if instance is not None and static_member.is_pointer != instance.pointer: + print(f"Found pointer discrepancy at {ea:X} on {class_name}") diff --git a/scripts/ida/form_builder.py b/scripts/ida/form_builder.py new file mode 100644 index 0000000000..d89c6ca370 --- /dev/null +++ b/scripts/ida/form_builder.py @@ -0,0 +1,107 @@ +import ida_kernwin +from webbrowser import open + +F = ida_kernwin.Form + +def build_spec(fields, change_cb): + pad = max((len(f.get("label", "")) for f in fields), default=0) + lines = ["STARTITEM 0", "BUTTON YES* Run", "BUTTON CANCEL Cancel"] + need_first_space = True + ctrls = {"FormChangeCb": F.FormChangeCb(change_cb)} + + prepend_line = "" + for fld in fields: + t, key = fld["type"], fld.get("key") + label = fld.get("label", "").ljust(pad) + tip = f"#{fld['tip']}#" if fld.get("tip") else "" + + if t == "space": + lines.append("") + if need_first_space: + need_first_space = False + lines.append("{FormChangeCb}") + elif t == "label": + label_text = fld["text"] + if "prepend" in fld and fld["prepend"]: + prepend_line = f"{prepend_line}{label_text}" + else: + lines.append(label_text) + elif t == "folder": + lines.append(f"{prepend_line}<{tip}{label}:{{{key}}}>") + ctrls[key] = F.DirInput(swidth=fld.get("width", 60), + value=fld.get("value", "")) + elif t == "text": + lines.append(f"{prepend_line}<{tip}{label}:{{{key}}}>") + ctrls[key] = F.StringInput(swidth=fld.get("width", 30), + value=fld.get("value", "")) + elif t == "number": + lines.append(f"{prepend_line}<{tip}{label}:{{{key}}}>") + ctrls[key] = F.NumericInput(tp=F.FT_DEC, value=fld.get("value", 0)) + elif t == "dropdown": + lines.append(f"{prepend_line}<{tip}{label}:{{{key}}}>") + ctrls[key] = F.DropdownListControl(items=fld["items"], readonly=True, + selval=fld.get("selval", 0)) + elif t == "button": + on_click = fld["on_click"] + lines.append(f"{prepend_line}<{tip}{label}:{{{key}}}>") + ctrls[key] = F.ButtonInput(lambda code, fn=on_click: fn()) + + elif t in ("checks", "radios"): + names = [] + items = fld["items"] + for i, (ikey, ilabel) in enumerate(items): + name = f"{key}__{ikey}" + names.append(name) + last = (i == len(items) - 1) + if last: + end = f">{{{key}}}>" if t == "checks" else f">{{{key}}}>" + else: + end = ">" + lines.append(f"{prepend_line}<{ilabel}:{{{name}}}{end}") + ctrls[name] = F.Control() if False else None + for n in names: + ctrls.pop(n, None) + if t == "checks": + ctrls[key] = F.ChkGroupControl(tuple(names)) + else: + ctrls[key] = F.RadGroupControl(tuple(names)) + else: + raise ValueError(f"Unknown field type: {t}") + + if t != "label": + prepend_line = "" + + if need_first_space: + raise ValueError(f"Did not receive a line of type space") + + return "\n".join(lines) + "\n", ctrls + +class DynamicForm(F): + def __init__(self, fields, on_change=None): + self.fields = fields + self._on_change_cb = on_change + template, ctrls = build_spec(fields, self._on_change) + F.__init__(self, template, ctrls) + + def _on_change(self, fid): + if self._on_change_cb: + self._on_change_cb(self, fid) + return 1 + + def results(self): + """All values keyed by each field's key.""" + out = {} + for fld in self.fields: + t, key = fld["type"], fld.get("key") + if t in ("folder", "text", "number"): + out[key] = getattr(self, key).value + elif t == "dropdown": + v = getattr(self, key).value + out[key] = fld["items"][v] if isinstance(v, int) else v + elif t == "checks": + mask = getattr(self, key).value + out[key] = {ik: bool(mask & (1 << i)) + for i, (ik, _) in enumerate(fld["items"])} + elif t == "radios": + out[key] = fld["items"][getattr(self, key).value][0] + return out \ No newline at end of file diff --git a/scripts/ida/full_run.py b/scripts/ida/full_run.py new file mode 100644 index 0000000000..c3293e5496 --- /dev/null +++ b/scripts/ida/full_run.py @@ -0,0 +1,17 @@ +import os +import idautils +import ida_auto +import idc + +working_directory = os.path.dirname(os.path.realpath(__file__)) + +print("Loading rename.py") +exec(open(os.path.join(working_directory, "rename.py")).read()) + +print("Loading struct_import.py") +exec(open(os.path.join(working_directory, "struct_import.py")).read()) + +for s in idautils.Segments(): + start = idc.get_segm_start(s) + end = idc.get_segm_end(s) + ida_auto.plan_range(start, end) \ No newline at end of file diff --git a/scripts/ida/ida_wrapper.py b/scripts/ida/ida_wrapper.py new file mode 100644 index 0000000000..cf2e735dd2 --- /dev/null +++ b/scripts/ida/ida_wrapper.py @@ -0,0 +1,602 @@ +import idaapi +import idc +import idautils +import ida_bytes +import ida_search +import ida_ida +import ida_idp +import ida_typeinf +import ida_name +import ida_funcs +import ida_srclang +import ida_hexrays +from structs_schema import ( + DefinedStruct, + DefinedStructFuncParam, + DefinedStructMemFunc, + DefinedStructStaticMember, + DefinedStructVFunc, + get_template_alias, +) +from form_builder import DynamicForm +from data_schema import get_yaml, DefinedData, DefinedDataClassVtable +from src_wrapper import layout_probe_prefix + +from platform import system +from pathlib import Path +from webbrowser import open as webopen +import os + +def pick_folder(form_content: list[dict[str, any]]) -> str | None: + f = DynamicForm(form_content) + f.Compile() + try: + if f.Execute() == 1: # 1 = OK, 0 = Cancel + return f.folder.value or None + return None + finally: + f.Free() + +fields = [ + { "type": "label", "text": "Select msvc"}, + { "type": "space" }, + { "type": "label", "text": "It should be located where you used:", "prepend": True}, + { "type": "button", "key": "msvcdownload", "label": "mstorsjo/msvc-wine", "on_click": lambda: webopen("https://github.com/mstorsjo/msvc-wine")}, + { "type": "space" }, + { "type": "folder", "key": "folder", "label": "Folder", "value": "", "tip": "Pick the folder where ./vsdownload.sh was run" } +] + +def first_subfolder(base: Path) -> Path | None: + for p in sorted(base.iterdir()): + if p.is_dir(): + return p + return None + +def _get_ida_major() -> int: + return idaapi.IDA_SDK_VERSION // 100 + +if _get_ida_major() < 9: + raise "Unsupported IDA version used. Need to be on IDA 9 or newer." + +class IdaInterface(object): + BADADDR = idc.BADADDR + + data_yaml: DefinedData + + def get_base_path(self): + return os.path.join( + os.path.dirname(os.path.realpath(__file__)), ".." + ) + + def get_data_path(self): + return os.path.join(self.get_base_path(), "data.yml") + + def load_data_yml(self): + with open(self.get_data_path()) as f: + self.data_yaml = get_yaml(f) + + def get_struct_path(self): + return os.path.join(self.get_base_path(), "structs.yml") + + def get_enum_id(self, name: str): + return idc.get_enum(name) + + def get_struct_id(self, name: str) -> int: + return ida_typeinf.get_named_type_tid(name) + + def get_named_type(self, name: str): + """Returns None if IDA does not know the type.""" + tinfo = ida_typeinf.tinfo_t() + if tinfo.get_named_type(idaapi.get_idati(), name): + return tinfo + + if name == "void": + return ida_typeinf.tinfo_t(ida_typeinf.BT_VOID) + + if name == "__fastcall": + # function pointer parameters are exported without their signature + name = "void (__fastcall *)()" + + if "<" in name and tinfo.get_named_type( + idaapi.get_idati(), get_template_alias(name) + ): + # use the type behind the typedef so IDA shows the real name + target = ida_typeinf.tinfo_t() + if target.get_named_type(idaapi.get_idati(), tinfo.get_final_type_name()): + return target + return tinfo + + terminated = name.rstrip(";") + ";" + if idaapi.parse_decl(tinfo, idaapi.get_idati(), terminated, idaapi.PT_SIL) is None: + return None + + return tinfo + + def get_tinfo_from_type(self, raw_type: str, array_size=0): + """Returns None if the type could not be built.""" + type = raw_type.rstrip("*") + ptr_count = len(raw_type) - len(type) + + tinfo = self.get_named_type(type) + if tinfo is None: + return None + + for _ in range(ptr_count): + ptr_tinfo = idaapi.tinfo_t() + if not ptr_tinfo.create_ptr(tinfo): + return None + tinfo = ptr_tinfo + + if array_size > 0: + array_tinfo = idaapi.tinfo_t() + if not array_tinfo.create_array(tinfo, array_size): + return None + tinfo = array_tinfo + + return tinfo + + def apply_func_type(self, ea: int, func_name: str, return_type: str, parameters: list[DefinedStructFuncParam]) -> bool: + """Returns False if one of the types is not known to IDA.""" + func_data = ida_typeinf.func_type_data_t() + func_data.cc = ida_typeinf.CM_CC_FASTCALL + + rettype = self.get_tinfo_from_type(return_type) + if rettype is None: + print(f"Error: {func_name} @ {ea:#x} has unknown return type {return_type!r}") + return False + func_data.rettype = rettype + + for param in parameters: + arg = ida_typeinf.funcarg_t() + arg_type = self.get_tinfo_from_type(param.type) + if arg_type is None: + print(f"Error: {func_name} @ {ea:#x} has unknown type {param.type!r} on parameter {param.name!r}") + return False + arg.type = arg_type + arg.name = param.name + func_data.push_back(arg) + + tif = ida_typeinf.tinfo_t() + if not tif.create_func(func_data): + print(f"Error: {func_name} @ {ea:#x} failed to create its function type") + return False + if not self.apply_tinfo(ea, tif): + print(f"Error: {func_name} @ {ea:#x} failed to apply {tif.dstr()!r}") + return False + return True + + def apply_tinfo(self, ea: int, tif) -> bool: + return ida_typeinf.apply_tinfo(ea, tif, ida_typeinf.TINFO_DEFINITE) + + def can_guess_func_type(self, ea: int) -> bool: + """Returns False if IDA can not work out the type, even after decompiling.""" + tif, funcdata = ida_typeinf.tinfo_t(), ida_typeinf.func_type_data_t() + + ida_typeinf.guess_tinfo(tif, ea) + if tif.get_func_details(funcdata): + return True + + ida_hexrays.decompile(ea) + ida_typeinf.guess_tinfo(tif, ea) + return tif.get_func_details(funcdata) + + def allow_template_type_names(self): + """Allow the spaces clang puts in template names, one rejected name drops every declaration.""" + ida_idp.process_config_directive( + 'TypeNameChars = "~.<>?!/,*&[]-+ "' + '"_:$()`\'{}"' + '"0123456789"' + '"ABCDEFGHIJKLMNOPQRSTUVWXYZ"' + '"abcdefghijklmnopqrstuvwxyz";' + ) + + def select_parser(self): + self.allow_template_type_names() + includes = "" + if system() != "Windows": + path = pick_folder(fields) + if path == None: + raise "Has to have msvc includes for the parsers to work with the project" + path = Path(path) + vc_path = path / "vc" / "tools" / "msvc" + vc_path = first_subfolder(vc_path) / "include" + window_kit = first_subfolder(path / "kits" / "10" / "include") + winsdk_ucrt_path = window_kit / "ucrt" + winsdk_um_path = window_kit / "um" + winsdk_shared_path = window_kit / "shared" + includes = f" -I{vc_path.as_posix()} -I\"{winsdk_ucrt_path.as_posix()}\" -I\"{winsdk_um_path.as_posix()}\" -I\"{winsdk_shared_path.as_posix()}\"" + + ida_srclang.set_parser_argv( + "clang", + f"-x c++ -target x86_64-pc-windows-msvc -fms-compatibility -fms-extensions -U__int64 -std=c++17{includes}", + ) + + def get_probe_layouts(self, source: str) -> dict[str, tuple[int, int]]: + """Size and offset of value in each probe struct of SrcInterface.get_layout_probe.""" + # a til of its own keeps the probes out of the database + til = ida_typeinf.new_til("cs_layout_probe.til", "FFXIVClientStructs layout probe") + try: + if ida_srclang.parse_decls_with_parser("clang", til, source, False) < 0: + raise RuntimeError("parse_decls_with_parser failed: the clang parser is not available") + + layouts: dict[str, tuple[int, int]] = {} + name = ida_typeinf.first_named_type(til, ida_typeinf.NTF_TYPE) + while name: + if name.startswith(layout_probe_prefix): + tinfo = ida_typeinf.tinfo_t() + udt = ida_typeinf.udt_type_data_t() + if tinfo.get_named_type(til, name) and tinfo.get_udt_details(udt): + for member in udt: + if member.name == "value" and member.size > 0: + layouts[name] = (member.size // 8, member.offset // 8) + name = ida_typeinf.next_named_type(til, name, ida_typeinf.NTF_TYPE) + return layouts + finally: + ida_typeinf.free_til(til) + + def parse_string(self, decl: str, vtables: list[str]) -> int: + """Returns the number of parser errors reported by IDA.""" + error_count = ida_srclang.parse_decls_with_parser("clang", None, decl, False) + if error_count < 0: + raise RuntimeError( + "parse_decls_with_parser failed: the clang parser is not available" + ) + if error_count > 0: + print(f"Error: clang parser reported {error_count} error(s) while importing declarations") + else: + print("Clang parser imported declarations without errors") + + vtable_errors = [] + for vtable in vtables: + error = self.mark_as_vtable(vtable) + if error is not None: + vtable_errors.append(error) + for error in vtable_errors: + print(f"Error: mark_as_vtable: {error}") + print(f"Marked {len(vtables) - len(vtable_errors)}/{len(vtables)} vtables ({len(vtable_errors)} failed)") + + return error_count + + def op_type_as_name(self, n: int): + for item in [x for x in dir(idc) if x.startswith("o_")]: + if getattr(idc, item) == n: + return item + + def get_vft_from_data(self, class_name: str) -> DefinedDataClassVtable | None: + class_data = self.data_yaml.classes.get(class_name) + if class_data is None or not class_data.vtbls: + return None + + return class_data.vtbls[0] + + def get_vfunc_ea_from_data(self, class_name: str, offset: int) -> int: + vtable = self.get_vft_from_data(class_name) + if vtable is None: + return idc.BADADDR + + vtable_ea = vtable.ea + self.get_image_base() - 0x140000000 + return self.get_qword(vtable_ea + offset) + + def set_func_name(self, ea, name, cmt=None, unique=False): + """With unique only default names are replaced, a taken name gets a number.""" + if unique: + if not idc.get_func_name(ea).startswith("sub_"): + return + base_name, count = name, 1 + while self.get_func_ea_by_name(name) not in (idc.BADADDR, ea): + count += 1 + name = f"{base_name}{count}" + print(name) + + ida_name.set_name(ea, name) + if cmt is not None: + ida_bytes.set_cmt(ea, cmt, 0) + + def get_image_base(self) -> int: + return idaapi.get_imagebase() + + def is_offset(self, ea: int) -> bool: + return idc.is_off0(idc.get_full_flags(ea)) + + def xrefs_to(self, ea: int) -> list[int]: + return [xref.to for xref in idautils.XrefsTo(ea)] + + def get_qword(self, ea: int) -> int: + return idc.get_qword(ea) + + def get_addr_name(self, ea: int) -> str: + name = idc.get_name(ea) + # code with no xrefs to it is not made a function due to how IDA detects function boundaries + if (name is None or name == '') and idc.get_segm_name(ea) == '.text': + ida_funcs.add_func(ea) + name = idc.get_name(ea) + return name + + def set_addr_name(self, ea: int, name: str) -> bool: + if ida_bytes.get_item_head(ea) != ea: + return True + result = idc.set_name(ea, name) + return bool(result) + + def get_comment(self, ea: int): + idc.get_cmt(ea, False) + + def set_comment(self, ea: int, comment: str): + idc.set_cmt(ea, comment, False) + + def get_func_name(self, ea: int) -> str: + """Name of the function containing ea.""" + return idc.get_func_name(ea) + + def get_func_start(self, ea: int) -> int: + """Start of the function containing ea.""" + return ida_funcs.get_func(ea).start_ea + + def get_string_from_ptr(self, ea: int) -> str: + val = idc.get_strlit_contents(ea) + if val == None: + val = idc.get_strlit_contents(idc.get_qword(ea)) + return val.decode('utf-8') + + def heads_before(self, ea: int, count: int): + """The instructions before ea, closest first.""" + for _ in range(count): + ea = idc.prev_head(ea) + yield ea + + def heads_until_call(self, ea: int, func_name: str): + """The instructions from ea up to the call of the named function.""" + while not (idc.print_insn_mnem(ea) == 'call' and idc.print_operand(ea, 0) == func_name): + yield ea + ea = idc.next_head(ea) + + def get_registers(self, eas, mnemonics=None) -> dict[str, dict[str, object]]: + """What the instructions write to each register, later ones win.""" + regs = {} + for ea in eas: + if mnemonics is not None and idc.print_insn_mnem(ea) not in mnemonics: + continue + val_text = idc.print_operand(ea, 1) + val = idc.get_operand_value(ea, 1) + if val_text.startswith('cs:'): + val_text = self.get_string_from_ptr(val) + regs[idc.print_operand(ea, 0)] = {'val_text': val_text, 'val': val} + return regs + + def get_func_xrefs(self, ea: int) -> list[int]: + """The addresses calling ea.""" + return [xref for xref in idautils.CodeRefsTo(ea, True) if idc.print_insn_mnem(xref).lower() == "call"] + + def format_class_name_for_vtbl(self, class_name: str) -> str: + return f"vtbl_{class_name}" + + def format_class_name_for_secondary_vtbl(self, class_name: str, secondary_name: str) -> str: + return f"vtbl_{class_name}___{secondary_name}" + + def format_class_name(self, class_name: str) -> str: + return class_name + + def format_vfunc_name(self, ea: int, current_func_name: str, proposed_func_name: str, class_name: str, base_class_names: list[str]) -> str | None: + if current_func_name.startswith("j_"): # jump + current_func_name = current_func_name.lstrip("j_") + + if current_func_name.startswith("qword_"): + idc.auto_mark_range(ea, ea + 1, idc.AU_CODE) + idc.create_insn(ea) + idc.add_func(ea) + current_func_name = self.get_addr_name(ea) + print(f"Info: qword in vtbl of {class_name} at 0x{ea:X}, it may be an offset to undefined code") + + # Previously renamed as a vfunc + if current_func_name.startswith(class_name): + # return the proposed func name in case it was updated since last run + current_class_name = current_func_name.rsplit(".", 1)[0] + return f"{current_class_name}.{proposed_func_name}" + + # This should have been handled in the base class + if any(current_func_name.startswith(name) for name in base_class_names): + return "" + + if current_func_name.startswith("sub_"): + return f"{class_name}.{proposed_func_name}" + + if current_func_name.startswith("unknown_libname_"): + return f"{class_name}.{proposed_func_name}" + + if current_func_name.startswith("nullsub_"): + return f"{class_name}.{proposed_func_name}_nullsub" + + if current_func_name.startswith("loc_"): + return f"{class_name}.{proposed_func_name}_loc" + + if current_func_name.startswith("locret_"): + return f"{class_name}.{proposed_func_name}_locret" + + # Name it later in a child class when it gets overridden + if current_func_name == "_purecall": + return "" + + # Mangled func names, thanks IDA + if current_func_name.startswith("?") or current_func_name.startswith("_"): + return f"{class_name}.{proposed_func_name}" + + return None + + def format_func_name(self, ea: int, current_func_name: str, proposed_func_name: str, class_name: str) -> str | None: + if current_func_name.startswith("j_"): # jump + current_func_name = current_func_name.lstrip("j_") + + # Mangled func names, thanks IDA + if proposed_func_name.startswith("?") or proposed_func_name.startswith("_"): + return proposed_func_name + + proposed_qualified_func_name = f"{class_name}.{proposed_func_name}" + if current_func_name == proposed_qualified_func_name: + return "" + + if any(current_func_name.startswith(prefix) for prefix in ("sub_", "nullsub_", "loc_", "qword_", "unknown_libname_", "?", "_")): + return proposed_qualified_func_name + + return None + + def search_binary(self, ea: int, pattern: str, flag: int = ida_search.SEARCH_DOWN): + return ida_bytes.find_bytes( + pattern, ea, None, ida_ida.inf_get_max_ea(), None, flag + ) + + def search_binary_all(self, pattern: str, ea: int = 0): + """Every address matching the pattern, starting from ea.""" + ea = self.search_binary(ea, pattern) + while ea != idc.BADADDR: + yield ea + ea = self.search_binary(ea + 1, pattern) + + def get_dword(self, ea: int): + return ida_bytes.get_original_dword(ea) + + def get_func_ea_by_name(self, name: str): + return ida_name.get_name_ea(0, name) + + def get_func_ea_by_sig(self, pattern: str): + ea = self.search_binary(0, pattern, ida_search.SEARCH_DOWN) + + if ida_funcs.get_func_start(ea) == idc.BADADDR: + ida_funcs.add_func(ea) + + func_start = ida_funcs.get_func_start(ea) + if func_start == idc.BADADDR: + return idc.BADADDR + + if func_start == ea: + return ea + mnem = idc.print_insn_mnem(ea) + if not mnem: + return idc.BADADDR + + opType0 = idc.get_operand_type(ea, 0) + if mnem == "jmp" or mnem == "call" or mnem[0] == "j": + if opType0 != idc.o_near and opType0 != idc.o_mem: + print(f"Error: Can't follow opType0 {self.op_type_as_name(opType0)}") + return idc.BADADDR + return idc.get_operand_value(ea, 0) + + if idc.next_head(ea) == ea + idc.get_item_size(ea) and idc.is_flow( + idc.get_full_flags(idc.next_head(ea)) + ): + return idc.next_head(ea) + + return idc.BADADDR + + def update_member_func(self, member_func: DefinedStructMemFunc, struct: DefinedStruct) -> None: + func_name = f"{struct.type}.{member_func.name}" + ea = self.get_func_ea_by_name(func_name) + if ea == idc.BADADDR: + ea = self.get_func_ea_by_sig(member_func.signature) + if ea == idc.BADADDR: + print( + f"Error: {func_name} not found bad sig? {member_func.signature}" + ) + return + if ida_funcs.get_func_name(ea) == f"sub_{ea:X}": + idc.set_name(ea, func_name) + self.apply_func_type(ea, func_name, member_func.return_type, member_func.parameters) + + def update_virt_func(self, virt_func: DefinedStructVFunc, struct: DefinedStruct): + func_name = f"{struct.type}.{virt_func.name}" + + ea = self.get_func_ea_by_name(func_name) + if ea == idc.BADADDR: + if self.get_vft_from_data(struct.type) is None: + return + ea = self.get_vfunc_ea_from_data(struct.type, virt_func.offset) + if ea in (0, idc.BADADDR): + print(f"Error: {func_name} not found and its VFT slot could not be resolved") + return + + actual_name = ida_funcs.get_func_name(ea) or "" + normalized_name = actual_name + for prefix in ("j_", "thunk_"): + if normalized_name.startswith(prefix): + normalized_name = normalized_name[len(prefix):] + + if normalized_name == "_purecall": + return + + primary_vtable = self.get_vft_from_data(struct.type) + primary_base = primary_vtable.base if primary_vtable else None + if primary_base: + base_ea = self.get_vfunc_ea_from_data(primary_base, virt_func.offset) + if base_ea == ea: + return + + if ( + normalized_name.endswith("." + virt_func.name) + and not normalized_name.startswith(struct.type + ".") + ): + return + self.apply_func_type(ea, func_name, virt_func.return_type, virt_func.parameters) + + def get_static_member_ea(self, static_member: DefinedStructStaticMember) -> int: + """Where the static member's signature resolves to, BADADDR if not found.""" + ea = self.search_binary(0, static_member.signature) + if ea == idc.BADADDR: + return ea + for follows in static_member.relative_offsets: + ea = ea + follows + ea = ea + 4 + self.get_dword(ea) + return ea + + def update_static_member(self, static_member: DefinedStructStaticMember, struct: DefinedStruct): + ea = self.get_static_member_ea(static_member) + if ea == idc.BADADDR: + print(f"Error: {static_member.signature} not found something is wrong") + return + if ida_bytes.get_item_head(ea) != ea and static_member.return_type != struct.type: + # points into an already typed item (ex. a field of the struct instance) + return + return_type = static_member.return_type + if static_member.is_pointer: + return_type = return_type + "*" + tif = self.get_tinfo_from_type(return_type) + if tif is None: + print(f"Error: static member of {struct.type} @ {ea:#x} has unknown type {return_type!r}") + else: + self.apply_tinfo(ea, tif) + ida_name.set_name(ea, f"g_{struct.type}_{static_member.name}") + + def mark_as_vtable(self, vtbl_name: str) -> str | None: + """Returns an error string on failure.""" + tif = ida_typeinf.tinfo_t() + + if not tif.get_named_type(None, vtbl_name): + return f"Type '{vtbl_name}' not found in type library" + + if not tif.is_struct(): + actual = ( + "union" if tif.is_union() else "enum" if tif.is_enum() else "unknown" + ) + return f"'{vtbl_name}' is not a struct (got {actual})" + + if tif.is_vftable(): + return None + + udt = ida_typeinf.udt_type_data_t() + if not tif.get_udt_details(udt): + return f"Failed to retrieve UDT details for '{vtbl_name}'" + + udt.set_vftable() + + if not tif.create_udt(udt, ida_typeinf.BTF_STRUCT): + return ( + f"Failed to reconstruct UDT for '{vtbl_name}' after setting vtable flag" + ) + + result = tif.set_named_type(None, vtbl_name, ida_typeinf.NTF_REPLACE) + + if result != ida_typeinf.TERR_OK: + return ( + f"Failed to write back '{vtbl_name}' to type library: " + f"{ida_typeinf.tinfo_errstr(result)}" + ) + + return None \ No newline at end of file diff --git a/scripts/ida/rename.py b/scripts/ida/rename.py new file mode 100644 index 0000000000..245f603a0d --- /dev/null +++ b/scripts/ida/rename.py @@ -0,0 +1,60 @@ +from os.path import join, dirname +import sys + +for path in (dirname(__file__), join(dirname(__file__), "..", "shared")): + if path not in sys.path: + sys.path.append(path) + +from ida_wrapper import IdaInterface +from class_factory import apply_data + +lua_namespace = "Client::Game::Event::Lua" + +def apply_lua_funcs(): + """Needs the LuaState names, so apply_data has to run first.""" + string_eas = ida.get_func_xrefs(ida.get_func_ea_by_name('Common::Lua::LuaState.SetStringField')) + func_eas = ida.get_func_xrefs(ida.get_func_ea_by_name('Common::Lua::LuaState.SetFunctionField')) + + string_eas = [ida.get_func_start(x) for x in string_eas] + + eas = {} + for x in func_eas: + regs = ida.get_registers(ida.heads_before(x, 4)) + regs['r8'] = ida.get_string_from_ptr(regs['r8']['val']) + eas.setdefault(ida.get_func_start(x), {'calls': []})['calls'].append({'off': x, 'regs': regs}) + + eas = {k: v for k,v in eas.items() if k in string_eas} + + for func_ea in eas: + eas[func_ea]['regs'] = ida.get_registers(ida.heads_until_call(func_ea, 'Common::Lua::LuaState.SetStringField'), ('mov',)) + + for func_ea in eas: + eas[func_ea]['name'] = eas[func_ea]['regs'][eas[func_ea]['regs']['r9']['val_text']]['val_text'] + eas[func_ea]['name_val'] = eas[func_ea]['regs'][eas[func_ea]['regs']['r9']['val_text']]['val'] + + for func_ea in eas: + if eas[func_ea]['name'] == 'r8': + eas[func_ea]['name'] = ida.get_func_name(func_ea).split('.')[0].split(':')[-1] + eas[func_ea]['name_val'] = func_ea + if eas[func_ea]['name'] == 'rax': + eas[func_ea]['name_val'] = ida.get_func_xrefs(func_ea)[0] + eas[func_ea]['name'] = ida.get_func_name(eas[func_ea]['name_val']).split('.')[0].split(':')[-1] + + for func_ea in eas: + name = f"{lua_namespace}::{eas[func_ea]['name']}" + ida.set_func_name(func_ea, f"{name}.SetupLuaClass") + for call in eas[func_ea]['calls']: + regs = call['regs'] + func_name = regs['r8'] + ida.set_func_name(regs['r9']['val'], f"{name}.{func_name}", unique=True) + +ida = IdaInterface() +ida.select_parser() +print("Loading data.yml") +ida.load_data_yml() + +print("Executing") +apply_data(ida, ida.data_yaml) +print("Naming lua functions") +apply_lua_funcs() +print("Done") diff --git a/scripts/ida/struct_import.py b/scripts/ida/struct_import.py new file mode 100644 index 0000000000..c1b06a0781 --- /dev/null +++ b/scripts/ida/struct_import.py @@ -0,0 +1,243 @@ +from os.path import join, dirname +import sys + +sys.path.append(join(dirname(__file__), "..", "shared")) + +from lumina_wrapper import get_excel_header_files, find_game_path, is_game_path +from exdschema_wrapper import ( + create_struct_from_header_and_schema, + get_exdschema_data, + exd_comment_patterns, + exd_func_patterns, +) +from src_wrapper import SrcInterface +from ida_wrapper import IdaInterface +from form_builder import DynamicForm +from structs_schema import get_yaml + +src = SrcInterface() +src.ida_vtables = True +ida = IdaInterface() +print("Loading data.yml") +ida.load_data_yml() + +check_fields = [ + {"type": "label", "text": "Check version match."}, + {"type": "space"}, + {"type": "label", "text": f"Loaded with ver: {ida.data_yaml.version}"}, + {"type": "label", "text": "Check if this matches with what is reported in the version files of the game."} +] + +form = DynamicForm(check_fields) +form.Compile() +try: + if form.Execute() != 1: + raise "Version is not correct please update repo" +finally: + form.Free() + +ida.select_parser() + +def ask_game_path() -> str: + game_path = find_game_path() + if game_path is None: + game_fields = [ + {"type": "label", "text": "The game install was not found through XLCore, XIVLauncher or the default folder."}, + {"type": "space"}, + {"type": "folder", "key": "game_path", "label": "Folder holding both game and boot"}, + ] + form = DynamicForm(game_fields) + form.Compile() + try: + if form.Execute() != 1: + raise RuntimeError("No game install selected") + game_path = form.results()["game_path"] + finally: + form.Free() + if not is_game_path(game_path): + raise RuntimeError(f"No game or boot folder in {game_path}, select the folder holding both") + print(f"Using game install at {game_path}") + return game_path + +fields = [ + {"type": "label", "text": "Select function types to update"}, + {"type": "space"}, + {"type": "label", "text": "Types to update"}, + { + "type": "checks", + "key": "function_types", + "items": [ + ("virtual", "Virtual functions"), + ("member", "Member functions"), + ], + }, + {"type": "space"}, + {"type": "label", "text": "Base name?"}, + { + "type": "radios", + "key": "base_name", + "items": [ + ("ffxiv_dx11.exe", "x64dbg offset (ffxiv_dx11.exe)"), + ("__ImageBase", "IDA defualt (__ImageBase)"), + ], + } +] +form = DynamicForm(fields) +form.Compile() +try: + if form.Execute() == 1: + _function_update_options = form.results()["function_types"] + ida.set_addr_name(ida.get_image_base(), form.results()["base_name"]) + else: + _function_update_options = { + "virtual": False, + "member": False, + } +finally: + form.Free() + +print("Grabbing ExdSchema and Lumina") + +excel_header_files = get_excel_header_files(ida.data_yaml.version, ask_game_path) +exdschema = get_exdschema_data("latest") + +excel_structs, excel_map = create_struct_from_header_and_schema( + excel_header_files, exdschema +) + +exd_map = {excel_map[k]: k for k in excel_map} + +excel_header, excel_vtables = src.build_export_string(excel_structs, 4, ida.get_probe_layouts) + + +print("Parsing structs.yml") + +with open(ida.get_struct_path()) as f: + import_structs = get_yaml(f) + +struct_header, struct_vtables = src.build_export_string(import_structs, 1, ida.get_probe_layouts, excel_header) + +print("Parsing combined Excel and FFXIV declarations") +ida.parse_string( + f"""{src.get_predef_header()} +{excel_header} +{struct_header} +""", + [*excel_vtables, *struct_vtables], +) + +def do_pattern(pattern, suffix, sheetSearchPattern, sheetSearchOffset): + for ea in ida.search_binary_all(pattern, 1): + if sheetSearchPattern == None: + sheetEa = ea + sheetSearchOffset + else: + sheetEa = ( + ida.search_binary(ea, sheetSearchPattern) + + sheetSearchOffset + ) + sheetIdx = ida.get_dword(sheetEa) + origName = ida.get_func_name(ea) + + # don't rename any funcs that are already named + if origName[0:4] == "sub_": + if exd_map.get(sheetIdx) == None: + print( + f"Func @ 0x{ea:X} references unknown sheet {sheetIdx}! sheet dword offset = 0x{sheetEa:X}" + ) + continue + + sheetName = exd_map[sheetIdx] + + if suffix == None: + suffix = "" + + fnName = "Component::Exd::ExdModule_Get%s%s" % ( + exd_map[sheetIdx], + suffix, + ) + + uniquifier = 0 + while True: + uniqueName = fnName + (f"_{uniquifier}" if uniquifier > 0 else "") + + if ida.get_func_ea_by_name(uniqueName) == ida.BADADDR and uniquifier > 0: + fnName = uniqueName + break + + uniquifier += 1 + + ida.set_func_name( + ea, fnName, "Sheet: {0} ({1})".format(sheetName, sheetIdx) + ) + + if suffix == "SheetIndex": + continue + + if not ida.can_guess_func_type(ea): + print("Failed to get func details for %s @ %X" % (fnName, ea)) + continue + + if suffix == "RowAndSubRowId": + tif = ida.get_tinfo_from_type( + f"Component::Exd::Sheets::{exd_map[sheetIdx]} *__fastcall func(unsigned int rowId, unsigned int subRowId);" + ) + elif suffix == "RowCount": + tif = ida.get_tinfo_from_type( + f"Component::Exd::Sheets::{exd_map[sheetIdx]} *__fastcall func();" + ) + else: + tif = ida.get_tinfo_from_type( + f"Component::Exd::Sheets::{exd_map[sheetIdx]} *__fastcall func(unsigned int rowIdOrIndex);" + ) + + ida.apply_tinfo(ea, tif) + + +print("Fixing up Excel functions") + +for pattern in exd_func_patterns: + suffix, search = exd_func_patterns[pattern] + if suffix is not None: + print(f"Finding exd funcs of {suffix}... please wait.") + searchPattern, searchOffset = search + do_pattern(pattern, suffix, searchPattern, searchOffset) + + +for pattern_key in exd_comment_patterns: + for ea in list(ida.search_binary_all(pattern_key)): + sheetIdx = ida.get_dword(ea + exd_comment_patterns[pattern_key]) + sheetName = exd_map.get(sheetIdx) + if sheetName is None: + print(f"Sheet getter @ 0x{ea:X} references unknown sheet {sheetIdx}!") + continue + ida.set_comment(ea, "Sheet: {0} ({1})".format(sheetName, sheetIdx)) + +if _function_update_options["virtual"]: + for struct in import_structs.structs: + if struct.virtual_functions: + print(f"Updating virtual functions for {struct.type}") + for virt_func in struct.virtual_functions: + if virt_func is None: + continue + + if virt_func.return_type is not None and virt_func.parameters is not None: + ida.update_virt_func(virt_func, struct) + +print("Fixing up CS defined functions") + +if _function_update_options["member"]: + for struct in import_structs.structs: + if struct.member_functions != []: + print(f"Updating member functions for {struct.type}") + for member_func in struct.member_functions: + ida.update_member_func(member_func, struct) + + if struct.static_member_functions: + print(f"Updating static member functions for {struct.type}") + for member_func in struct.static_member_functions: + ida.update_member_func(member_func, struct) + + if struct.static_members: + print(f"Updating static members for {struct.type}") + for member in struct.static_members: + ida.update_static_member(member, struct) \ No newline at end of file diff --git a/ida/idauser.cfg b/scripts/idauser.cfg similarity index 100% rename from ida/idauser.cfg rename to scripts/idauser.cfg diff --git a/ida/old/data_2020.12.29.0000.0000.yml b/scripts/old/data_2020.12.29.0000.0000.yml similarity index 100% rename from ida/old/data_2020.12.29.0000.0000.yml rename to scripts/old/data_2020.12.29.0000.0000.yml diff --git a/ida/old/data_2021.01.28.0000.0000.yml b/scripts/old/data_2021.01.28.0000.0000.yml similarity index 100% rename from ida/old/data_2021.01.28.0000.0000.yml rename to scripts/old/data_2021.01.28.0000.0000.yml diff --git a/ida/old/data_2021.02.10.0000.0000.yml b/scripts/old/data_2021.02.10.0000.0000.yml similarity index 100% rename from ida/old/data_2021.02.10.0000.0000.yml rename to scripts/old/data_2021.02.10.0000.0000.yml diff --git a/ida/old/data_2021.04.07.0000.0000.yml b/scripts/old/data_2021.04.07.0000.0000.yml similarity index 100% rename from ida/old/data_2021.04.07.0000.0000.yml rename to scripts/old/data_2021.04.07.0000.0000.yml diff --git a/ida/old/data_2021.04.21.0000.0000.yml b/scripts/old/data_2021.04.21.0000.0000.yml similarity index 100% rename from ida/old/data_2021.04.21.0000.0000.yml rename to scripts/old/data_2021.04.21.0000.0000.yml diff --git a/ida/old/data_2021.05.18.0000.0000.yml b/scripts/old/data_2021.05.18.0000.0000.yml similarity index 100% rename from ida/old/data_2021.05.18.0000.0000.yml rename to scripts/old/data_2021.05.18.0000.0000.yml diff --git a/ida/old/data_2021.05.29.0000.0000.yml b/scripts/old/data_2021.05.29.0000.0000.yml similarity index 100% rename from ida/old/data_2021.05.29.0000.0000.yml rename to scripts/old/data_2021.05.29.0000.0000.yml diff --git a/ida/old/data_2021.06.16.0000.0000.yml b/scripts/old/data_2021.06.16.0000.0000.yml similarity index 100% rename from ida/old/data_2021.06.16.0000.0000.yml rename to scripts/old/data_2021.06.16.0000.0000.yml diff --git a/ida/old/data_2021.07.15.0000.0000.yml b/scripts/old/data_2021.07.15.0000.0000.yml similarity index 100% rename from ida/old/data_2021.07.15.0000.0000.yml rename to scripts/old/data_2021.07.15.0000.0000.yml diff --git a/ida/old/data_2021.08.17.0000.0000.yml b/scripts/old/data_2021.08.17.0000.0000.yml similarity index 100% rename from ida/old/data_2021.08.17.0000.0000.yml rename to scripts/old/data_2021.08.17.0000.0000.yml diff --git a/ida/old/data_2021.11.28.0000.0000.yml b/scripts/old/data_2021.11.28.0000.0000.yml similarity index 100% rename from ida/old/data_2021.11.28.0000.0000.yml rename to scripts/old/data_2021.11.28.0000.0000.yml diff --git a/ida/old/data_2021.12.16.0000.0000.yml b/scripts/old/data_2021.12.16.0000.0000.yml similarity index 100% rename from ida/old/data_2021.12.16.0000.0000.yml rename to scripts/old/data_2021.12.16.0000.0000.yml diff --git a/ida/old/data_2021.12.24.0000.0000.yml b/scripts/old/data_2021.12.24.0000.0000.yml similarity index 100% rename from ida/old/data_2021.12.24.0000.0000.yml rename to scripts/old/data_2021.12.24.0000.0000.yml diff --git a/ida/old/data_2022.01.25.0000.0000.yml b/scripts/old/data_2022.01.25.0000.0000.yml similarity index 100% rename from ida/old/data_2022.01.25.0000.0000.yml rename to scripts/old/data_2022.01.25.0000.0000.yml diff --git a/ida/old/data_2022.03.01.0000.0000.yml b/scripts/old/data_2022.03.01.0000.0000.yml similarity index 100% rename from ida/old/data_2022.03.01.0000.0000.yml rename to scripts/old/data_2022.03.01.0000.0000.yml diff --git a/ida/old/data_2022.04.07.0000.0000.yml b/scripts/old/data_2022.04.07.0000.0000.yml similarity index 100% rename from ida/old/data_2022.04.07.0000.0000.yml rename to scripts/old/data_2022.04.07.0000.0000.yml diff --git a/ida/old/data_2022.04.20.0000.0000.yml b/scripts/old/data_2022.04.20.0000.0000.yml similarity index 100% rename from ida/old/data_2022.04.20.0000.0000.yml rename to scripts/old/data_2022.04.20.0000.0000.yml diff --git a/ida/old/data_2022.05.19.0000.0000.yml b/scripts/old/data_2022.05.19.0000.0000.yml similarity index 100% rename from ida/old/data_2022.05.19.0000.0000.yml rename to scripts/old/data_2022.05.19.0000.0000.yml diff --git a/ida/old/data_2022.05.27.0000.0000.yml b/scripts/old/data_2022.05.27.0000.0000.yml similarity index 100% rename from ida/old/data_2022.05.27.0000.0000.yml rename to scripts/old/data_2022.05.27.0000.0000.yml diff --git a/ida/old/data_2022.06.21.0000.0000.yml b/scripts/old/data_2022.06.21.0000.0000.yml similarity index 100% rename from ida/old/data_2022.06.21.0000.0000.yml rename to scripts/old/data_2022.06.21.0000.0000.yml diff --git a/ida/old/data_2022.07.08.0000.0000.yml b/scripts/old/data_2022.07.08.0000.0000.yml similarity index 100% rename from ida/old/data_2022.07.08.0000.0000.yml rename to scripts/old/data_2022.07.08.0000.0000.yml diff --git a/ida/old/data_2022.08.17.0000.0000.yml b/scripts/old/data_2022.08.17.0000.0000.yml similarity index 100% rename from ida/old/data_2022.08.17.0000.0000.yml rename to scripts/old/data_2022.08.17.0000.0000.yml diff --git a/ida/old/data_2022.08.25.0000.0000.yml b/scripts/old/data_2022.08.25.0000.0000.yml similarity index 100% rename from ida/old/data_2022.08.25.0000.0000.yml rename to scripts/old/data_2022.08.25.0000.0000.yml diff --git a/ida/old/data_2022.09.07.0000.0000.yml b/scripts/old/data_2022.09.07.0000.0000.yml similarity index 100% rename from ida/old/data_2022.09.07.0000.0000.yml rename to scripts/old/data_2022.09.07.0000.0000.yml diff --git a/ida/old/data_2022.10.05.0000.0000.yml b/scripts/old/data_2022.10.05.0000.0000.yml similarity index 100% rename from ida/old/data_2022.10.05.0000.0000.yml rename to scripts/old/data_2022.10.05.0000.0000.yml diff --git a/ida/old/data_2022.10.26.0000.0000.yml b/scripts/old/data_2022.10.26.0000.0000.yml similarity index 100% rename from ida/old/data_2022.10.26.0000.0000.yml rename to scripts/old/data_2022.10.26.0000.0000.yml diff --git a/ida/old/data_2022.11.09.0000.0000.yml b/scripts/old/data_2022.11.09.0000.0000.yml similarity index 100% rename from ida/old/data_2022.11.09.0000.0000.yml rename to scripts/old/data_2022.11.09.0000.0000.yml diff --git a/ida/old/data_2022.12.23.0000.0000.yml b/scripts/old/data_2022.12.23.0000.0000.yml similarity index 100% rename from ida/old/data_2022.12.23.0000.0000.yml rename to scripts/old/data_2022.12.23.0000.0000.yml diff --git a/ida/old/data_2023.01.11.0000.0000.yml b/scripts/old/data_2023.01.11.0000.0000.yml similarity index 100% rename from ida/old/data_2023.01.11.0000.0000.yml rename to scripts/old/data_2023.01.11.0000.0000.yml diff --git a/ida/old/data_2023.01.17.0000.0000.yml b/scripts/old/data_2023.01.17.0000.0000.yml similarity index 100% rename from ida/old/data_2023.01.17.0000.0000.yml rename to scripts/old/data_2023.01.17.0000.0000.yml diff --git a/ida/old/data_2023.02.03.0000.0000.yml b/scripts/old/data_2023.02.03.0000.0000.yml similarity index 100% rename from ida/old/data_2023.02.03.0000.0000.yml rename to scripts/old/data_2023.02.03.0000.0000.yml diff --git a/ida/old/data_2023.02.28.0000.0000.yml b/scripts/old/data_2023.02.28.0000.0000.yml similarity index 100% rename from ida/old/data_2023.02.28.0000.0000.yml rename to scripts/old/data_2023.02.28.0000.0000.yml diff --git a/ida/old/data_2023.03.24.0000.0000.yml b/scripts/old/data_2023.03.24.0000.0000.yml similarity index 100% rename from ida/old/data_2023.03.24.0000.0000.yml rename to scripts/old/data_2023.03.24.0000.0000.yml diff --git a/ida/old/data_2023.05.18.0000.0000.yml b/scripts/old/data_2023.05.18.0000.0000.yml similarity index 100% rename from ida/old/data_2023.05.18.0000.0000.yml rename to scripts/old/data_2023.05.18.0000.0000.yml diff --git a/ida/old/data_2023.05.31.0000.0000.yml b/scripts/old/data_2023.05.31.0000.0000.yml similarity index 100% rename from ida/old/data_2023.05.31.0000.0000.yml rename to scripts/old/data_2023.05.31.0000.0000.yml diff --git a/ida/old/data_2023.06.14.0000.0000.yml b/scripts/old/data_2023.06.14.0000.0000.yml similarity index 100% rename from ida/old/data_2023.06.14.0000.0000.yml rename to scripts/old/data_2023.06.14.0000.0000.yml diff --git a/ida/old/data_2023.07.06.0000.0000.yml b/scripts/old/data_2023.07.06.0000.0000.yml similarity index 100% rename from ida/old/data_2023.07.06.0000.0000.yml rename to scripts/old/data_2023.07.06.0000.0000.yml diff --git a/ida/old/data_2023.07.26.0000.0000.yml b/scripts/old/data_2023.07.26.0000.0000.yml similarity index 100% rename from ida/old/data_2023.07.26.0000.0000.yml rename to scripts/old/data_2023.07.26.0000.0000.yml diff --git a/ida/old/data_2023.09.28.0000.0000.yml b/scripts/old/data_2023.09.28.0000.0000.yml similarity index 100% rename from ida/old/data_2023.09.28.0000.0000.yml rename to scripts/old/data_2023.09.28.0000.0000.yml diff --git a/ida/old/data_2023.10.23.0000.0000.yml b/scripts/old/data_2023.10.23.0000.0000.yml similarity index 100% rename from ida/old/data_2023.10.23.0000.0000.yml rename to scripts/old/data_2023.10.23.0000.0000.yml diff --git a/ida/old/data_2023.11.09.0000.0000.yml b/scripts/old/data_2023.11.09.0000.0000.yml similarity index 100% rename from ida/old/data_2023.11.09.0000.0000.yml rename to scripts/old/data_2023.11.09.0000.0000.yml diff --git a/ida/old/data_2024.01.06.0000.0000.yml b/scripts/old/data_2024.01.06.0000.0000.yml similarity index 100% rename from ida/old/data_2024.01.06.0000.0000.yml rename to scripts/old/data_2024.01.06.0000.0000.yml diff --git a/ida/old/data_2024.02.05.0000.0000.yml b/scripts/old/data_2024.02.05.0000.0000.yml similarity index 100% rename from ida/old/data_2024.02.05.0000.0000.yml rename to scripts/old/data_2024.02.05.0000.0000.yml diff --git a/ida/old/data_2024.03.08.0000.0000.yml b/scripts/old/data_2024.03.08.0000.0000.yml similarity index 100% rename from ida/old/data_2024.03.08.0000.0000.yml rename to scripts/old/data_2024.03.08.0000.0000.yml diff --git a/ida/old/data_2024.03.27.0000.0000.yml b/scripts/old/data_2024.03.27.0000.0000.yml similarity index 100% rename from ida/old/data_2024.03.27.0000.0000.yml rename to scripts/old/data_2024.03.27.0000.0000.yml diff --git a/ida/old/data_2024.04.23.0000.0000.yml b/scripts/old/data_2024.04.23.0000.0000.yml similarity index 100% rename from ida/old/data_2024.04.23.0000.0000.yml rename to scripts/old/data_2024.04.23.0000.0000.yml diff --git a/ida/old/data_2024.06.18.0000.0000.yml b/scripts/old/data_2024.06.18.0000.0000.yml similarity index 100% rename from ida/old/data_2024.06.18.0000.0000.yml rename to scripts/old/data_2024.06.18.0000.0000.yml diff --git a/ida/old/data_2024.07.06.0000.0000.yml b/scripts/old/data_2024.07.06.0000.0000.yml similarity index 100% rename from ida/old/data_2024.07.06.0000.0000.yml rename to scripts/old/data_2024.07.06.0000.0000.yml diff --git a/ida/old/data_2024.07.10.0001.0000.yml b/scripts/old/data_2024.07.10.0001.0000.yml similarity index 100% rename from ida/old/data_2024.07.10.0001.0000.yml rename to scripts/old/data_2024.07.10.0001.0000.yml diff --git a/ida/old/data_2024.07.24.0000.0000.yml b/scripts/old/data_2024.07.24.0000.0000.yml similarity index 100% rename from ida/old/data_2024.07.24.0000.0000.yml rename to scripts/old/data_2024.07.24.0000.0000.yml diff --git a/ida/old/data_2024.08.02.0000.0000.yml b/scripts/old/data_2024.08.02.0000.0000.yml similarity index 100% rename from ida/old/data_2024.08.02.0000.0000.yml rename to scripts/old/data_2024.08.02.0000.0000.yml diff --git a/ida/old/data_2024.08.21.0000.0000.yml b/scripts/old/data_2024.08.21.0000.0000.yml similarity index 100% rename from ida/old/data_2024.08.21.0000.0000.yml rename to scripts/old/data_2024.08.21.0000.0000.yml diff --git a/ida/old/data_2024.11.06.0000.0000.yml b/scripts/old/data_2024.11.06.0000.0000.yml similarity index 100% rename from ida/old/data_2024.11.06.0000.0000.yml rename to scripts/old/data_2024.11.06.0000.0000.yml diff --git a/ida/old/data_2024.11.20.0000.0000.yml b/scripts/old/data_2024.11.20.0000.0000.yml similarity index 100% rename from ida/old/data_2024.11.20.0000.0000.yml rename to scripts/old/data_2024.11.20.0000.0000.yml diff --git a/ida/old/data_2024.12.07.0000.0000.yml b/scripts/old/data_2024.12.07.0000.0000.yml similarity index 100% rename from ida/old/data_2024.12.07.0000.0000.yml rename to scripts/old/data_2024.12.07.0000.0000.yml diff --git a/ida/old/data_2025.01.14.0000.0000.yml b/scripts/old/data_2025.01.14.0000.0000.yml similarity index 100% rename from ida/old/data_2025.01.14.0000.0000.yml rename to scripts/old/data_2025.01.14.0000.0000.yml diff --git a/ida/old/data_2025.01.28.0000.0000.yml b/scripts/old/data_2025.01.28.0000.0000.yml similarity index 100% rename from ida/old/data_2025.01.28.0000.0000.yml rename to scripts/old/data_2025.01.28.0000.0000.yml diff --git a/ida/old/data_2025.02.19.0000.0000.yml b/scripts/old/data_2025.02.19.0000.0000.yml similarity index 100% rename from ida/old/data_2025.02.19.0000.0000.yml rename to scripts/old/data_2025.02.19.0000.0000.yml diff --git a/ida/old/data_2025.02.27.0000.0000.yml b/scripts/old/data_2025.02.27.0000.0000.yml similarity index 100% rename from ida/old/data_2025.02.27.0000.0000.yml rename to scripts/old/data_2025.02.27.0000.0000.yml diff --git a/ida/old/data_2025.03.18.0000.0000.yml b/scripts/old/data_2025.03.18.0000.0000.yml similarity index 100% rename from ida/old/data_2025.03.18.0000.0000.yml rename to scripts/old/data_2025.03.18.0000.0000.yml diff --git a/ida/old/data_2025.03.27.0000.0000.yml b/scripts/old/data_2025.03.27.0000.0000.yml similarity index 100% rename from ida/old/data_2025.03.27.0000.0000.yml rename to scripts/old/data_2025.03.27.0000.0000.yml diff --git a/ida/old/data_2025.04.16.0000.0000.yml b/scripts/old/data_2025.04.16.0000.0000.yml similarity index 100% rename from ida/old/data_2025.04.16.0000.0000.yml rename to scripts/old/data_2025.04.16.0000.0000.yml diff --git a/ida/old/data_2025.05.17.0000.0000.yml b/scripts/old/data_2025.05.17.0000.0000.yml similarity index 100% rename from ida/old/data_2025.05.17.0000.0000.yml rename to scripts/old/data_2025.05.17.0000.0000.yml diff --git a/ida/old/data_2025.06.10.0000.0000.yml b/scripts/old/data_2025.06.10.0000.0000.yml similarity index 100% rename from ida/old/data_2025.06.10.0000.0000.yml rename to scripts/old/data_2025.06.10.0000.0000.yml diff --git a/ida/old/data_2025.06.19.0000.0000.yml b/scripts/old/data_2025.06.19.0000.0000.yml similarity index 100% rename from ida/old/data_2025.06.19.0000.0000.yml rename to scripts/old/data_2025.06.19.0000.0000.yml diff --git a/ida/old/data_2025.06.28.0000.0000.yml b/scripts/old/data_2025.06.28.0000.0000.yml similarity index 100% rename from ida/old/data_2025.06.28.0000.0000.yml rename to scripts/old/data_2025.06.28.0000.0000.yml diff --git a/ida/old/data_2025.07.30.0000.0000.yml b/scripts/old/data_2025.07.30.0000.0000.yml similarity index 100% rename from ida/old/data_2025.07.30.0000.0000.yml rename to scripts/old/data_2025.07.30.0000.0000.yml diff --git a/ida/old/data_2025.08.07.0000.0000.yml b/scripts/old/data_2025.08.07.0000.0000.yml similarity index 100% rename from ida/old/data_2025.08.07.0000.0000.yml rename to scripts/old/data_2025.08.07.0000.0000.yml diff --git a/ida/old/data_2025.08.22.0000.0000.yml b/scripts/old/data_2025.08.22.0000.0000.yml similarity index 100% rename from ida/old/data_2025.08.22.0000.0000.yml rename to scripts/old/data_2025.08.22.0000.0000.yml diff --git a/ida/old/data_2025.09.04.0000.0000.yml b/scripts/old/data_2025.09.04.0000.0000.yml similarity index 100% rename from ida/old/data_2025.09.04.0000.0000.yml rename to scripts/old/data_2025.09.04.0000.0000.yml diff --git a/ida/old/data_2025.09.30.0000.0000.yml b/scripts/old/data_2025.09.30.0000.0000.yml similarity index 100% rename from ida/old/data_2025.09.30.0000.0000.yml rename to scripts/old/data_2025.09.30.0000.0000.yml diff --git a/ida/old/data_2025.10.13.0000.0000.yml b/scripts/old/data_2025.10.13.0000.0000.yml similarity index 100% rename from ida/old/data_2025.10.13.0000.0000.yml rename to scripts/old/data_2025.10.13.0000.0000.yml diff --git a/ida/old/data_2025.10.30.0000.0000.yml b/scripts/old/data_2025.10.30.0000.0000.yml similarity index 100% rename from ida/old/data_2025.10.30.0000.0000.yml rename to scripts/old/data_2025.10.30.0000.0000.yml diff --git a/ida/old/data_2025.12.09.0000.0000.yml b/scripts/old/data_2025.12.09.0000.0000.yml similarity index 100% rename from ida/old/data_2025.12.09.0000.0000.yml rename to scripts/old/data_2025.12.09.0000.0000.yml diff --git a/ida/old/data_2025.12.18.0000.0000.yml b/scripts/old/data_2025.12.18.0000.0000.yml similarity index 100% rename from ida/old/data_2025.12.18.0000.0000.yml rename to scripts/old/data_2025.12.18.0000.0000.yml diff --git a/ida/old/data_2025.12.23.0000.0000.yml b/scripts/old/data_2025.12.23.0000.0000.yml similarity index 100% rename from ida/old/data_2025.12.23.0000.0000.yml rename to scripts/old/data_2025.12.23.0000.0000.yml diff --git a/ida/old/data_2026.01.21.0000.0000.yml b/scripts/old/data_2026.01.21.0000.0000.yml similarity index 100% rename from ida/old/data_2026.01.21.0000.0000.yml rename to scripts/old/data_2026.01.21.0000.0000.yml diff --git a/ida/old/data_2026.01.30.0000.0000.yml b/scripts/old/data_2026.01.30.0000.0000.yml similarity index 100% rename from ida/old/data_2026.01.30.0000.0000.yml rename to scripts/old/data_2026.01.30.0000.0000.yml diff --git a/ida/old/data_2026.02.20.0000.0000.yml b/scripts/old/data_2026.02.20.0000.0000.yml similarity index 100% rename from ida/old/data_2026.02.20.0000.0000.yml rename to scripts/old/data_2026.02.20.0000.0000.yml diff --git a/ida/old/data_2026.03.07.0000.0000.yml b/scripts/old/data_2026.03.07.0000.0000.yml similarity index 100% rename from ida/old/data_2026.03.07.0000.0000.yml rename to scripts/old/data_2026.03.07.0000.0000.yml diff --git a/ida/old/data_2026.03.17.0000.0000.yml b/scripts/old/data_2026.03.17.0000.0000.yml similarity index 100% rename from ida/old/data_2026.03.17.0000.0000.yml rename to scripts/old/data_2026.03.17.0000.0000.yml diff --git a/ida/old/data_2026.04.21.0000.0000.yml b/scripts/old/data_2026.04.21.0000.0000.yml similarity index 100% rename from ida/old/data_2026.04.21.0000.0000.yml rename to scripts/old/data_2026.04.21.0000.0000.yml diff --git a/ida/old/data_2026.05.01.0000.0000.yml b/scripts/old/data_2026.05.01.0000.0000.yml similarity index 100% rename from ida/old/data_2026.05.01.0000.0000.yml rename to scripts/old/data_2026.05.01.0000.0000.yml diff --git a/ida/old/data_2026.05.25.0000.0000.yml b/scripts/old/data_2026.05.25.0000.0000.yml similarity index 100% rename from ida/old/data_2026.05.25.0000.0000.yml rename to scripts/old/data_2026.05.25.0000.0000.yml diff --git a/ida/old/data_2026.06.10.0000.0000.yml b/scripts/old/data_2026.06.10.0000.0000.yml similarity index 100% rename from ida/old/data_2026.06.10.0000.0000.yml rename to scripts/old/data_2026.06.10.0000.0000.yml diff --git a/ida/old/data_2026.06.18.0000.0000.yml b/scripts/old/data_2026.06.18.0000.0000.yml similarity index 100% rename from ida/old/data_2026.06.18.0000.0000.yml rename to scripts/old/data_2026.06.18.0000.0000.yml diff --git a/ida/old/data_2026.07.16.0001.0000.yml b/scripts/old/data_2026.07.16.0001.0000.yml similarity index 100% rename from ida/old/data_2026.07.16.0001.0000.yml rename to scripts/old/data_2026.07.16.0001.0000.yml diff --git a/ida/old/data_2026.08.05.0000.0000.yml b/scripts/old/data_2026.08.05.0000.0000.yml similarity index 100% rename from ida/old/data_2026.08.05.0000.0000.yml rename to scripts/old/data_2026.08.05.0000.0000.yml diff --git a/ida/old/data_2026.08.11.0000.0000.yml b/scripts/old/data_2026.08.11.0000.0000.yml similarity index 100% rename from ida/old/data_2026.08.11.0000.0000.yml rename to scripts/old/data_2026.08.11.0000.0000.yml diff --git a/ida/old/data_2026.09.01.0000.0000.yml b/scripts/old/data_2026.09.01.0000.0000.yml similarity index 100% rename from ida/old/data_2026.09.01.0000.0000.yml rename to scripts/old/data_2026.09.01.0000.0000.yml diff --git a/ida/old/data_2026.09.15.0000.0000.yml b/scripts/old/data_2026.09.15.0000.0000.yml similarity index 100% rename from ida/old/data_2026.09.15.0000.0000.yml rename to scripts/old/data_2026.09.15.0000.0000.yml diff --git a/ida/old/data_2026.10.06.0000.0000.yml b/scripts/old/data_2026.10.06.0000.0000.yml similarity index 100% rename from ida/old/data_2026.10.06.0000.0000.yml rename to scripts/old/data_2026.10.06.0000.0000.yml diff --git a/ida/old/ffxiv_idarename_2020.10.21.0001.0000.py b/scripts/old/ffxiv_idarename_2020.10.21.0001.0000.py similarity index 100% rename from ida/old/ffxiv_idarename_2020.10.21.0001.0000.py rename to scripts/old/ffxiv_idarename_2020.10.21.0001.0000.py diff --git a/ida/old/ffxiv_idarename_2020.12.02.0000.0000.py b/scripts/old/ffxiv_idarename_2020.12.02.0000.0000.py similarity index 100% rename from ida/old/ffxiv_idarename_2020.12.02.0000.0000.py rename to scripts/old/ffxiv_idarename_2020.12.02.0000.0000.py diff --git a/ida/old/ffxiv_idarename_2020.12.15.0000.0000.py b/scripts/old/ffxiv_idarename_2020.12.15.0000.0000.py similarity index 100% rename from ida/old/ffxiv_idarename_2020.12.15.0000.0000.py rename to scripts/old/ffxiv_idarename_2020.12.15.0000.0000.py diff --git a/ida/ffxiv_addonmagic.py b/scripts/old_scripts/ffxiv_addonmagic.py similarity index 100% rename from ida/ffxiv_addonmagic.py rename to scripts/old_scripts/ffxiv_addonmagic.py diff --git a/ida/ffxiv_sigmaker.py b/scripts/old_scripts/ffxiv_sigmaker.py similarity index 100% rename from ida/ffxiv_sigmaker.py rename to scripts/old_scripts/ffxiv_sigmaker.py diff --git a/ida/poetry.lock b/scripts/poetry.lock similarity index 100% rename from ida/poetry.lock rename to scripts/poetry.lock diff --git a/ida/pyproject.toml b/scripts/pyproject.toml similarity index 100% rename from ida/pyproject.toml rename to scripts/pyproject.toml diff --git a/scripts/shared/class_factory.py b/scripts/shared/class_factory.py new file mode 100644 index 0000000000..df6eeaab06 --- /dev/null +++ b/scripts/shared/class_factory.py @@ -0,0 +1,326 @@ +from __future__ import annotations + +import itertools +from collections import deque +from dataclasses import dataclass, field +from typing import Any + +from anytree import Node, RenderTree, PreOrderIter + +from data_schema import DefinedData, DefinedDataClass, DefinedDataClassInstance, DefinedDataClassVtable + +STANDARD_IMAGE_BASE = 0x140000000 + + +@dataclass(eq=False) +class Vtbl: + """A class's vtbl.""" + # Address of vtbl + ea: int + # Name of the base class this vtbl inherits, if it exists + base_name: str | None = None + # Explicit size of vtbl used to override cases where size detection fails + size: int | None = None + resolved_base: FfxivClass | None = None + + @classmethod + def from_data(cls, vtbl_data: DefinedDataClassVtable, rebase_offset: int = 0) -> Vtbl: + """A vtbl from its data.yml definition, rebased by rebase_offset.""" + return cls(ea=vtbl_data.ea + rebase_offset, base_name=vtbl_data.base, size=vtbl_data.size) + + +@dataclass(eq=False, repr=False) +class FfxivClass: + """A class of data.yml.""" + # Disassembler api, see BaseApi in ffxiv_idarename.py + api: Any + name: str + vtbls: list[Vtbl] = field(default_factory=list) + # Mapping of vtbl index to func names + vfuncs: dict[int, str] = field(default_factory=dict) + # Mapping of effective addresses to func names + funcs: dict[int, str] = field(default_factory=dict) + # List of global instances of the object + instances: list[DefinedDataClassInstance] = field(default_factory=list) + first_base: FfxivClass | None = None + + _main_vtbl_size: int = 0 + _inheritance_tree: Node | None = None + _finalized: bool = False + + @classmethod + def from_data(cls, api: Any, class_name: str, class_data: DefinedDataClass) -> FfxivClass: + """A class from its data.yml definition.""" + rebase_offset = api.get_image_base() - STANDARD_IMAGE_BASE + + return cls( + api=api, + name=class_name, + vtbls=[Vtbl.from_data(vtbl, rebase_offset) for vtbl in class_data.vtbls], + vfuncs={vfunc.num: vfunc.name for vfunc in class_data.vfuncs}, + funcs={func.num + rebase_offset: func.name for func in class_data.functions}, + instances=class_data.instances, + ) + + # region vtbl_size + + @property + def main_vtbl_size(self) -> int: + """The vfunc count, the data.yml size or up to the first non-offset or xref.""" + if not self.vtbls: + return self._main_vtbl_size + + if self.vtbls[0].size: + self._main_vtbl_size = self.vtbls[0].size + return self._main_vtbl_size + + if self._main_vtbl_size == 0: + self._main_vtbl_size = 1 # skip the first entry + for ea in itertools.count(self.vtbls[0].ea + 8, 8): + if self.api.get_addr_name(ea) != '': + break + + if self.api.is_offset(ea) and self.api.xrefs_to(ea) == []: + self._main_vtbl_size += 1 + else: + break + + if self.vtbls[0].resolved_base and self._main_vtbl_size < self.vtbls[0].resolved_base._main_vtbl_size: + print( + "Error: The sum of \"{0}\"'s base vtbl sizes ({1}) is greater than the actual class itself ({2})" + .format(self.name, self.vtbls[0].resolved_base._main_vtbl_size, self._main_vtbl_size)) + + return self._main_vtbl_size + + # endregion + + @property + def inheritance_tree(self) -> Node: + def recurse_tree(current_node: Node, current_class: FfxivClass, all_bases: set[str]): + for vtbl in current_class.vtbls: + if vtbl.resolved_base and vtbl.resolved_base.name not in all_bases: + new_node = Node(vtbl.resolved_base.name, parent=current_node) + all_bases.add(vtbl.resolved_base.name) + recurse_tree(new_node, vtbl.resolved_base, all_bases) + + if not self._inheritance_tree: + self._inheritance_tree = Node(self.name) + base_set = set() + recurse_tree(self._inheritance_tree, self, base_set) + + return self._inheritance_tree + + # region finalized + + @property + def finalized(self) -> bool: + """If the class and its hierarchy have been written out.""" + if self.vtbls: + return self._finalized and all( + vtbl.resolved_base is not None and vtbl.resolved_base._finalized for vtbl in self.vtbls) + else: + return self._finalized + + @finalized.setter + def finalized(self, value: bool): + """Set if the class has been written out.""" + self._finalized = value + + # endregion + + # region finalize + + def finalize(self): + """Write out this class.""" + self._inherit_func_names_from_main_base() + self._comment_vtbls_with_inheritance_tree() + + self._write_vtbl_functions() + self._write_funcs() + + self._write_instances() + + self.finalized = True + + def _inherit_func_names_from_main_base(self): + """A base is finalized before its child, so it has the vfunc names of its bases.""" + if self.vtbls and self.vtbls[0].resolved_base: + for idx, base_vfunc_name in self.vtbls[0].resolved_base.vfuncs.items(): + if idx in self.vfuncs: + print("Warning: 0x{0:X} \"{1}\" overwrites the name of inherited function \"{2}\"".format( + self.vtbls[0].ea, self.name, base_vfunc_name)) + else: + self.vfuncs[idx] = base_vfunc_name + + def _comment_vtbls_with_inheritance_tree(self): + """Comment the inheritance tree at the start of the vtbl.""" + for vtbl in self.vtbls: + comment = self.api.get_comment(vtbl.ea) or "" + + for pre, _, node in RenderTree(self.inheritance_tree): + if comment: + comment += "\n" + comment += pre + self.api.format_class_name(node.name) + self.api.set_comment(vtbl.ea, comment) + + def write_vtbl_names(self): + """Write out the vtbl names.""" + self.api.set_addr_name(self.vtbls[0].ea, self.api.format_class_name_for_vtbl(self.name)) + for vtbl in self.vtbls[1:]: + self.api.set_addr_name(vtbl.ea, self.api.format_class_name_for_secondary_vtbl(self.name, + vtbl.base_name)) + + def _write_vtbl_functions(self): + """Write out the vtbl function names.""" + + def collect_vtbl_functions(ea: int, size: int, class_name: str, vfunc_names: dict[int, str], + base_class_names: list[str]) -> list[tuple[int, str]]: + vtbl_builder = [] + for func_idx in range(0, size): + vtbl_vfunc_ea = ea + func_idx * 8 + vfunc_ea = self.api.get_qword(vtbl_vfunc_ea) # type: int + + current_func_name = self.api.get_addr_name(vfunc_ea) # type: str + proposed_func_name = vfunc_names.get(func_idx, "vf{0}".format(func_idx)) + formatted_class_name = self.api.format_class_name(class_name) + + formatted_func_name = self.api.format_vfunc_name(vfunc_ea, current_func_name, proposed_func_name, + formatted_class_name, + base_class_names) + + if formatted_func_name == "": + pass + elif formatted_func_name is None: + print( + "Error: Function at 0x{0:X} had unexpected name \"{1}\" during naming of {2}.{3} (vtbl[{4}])" + .format(vfunc_ea, current_func_name, self.name, proposed_func_name, func_idx)) + else: + vtbl_builder.append((vfunc_ea, formatted_func_name)) + + return vtbl_builder + + if self.vtbls: + formatted_base_class_names = [self.api.format_class_name(node.name) for node in + PreOrderIter(self.inheritance_tree)] + formatted_base_class_names.remove(self.name) + for idx, vtbl in enumerate(self.vtbls): + if idx == 0: + funcs = collect_vtbl_functions(vtbl.ea, self.main_vtbl_size, self.name, self.vfuncs, + formatted_base_class_names) + else: + funcs = collect_vtbl_functions(vtbl.ea, vtbl.resolved_base.main_vtbl_size, + self.name + "___" + vtbl.resolved_base.name, + vtbl.resolved_base.vfuncs, formatted_base_class_names) + for (func_ea, func_name) in funcs: + self.api.set_addr_name(func_ea, func_name) + + def _write_funcs(self): + """Write the names of all non-vtbl funcs.""" + for func_ea, proposed_func_name in self.funcs.items(): + current_func_name = self.api.get_addr_name(func_ea) # type: str + + func_name = self.api.format_func_name(func_ea, current_func_name, proposed_func_name, self.name) + if func_name == "": + pass + elif func_name is None: + print("Error: Function at 0x{0:X} had unexpected name \"{1}\" during naming of {2}.{3}" + .format(func_ea, current_func_name, self.name, proposed_func_name)) + else: + self.api.set_addr_name(func_ea, func_name) + + def _write_instances(self): + """Write the names of all instances.""" + for instance in self.instances: + name = "g_{}_{}".format(self.name, instance.name or "Instance") + self.api.set_addr_name(instance.ea, name) + + # endregion + + def __repr__(self): + return "<{0}(\"{1}\")>".format(self.__class__.__name__, self.name) + + +@dataclass +class FfxivClassFactory: + # Disassembler api, see BaseApi in ffxiv_idarename.py + api: Any + _vtbl_addresses: list[int] = field(default_factory=list) + _classes: dict[str, FfxivClass] = field(default_factory=dict) + _finalize_stack: deque[FfxivClass] = field(default_factory=deque) + + def register(self, class_name: str, class_data: DefinedDataClass | None = None): + """Register a class, a placeholder if class_data is omitted.""" + if class_data is None: + class_data = DefinedDataClass() + + for vtbl in class_data.vtbls: + if vtbl.ea != 0x0 and vtbl.ea in self._vtbl_addresses: + print("Error: Multiple vtables are defined at 0x{0:X}".format(vtbl.ea)) + return + + if class_name in self._classes: + print("Error: Multiple classes are registered with the name \"{0}\"".format(class_name)) + return + for vtbl in class_data.vtbls: + self._vtbl_addresses.append(vtbl.ea) + + self._classes[class_name] = FfxivClass.from_data(self.api, class_name, class_data) + + def finalize(self): + """Name every class.""" + self._resolve_base_classes() + + # We write the vtbl names first for an added check so that + # the size finder will not advance past a named offset. + for cls in self._classes.values(): + if cls.vtbls: + cls.write_vtbl_names() + + for cls in self._classes.values(): + self._finalize_class(cls) + + def _resolve_base_classes(self): + """Resolve the base classes, missing ones get a stub entry and a warning.""" + for cls in list(self._classes.values()): + for idx, vtbl in enumerate(cls.vtbls): + if vtbl.resolved_base is None and vtbl.base_name: + if vtbl.base_name not in self._classes: + print("Warning: Inherited class \"{0}\" is not documented, add a placeholder entry".format( + vtbl.base_name)) + self.register(class_name=vtbl.base_name) + vtbl.resolved_base = self._classes[vtbl.base_name] + if idx == 0: + cls.first_base = self._classes[vtbl.base_name] + + def _finalize_class(self, cls: FfxivClass): + """Name a single class.""" + if cls in self._finalize_stack: + names = [c.name for c in self._finalize_stack] + [cls.name] + names = "\n".join([" - {0}".format(name) for name in names]) + raise ValueError("Inheritance cycle detected: \n{0}".format(names)) + + if not cls.finalized: + self._finalize_stack.append(cls) + for vtbl in cls.vtbls: + if vtbl.resolved_base and not vtbl.resolved_base.finalized: + self._finalize_class(vtbl.resolved_base) + cls.finalize() + + self._finalize_stack.pop() + + +def apply_data(api: Any, data: DefinedData): + """Name everything in data.yml.""" + for section in (data.globals, data.functions): + for ea, name in (section or {}).items(): + if not isinstance(ea, int): + print('Warning: {0} has an invalid address {1}'.format(name, ea)) + continue + api.set_addr_name(ea, name) + + if data.classes: + factory = FfxivClassFactory(api) + for class_name, class_data in data.classes.items(): + factory.register(class_name, class_data) + + factory.finalize() diff --git a/scripts/shared/data_schema.py b/scripts/shared/data_schema.py new file mode 100644 index 0000000000..c8ccdf30dc --- /dev/null +++ b/scripts/shared/data_schema.py @@ -0,0 +1,91 @@ +from dataclasses import dataclass, field, fields +from typing import TypeAlias +from yaml import load + +from yaml_loader import Loader + + +@dataclass +class DefinedDataClassInstance: + ea: int + pointer: bool = False + name: str | None = None + + +@dataclass +class DefinedDataClassVtable: + ea: int + base: str | None = None + size: int | None = None + + +@dataclass +class DefinedDataClassFunction: + num: int + name: str + + +@dataclass +class DefinedDataClass: + instances: list[DefinedDataClassInstance] = field(default_factory=list) + vtbls: list[DefinedDataClassVtable] = field(default_factory=list) + functions: list[DefinedDataClassFunction] = field( + default_factory=list, metadata={"yaml_name": "funcs"} + ) + vfuncs: list[DefinedDataClassFunction] = field(default_factory=list) + + +@dataclass +class DefinedData: + classes: dict[str, DefinedDataClass] = field(default_factory=dict) + version: str | None = None + globals: dict[int, str] = field(default_factory=dict) + functions: dict[int, str] = field(default_factory=dict) + + +YamlExport: TypeAlias = dict[str, object] + + +def get_yaml(stream, loader=Loader) -> DefinedData: + dic: YamlExport = load(stream, loader) + classes = {} + class_keys = {f.metadata.get("yaml_name", f.name) for f in fields(DefinedDataClass)} + for name, class_values in (dic.get("classes") or {}).items(): + class_values = class_values or {} + values = dict(class_values) + + for leftover in class_values.keys() - class_keys: + print('Warning: Extra key "{0}" present in {1}'.format(leftover, name)) + + for key, value_type in ( + ("instances", DefinedDataClassInstance), + ("vtbls", DefinedDataClassVtable), + ): + if key in class_values and class_values[key] is not None: + values[key] = [ + _from_dict(value_type, value) for value in class_values[key] + ] + else: + values.pop(key, None) + + for yaml_key in ("funcs", "vfuncs"): + if yaml_key in class_values and class_values[yaml_key] is not None: + values[yaml_key] = [ + DefinedDataClassFunction(num, function_name) + for num, function_name in class_values[yaml_key].items() + ] + else: + values.pop(yaml_key, None) + + classes[name] = _from_dict(DefinedDataClass, values) + + return _from_dict(DefinedData, {**dic, "classes": classes}) + + +def _from_dict(data_class, values): + class_values = {} + for data_field in fields(data_class): + key = data_field.metadata.get("yaml_name", data_field.name) + if key in values: + class_values[data_field.name] = values[key] + return data_class(**class_values) diff --git a/scripts/shared/exdschema_wrapper.py b/scripts/shared/exdschema_wrapper.py new file mode 100644 index 0000000000..f4e099e69a --- /dev/null +++ b/scripts/shared/exdschema_wrapper.py @@ -0,0 +1,311 @@ +from urllib.request import urlopen +from zipfile import ZipFile +from io import BytesIO +from json import load as jsonload +from dataclasses import dataclass +from yaml import load as yamlload +from typing import TypeAlias, Union +from lumina_wrapper import ExcelColumnDefinition, ExcelColumnDataType +from structs_schema import ( + DefinedStruct, + DefinedStructEnum, + DefinedStructFixedField, + DefinedStructField, + DefinedStructExport, + Fields, +) + +from yaml_loader import Loader + +exd_namespace = "Component::Exd::Sheets" + +ExdSchemaYaml: TypeAlias = list[dict[str, Union[str, int, "ExdSchemaYaml"]]] + + +@dataclass +class Definition: + name: str + comment: str | None + + def size(self) -> int: + return 1 + + +@dataclass +class RepeatDefinition(Definition): + count: int + fields: "DefinitionFields" + + def size(self) -> int: + return len(self.fields) * self.count if len(self.fields) > 0 else self.count + + +DefinitionFields: TypeAlias = list[Union[Definition, RepeatDefinition]] + + +def get_exdschema_versions(): + print("Fetching https://api.github.com/repos/xivdev/exdschema/branches") + with urlopen("https://api.github.com/repos/xivdev/exdschema/branches") as response: + vers: list[str] = [] + for branch in jsonload(response): + vers.append(branch["name"]) + return vers + + +def process_exdschema_definition(fields: ExdSchemaYaml): + definitions: DefinitionFields = [] + for field in fields: + if "type" in field and field["type"] == "array": + definitions.append( + RepeatDefinition( + field["name"] if "name" in field else "", + field["comment"] if "comment" in field else None, + field["count"], + ( + process_exdschema_definition(field["fields"]) + if "fields" in field + else [] + ), + ) + ) + elif "name" in field: + definitions.append( + Definition( + field["name"], field["comment"] if "comment" in field else None + ) + ) + return definitions + + +def get_exdschema_data(version: str): + print(f"Downloading https://github.com/xivdev/EXDSchema/archive/refs/heads/{version}.zip") + with urlopen( + f"https://github.com/xivdev/EXDSchema/archive/refs/heads/{version}.zip" + ) as response: + with ZipFile(BytesIO(response.read())) as z: + exd_schema_map: dict[str, DefinitionFields] = {} + for file in z.namelist(): + if file.endswith(".yml") and ".github" not in file: + yaml_data = yamlload(z.read(file), Loader) + exd_schema_map[file.rsplit(".", 1)[0].rsplit("/")[1]] = ( + process_exdschema_definition(yaml_data["fields"]) + ) + return exd_schema_map + + +def get_size_from_string(type: str) -> int: + if type == "int8_t" or type == "uint8_t" or type == "bool": + return 1 + if type == "int16_t" or type == "uint16_t": + return 2 + if type == "int32_t" or type == "uint32_t" or type == "float": + return 4 + if ( + type == "int64_t" + or type == "uint64_t" + or type == "double" + or type.endswith("*") + ): + return 8 + return 0 + + +def find_struct(structs: list[DefinedStruct], name: str): + for struct in structs: + if struct.name == name: + return struct + return None + + +def map_header_and_schema( + header: list[ExcelColumnDefinition], + schema: DefinitionFields, + header_index_offset: int = 0, + schema_index_offset: int = 0, + schema_name: str = "", +): + fields: Fields = [] + enums: list[DefinedStructEnum] = [] + enum_field_map: dict[int, int] = {} + structs: list[DefinedStruct] = [] + prev_offset = 0 + while prev_offset < len(schema): + header_offset = sum([f.size() if isinstance(f, RepeatDefinition) else 1 for f in schema[:prev_offset]]) + definition = header[header_offset + schema_index_offset] + field = schema[prev_offset] + definition_offset = definition.offset - header_index_offset + + if isinstance(field, RepeatDefinition): + exported_enums, exported_structs = map_header_and_schema( + header, field.fields, definition.offset, header_offset, schema_name + field.name + ) + if len(exported_structs) == 0: + base_type = definition.get_base_type_string() + fields.append( + DefinedStructFixedField( + field.name, + base_type, + definition_offset, + False, + [], + [], + field.count, + False, + ) + ) + else: + fields.append( + DefinedStructFixedField( + field.name, + exported_structs[-1].type, + definition_offset, + False, + [], + [], + field.count, + False, + ) + ) + structs.extend(exported_structs) + enums.extend(exported_enums) + else: + if definition.is_packed_bool(): + if definition_offset in enum_field_map: + enums[enum_field_map[definition_offset]].values[field.name] = ( + definition.type - ExcelColumnDataType.PackedBool0 + ) + else: + enum_index = len(enums) + enum_field_map[definition_offset] = enum_index + enum_name = f"{schema_name}PackedBool{definition_offset:X}" + enum_type = f"{exd_namespace}::{enum_name}" + enums.append( + DefinedStructEnum( + enum_name, + enum_type, + exd_namespace, + definition.get_base_type_string(), + True, + { + field.name: definition.type + - ExcelColumnDataType.PackedBool0 + }, + ) + ) + fields.append( + DefinedStructField( + f"PackedBool{definition_offset:X}", + enum_type, + definition_offset, + False, + [], + [], + ) + ) + else: + fields.append( + DefinedStructField( + field.name, + definition.get_base_type_string(), + definition_offset, + False, + [], + [], + ) + ) + + prev_offset += 1 + + size: int = 0 + for field in fields: + if isinstance(field, DefinedStructFixedField): + struct = find_struct(structs, field.type) + if struct == None: + field_size = get_size_from_string(field.type) + if field_size == 0: + field_size = 1 + size += field_size + else: + size += struct.size * field.size + else: + field_size = get_size_from_string(field.type) + if field_size == 0: + field_size = 1 + size += field_size + + size = (size + 3) & ~3 + + if len(fields) != 0: + structs.append( + DefinedStruct( + schema_name, + f"{exd_namespace}::{schema_name}", + exd_namespace, + fields, + size, + None, + None, + [], + False, + None, + None, + [], + [], + [], + False + ) + ) + + return (enums, structs) + + +def create_struct_from_header_and_schema( + headers: dict[str, tuple[int, list[ExcelColumnDefinition]]], + schemas: dict[str, DefinitionFields], +) -> tuple[DefinedStructExport, dict[str, int]]: + struct_export = DefinedStructExport([], []) + excel_map: dict[str, int] = {} + for excel_header_name in headers: + index, header = headers[excel_header_name] + schema = schemas[excel_header_name] + excel_map[excel_header_name] = index + enums, structs = map_header_and_schema(header, schema, 0, 0, excel_header_name) + struct_export.enums.extend(enums) + struct_export.structs.extend(structs) + + struct_export.enums.append(DefinedStructEnum( + "SheetsEnum", + "Component::Exd::SheetsEnum", + "Component::Exd", + "uint32_t", + False, + excel_map)) + + return (struct_export, excel_map) + +exd_func_patterns: dict[str, tuple[str, tuple[str | None, int]]] = { + "48 83 EC 28 48 8B 05 ? ? ? ? 44 8B C1 BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ("Row", ("48 8B 05 ? ? ? ? 44 8B C1 BA", 11)), + "48 83 EC 28 85 C9 74 20 48 8B 05 ? ? ? ? 44 8B C1 BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ("Row", ("48 8B 05 ? ? ? ? 44 8B C1 BA", 11)), + "48 83 EC 28 48 8B 05 ? ? ? ? BA ? ? ? ? 44 0F B6 C1 48 8B 88 ? ? ? ? E8": ("Row", ("48 8B 05 ? ? ? ? BA", 8)), + "48 83 EC 28 48 8B 05 ? ? ? ? BA ? ? ? ? 44 0F B7 C1 48 8B 88 ? ? ? ? E8": ("Row", ("48 8B 05 ? ? ? ? BA", 8)), + "48 83 EC 28 48 8B 05 ? ? ? ? BA ? ? ? ? 44 0F B7 81 ? ? ? ? 48 8B 88 ? ? ? ? E8": ("Row", ("48 8B 05 ? ? ? ? BA", 8)), + "40 53 48 83 EC 20 48 8B 05 ? ? ? ? 44 8B C1 8B DA BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ("Row", ("44 8B C1 8B DA BA", 6)), + "48 83 EC 28 48 8B 05 ? ? ? ? 44 8B C1 84 D2 BA ? ? ? ? 48 8B 88 ? ? ? ? 74 07 E8": ("Row", ("44 8B C1 84 D2 BA", 6)), + "48 83 EC 38 48 8B 05 ? ? ? ? 44 8B CA 44 8B C1 48 C7 44 24 ? ? ? ? ? BA ? ? ? ? 48 C7 44 24 ? ? ? ? ? 48 8B 88 ? ? ? ? E8": ("RowAndSubRowId", ("C1 48 C7 44 24 ? ? ? ? ? BA", 11)), + "48 83 EC 38 48 8B 05 ? ? ? ? 44 8B C1 44 0F ? CA BA ? ? ? ? 48 C7 44 24 28 ? ? ? ? 48 C7 44 24 20 ? ? ? ? 48 8B 88 ? ? ? ? E8": ("RowAndSubRowId", ("44 8B C1 44 0F ? CA BA", 8)), + "48 83 EC 28 48 8B 05 ? ? ? ? 44 8D 81 ? ? ? ? BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ("RowIndex", ("05 ? ? ? ? 44 8D 81 ? ? ? ? BA", 13)), + "48 83 EC 28 8D 41 ? 3D ? ? ? ? 77 20 48 8B 05 ? ? ? ? 44 8B C1 BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ("RowIndex", ("48 8B 05 ? ? ? ? 44 8B C1 BA", 11)), + "48 83 EC 28 48 8B 05 ? ? ? ? BA ? ? ? ? 48 8B 88 ? ? ? ? E8": ("SheetIndex", ("48 8B 05 ? ? ? ? BA", 8)), + "48 8B 05 ? ? ? ? BA ? ? ? ? 48 8B 88 ? ? ? ? E9 ? ? ? ?": ("RowCount", (None, 8)) +} + +exd_comment_patterns: dict[str, int] = { + "48 8B 05 ? ? ? ? BA ? ? ? ? 48 8B 88 40 2B 00 00": 8, + "48 8B 05 ? ? ? ? BA ? ? ? ? 44 0F": 8, + "48 8B 05 ? ? ? ? 44 8B C1 8B DA BA": 13, + "48 8B 05 ? ? ? ? 44 8B C1 BA": 11, + "48 8B 05 ? ? ? ? 44 8B CA 44 8B C1 48 C7 44 24 ? ? ? ? ? BA": 23, + "48 8B 05 ? ? ? ? 44 8B C1 84 D2 BA": 13, + "48 8B 05 ? ? ? ? 44 8B C1 44 0F ? CA BA": 15, + "48 8B 05 ? ? ? ? 44 8D 81 ? ? ? ? BA": 15 +} \ No newline at end of file diff --git a/scripts/shared/indent_writer.py b/scripts/shared/indent_writer.py new file mode 100644 index 0000000000..178b3b6cf2 --- /dev/null +++ b/scripts/shared/indent_writer.py @@ -0,0 +1,32 @@ +from dataclasses import dataclass + +@dataclass +class IndentWriterValue: + indent_level: int + line: str + + def __str__(self): + return " " * (self.indent_level * 2) + self.line + +class IndentWriter(object): + def __init__(self): + self.indent_level: int = 0 + self.lines: list[IndentWriterValue] = [] + + def indent(self): + self.indent_level += 1 + + def unindent(self): + self.indent_level -= 1 + if (self.indent_level < 0): + self.indent_level = 0 + + def append(self, line: str): + self.lines.append(IndentWriterValue(self.indent_level, line)) + + def __iadd__(self, line: str): + self.append(line) + return self + + def __str__(self): + return "\n".join([str(line) for line in self.lines]) \ No newline at end of file diff --git a/scripts/shared/lumina_wrapper.py b/scripts/shared/lumina_wrapper.py new file mode 100644 index 0000000000..498e9e616f --- /dev/null +++ b/scripts/shared/lumina_wrapper.py @@ -0,0 +1,334 @@ +from urllib.request import urlopen +from zipfile import ZipFile +from io import BytesIO +from os.path import join, basename, abspath, dirname, isdir, isfile +from os import getenv, getcwd, makedirs, listdir +from sys import path +from shutil import rmtree +from xml.dom import pulldom +from pythonnet import load as dotnetload +from json import load as jsonload +from yaml import load as yamlload, dump as yamldump +from yaml_loader import Loader, Dumper +from typing import Callable +from enum import IntEnum +import re +from dataclasses import dataclass +import gc + +default_game_path = r"C:\Program Files (x86)\SquareEnix\FINAL FANTASY XIV - A Realm Reborn" +steam_install_folder = "FINAL FANTASY XIV Online" +# one file per data.yml version, so Lumina only reads the game once for each version +excel_header_folder = abspath(join(dirname(__file__), "..", "excel_headers")) + + +def is_game_path(game_path: str | None) -> bool: + """If the folder holds both game and boot.""" + return bool(game_path) and isdir(join(game_path, "game", "sqpack")) and isdir(join(game_path, "boot")) + + +def _get_xlcore_game_path() -> str | None: + if getenv("HOME") is None: + return None + launcher_ini = join(getenv("HOME"), ".xlcore", "launcher.ini") + if not isfile(launcher_ini): + return None + with open(launcher_ini, "r") as f: + for line in f: + if line.startswith("GamePath="): + return line.split("=", 1)[1].strip() + return None + + +def _get_xivlauncher_game_path() -> str | None: + if getenv("APPDATA") is None: + return None + launcher_config = join(getenv("APPDATA"), "XIVLauncher", "launcherConfigV3.json") + if not isfile(launcher_config): + return None + with open(launcher_config, "r") as f: + return jsonload(f).get("GamePath") + + +def _get_steam_paths() -> list[str]: + steam_paths = [] + try: + import winreg + + with winreg.OpenKey(winreg.HKEY_CURRENT_USER, r"Software\Valve\Steam") as key: + steam_paths.append(winreg.QueryValueEx(key, "SteamPath")[0]) + except (ImportError, OSError): + pass + steam_paths.append(r"C:\Program Files (x86)\Steam") + if getenv("HOME") is not None: + steam_paths.extend( + join(getenv("HOME"), *folder) + for folder in ( + (".local", "share", "Steam"), + (".steam", "steam"), + (".var", "app", "com.valvesoftware.Steam", ".local", "share", "Steam"), + ) + ) + return steam_paths + + +def _get_steam_game_paths() -> list[str]: + """The game folder in every Steam library.""" + libraries = [] + for steam_path in _get_steam_paths(): + libraries.append(steam_path) + library_folders = join(steam_path, "steamapps", "libraryfolders.vdf") + if isfile(library_folders): + with open(library_folders, "r", encoding="utf-8") as f: + libraries.extend( + library.replace("\\\\", "\\") + for library in re.findall(r'"path"\s+"([^"]+)"', f.read()) + ) + return [join(library, "steamapps", "common", steam_install_folder) for library in dict.fromkeys(libraries)] + + +def find_game_path() -> str | None: + """The game install from it's known locations, None if none of them hold it.""" + for get_paths in ( + lambda: [_get_xlcore_game_path()], + lambda: [_get_xivlauncher_game_path()], + lambda: [default_game_path], + _get_steam_game_paths, + ): + try: + game_paths = get_paths() + except (OSError, ValueError): + # a config that can not be read is the same as not having one + continue + for game_path in game_paths: + if is_game_path(game_path): + return game_path + return None + + +class ExcelColumnDataType(IntEnum): + String = 0x0 + Bool = 0x1 + Int8 = 0x2 + UInt8 = 0x3 + Int16 = 0x4 + UInt16 = 0x5 + Int32 = 0x6 + UInt32 = 0x7 + # unused? + Unk = 0x8 + Float32 = 0x9 + Int64 = 0xA + UInt64 = 0xB + # unused? + Unk2 = 0xC + # 0 is read like data & 1, 1 is like data & 2, 2 = data & 4, etc... + PackedBool0 = 0x19 + PackedBool1 = 0x1A + PackedBool2 = 0x1B + PackedBool3 = 0x1C + PackedBool4 = 0x1D + PackedBool5 = 0x1E + PackedBool6 = 0x1F + PackedBool7 = 0x20 + + +@dataclass +class ExcelColumnDefinition: + type: ExcelColumnDataType + offset: int + + def is_packed_bool(self): + return self.type >= ExcelColumnDataType.PackedBool0 + + def get_base_type_string(self): + match self.type: + case ExcelColumnDataType.Bool: + return "bool" + case ExcelColumnDataType.UInt8: + return "uint8_t" + case ExcelColumnDataType.Int8: + return "int8_t" + case ExcelColumnDataType.UInt16: + return "uint16_t" + case ExcelColumnDataType.Int16: + return "int16_t" + case ExcelColumnDataType.String | ExcelColumnDataType.UInt32: + return "uint32_t" + case ExcelColumnDataType.Int32: + return "int32_t" + case ExcelColumnDataType.Float32: + return "float" + case ExcelColumnDataType.Int64: + return "int64_t" + case ExcelColumnDataType.UInt64: + return "uint64_t" + case _ if self.is_packed_bool(): + return "uint8_t" + case _: + raise ValueError(f"Unhandled type: {self.type}") + + def __lt__(self, other): + return self.offset < other.offset + + def __repr__(self): + return self.get_base_type_string() + + +def get_nugpkg_and_extract(tempdirname: str, package: str, version: str = ""): + with urlopen( + f"https://www.nuget.org/api/v2/package/{package}/{version}" + ) as response: + nupkg = response.read() + with ZipFile(BytesIO(nupkg)) as z: + for file in z.namelist(): + file_path = join(tempdirname, basename(file)) + with open(file_path, "wb") as f: + f.write(z.read(file)) + + +def get_nugpkg_deps(nuspec_path: str) -> list[tuple[str, str]]: + deps = [] + stream = pulldom.parse(nuspec_path) + for event, node in stream: + if event == "START_ELEMENT" and node.nodeName == "dependency": + stream.expandNode(node) + deps.append((node.getAttribute("id"), node.getAttribute("version"))) + return deps + + +def get_nugpkg(tempdirname: str, package: str, version: str = ""): + ver_str = f"version {version}" if version != "" else "latest version" + print(f"Requesting nugpkg {package} with {ver_str}") + get_nugpkg_and_extract(tempdirname, package, version) + return get_nugpkg_deps(join(tempdirname, f"{package}.nuspec")) + + +def get_excel_header_file_path(version: str) -> str: + return join(excel_header_folder, f"{version}.yml") + + +def _load_excel_header_files(file_path: str) -> dict[str, tuple[int, list[ExcelColumnDefinition]]]: + with open(file_path, "r", encoding="utf-8") as f: + excel_header_files = yamlload(f, Loader) + return { + name: (id, [ExcelColumnDefinition(type, offset) for type, offset in columns]) + for name, (id, columns) in excel_header_files.items() + } + + +def _save_excel_header_files(file_path: str, excel_header_files: dict[str, tuple[int, list[ExcelColumnDefinition]]]): + makedirs(excel_header_folder, exist_ok=True) + with open(file_path, "w", encoding="utf-8") as f: + yamldump( + { + name: [id, [[int(column.type), column.offset] for column in columns]] + for name, (id, columns) in excel_header_files.items() + }, + f, Dumper, default_flow_style=None, sort_keys=False, width=1 << 16, + ) + + +def get_excel_header_files( + version: str | None, get_game_path: Callable[[], str] +) -> dict[str, tuple[int, list[ExcelColumnDefinition]]]: + """The saved headers of the version, read with Lumina if there are none.""" + if not version: + return _read_excel_header_files(get_game_path()) + + file_path = get_excel_header_file_path(version) + if isfile(file_path): + print(f"Using Excel headers from {file_path}") + return _load_excel_header_files(file_path) + + excel_header_files = _read_excel_header_files(get_game_path()) + _save_excel_header_files(file_path, excel_header_files) + print(f"Saved Excel headers to {file_path}") + return excel_header_files + + +def _read_excel_header_files(game_path: str) -> dict[str, tuple[int, list[ExcelColumnDefinition]]]: + """The headers Lumina reads from the game install.""" + tempdirname = join(getcwd(), "luminatmp") + makedirs(tempdirname, exist_ok=True) + dotnetload("coreclr") + def _isolated_clr(): + import clr + + deps = get_nugpkg(tempdirname, "Lumina") + deps_gotten: dict[str, str] = {} + + while len(deps) > 0: + local_deps = [] + for package, version in deps: + if package not in deps_gotten or deps_gotten[package] != version: + local_deps.extend(get_nugpkg(tempdirname, package, version)) + deps_gotten[package] = version + deps = local_deps + + path.append(tempdirname) + + from System.Runtime.Loader import AssemblyLoadContext # type: ignore + from System import Activator, GC # type: ignore + from System.IO import File, MemoryStream # type: ignore + + ctx = AssemblyLoadContext("temp", isCollectible=True) + + def _isolated_extract(context): + assemblies: dict = {} + + for file in listdir(tempdirname): + if file.endswith(".dll"): + # LoadFromAssemblyPath would keep the file locked until the process exits + assemblies[file] = context.LoadFromStream( + MemoryStream(File.ReadAllBytes(abspath(join(tempdirname, file)))) + ) + + assembly = assemblies["Lumina.dll"] + game_data_type = assembly.GetType("Lumina.GameData") + game_data_instance = Activator.CreateInstance( + game_data_type, [join(game_path, "game", "sqpack"), None] + ) + + excel_list_type = assembly.GetType("Lumina.Data.Files.Excel.ExcelListFile") + excel_header_type = assembly.GetType("Lumina.Data.Files.Excel.ExcelHeaderFile") + + root_exl = game_data_instance.GetFile[excel_list_type]("exd/root.exl") + + excel_header_files: dict[str, tuple[int, list[ExcelColumnDefinition]]] = {} + + for kvp in root_exl.ExdMap: + key = kvp.Key + value = kvp.Value + + if value != -1: + defs: list[ExcelColumnDefinition] = [] + header = game_data_instance.GetFile[excel_header_type](f"exd/{key}.exh") + for column_definition in header.ColumnDefinitions: + defs.append( + ExcelColumnDefinition( + int(column_definition.Type), column_definition.Offset + ) + ) + excel_header_files[key] = (value, sorted(defs)) + + + return excel_header_files + + excel_header_files = _isolated_extract(ctx) + + ctx.Unload() + + del ctx + + gc.collect() + GC.Collect() + GC.WaitForPendingFinalizers() + return excel_header_files + + # pythonnet.unload() would break every later use of clr in the same process + excel_header_files = _isolated_clr() + + rmtree(tempdirname, ignore_errors=True) + + return excel_header_files diff --git a/scripts/shared/src_wrapper.py b/scripts/shared/src_wrapper.py new file mode 100644 index 0000000000..d746f92b89 --- /dev/null +++ b/scripts/shared/src_wrapper.py @@ -0,0 +1,1388 @@ +from structs_schema import ( + DefinedStruct, + DefinedStructFixedField, + DefinedStructField, + DefinedStructExport, + DefinedStructEnum, + DefinedStructVFunc, + get_template_alias, +) +from typing import Callable + +from indent_writer import IndentWriter, IndentWriterValue + +# a probe struct per type the schema can not size, the offset of value is the alignment of the type +layout_probe_prefix = "__cs_layout_probe_" + +class SrcInterface(object): + full_padding: bool = True + ida_vtables: bool = False + + def __init__(self): + self._layout_cache: dict[tuple[str, tuple[str, ...]], tuple[int, bool]] = {} + self._external_layouts: dict[str, tuple[int, int]] = {} + # inline structs of a template use its parameters, ex. the Node of StdLinkedList + self._inherited_template_types: dict[str, tuple[str, ...]] = {} + + def get_type_and_type_size_from_gap(self, gap: int, offset: int = 0) -> tuple[str, int]: + if offset == 0: + offset = gap + if offset % 8 == 0 and gap >= 8: + return ("uint64_t", 8) + elif offset % 4 == 0 and gap >= 4: + return ("uint32_t", 4) + elif offset % 2 == 0 and gap >= 2: + return ("uint16_t", 2) + else: + return ("uint8_t", 1) + + def get_size_from_string(self, type: str) -> int: + if type in ("int8_t", "uint8_t", "bool", "char", "unsigned char", "signed char"): + return 1 + if type in ("int16_t", "uint16_t", "wchar_t"): + return 2 + if type in ("int32_t", "uint32_t", "float"): + return 4 + if ( + type == "int64_t" + or type == "uint64_t" + or type == "double" + or type.endswith("*") + ): + return 8 + return 0 + + def is_base_type(self, type: str) -> bool: + type = type.rstrip('*') + return type in ("byte", "wchar_t", "char", "float", "double", "bool", "int8_t", "uint8_t", "int16_t", "uint16_t", "int32_t", "uint32_t", "int64_t", "uint64_t", "void") + + def _split_template_type(self, type_name: str) -> tuple[str, list[str]]: + type_name = type_name.strip() + template_start = type_name.find("<") + if template_start < 0: + return type_name, [] + + depth = 0 + argument_start = template_start + 1 + arguments: list[str] = [] + for index in range(template_start, len(type_name)): + char = type_name[index] + if char == "<": + depth += 1 + elif char == ">": + depth -= 1 + if depth == 0: + if argument_start < index: + arguments.append(type_name[argument_start:index].strip()) + if type_name[index + 1 :].strip(): + return type_name, [] + return type_name[:template_start].strip(), arguments + elif char == "," and depth == 1: + arguments.append(type_name[argument_start:index].strip()) + argument_start = index + 1 + + return type_name, [] + + def _normalize_type(self, type_name: str) -> str: + """Normalize whitespace without losing nested template structure.""" + type_name = type_name.strip() + pointer_suffix = "" + while type_name.endswith("*"): + pointer_suffix += "*" + type_name = type_name[:-1].strip() + + base_type, template_args = self._split_template_type(type_name) + if template_args: + normalized = f"{base_type}<{','.join(self._normalize_type(arg) for arg in template_args)}>" + else: + normalized = " ".join(base_type.split()) + return normalized + pointer_suffix + + def _qualify_template_arguments( + self, type_name: str, template_parameters: tuple[str, ...] = () + ) -> str: + """Make non-primitive template arguments independent of the current namespace.""" + type_name = type_name.strip() + pointer_suffix = "" + while type_name.endswith("*"): + pointer_suffix += "*" + type_name = type_name[:-1].strip() + + base_type, template_args = self._split_template_type(type_name) + if not template_args: + return type_name + pointer_suffix + + qualified_args = [] + for argument in template_args: + argument = self._qualify_template_arguments(argument, template_parameters) + argument_base = argument + while argument_base.endswith("*"): + argument_base = argument_base[:-1].strip() + if ( + argument_base not in template_parameters + and not self.is_base_type(argument_base) + and not argument_base.startswith("::") + ): + argument = f"::{argument}" + qualified_args.append(argument) + return f"{base_type}<{', '.join(qualified_args)}>{pointer_suffix}" + + def _get_struct_size( + self, + type_name: str, + struct_lookup: dict[str, int], + ) -> int | None: + normalized_type = self._normalize_type(type_name) + if normalized_type in struct_lookup: + return struct_lookup[normalized_type] + + return struct_lookup.get(self.get_short_name(normalized_type)) + + def _get_field_type( + self, + field: DefinedStructField | DefinedStructFixedField, + template_struct_lookup: dict[str, DefinedStruct], + ) -> str: + field_type = field.type.strip() + pointer_suffix = "" + while field_type.endswith("*"): + pointer_suffix += "*" + field_type = field_type[:-1].strip() + + if ( + field.template_args + and not self._split_template_type(field_type)[1] + and self._normalize_type(field_type) in template_struct_lookup + ): + template_args = [f"::{template_arg}" if not self.is_base_type(template_arg) else template_arg for template_arg in field.template_args] + field_type += f"<{', '.join(template_args)}>" + return field_type + pointer_suffix + + def _substitute_template_types( + self, type_name: str, substitutions: dict[str, str] + ) -> str: + type_name = type_name.strip() + pointer_suffix = "" + while type_name.endswith("*"): + pointer_suffix += "*" + type_name = type_name[:-1].strip() + + base_type, template_args = self._split_template_type(type_name) + if template_args: + substituted = ( + f"{base_type}<" + f"{', '.join(self._substitute_template_types(arg, substitutions) for arg in template_args)}>" + ) + else: + substituted = substitutions.get(base_type, base_type) + return substituted + pointer_suffix + + def get_size_from_type( + self, + type_name: str, + struct_lookup: dict[str, int], + enum_lookup: dict[str, int], + template_struct_lookup: dict[str, DefinedStruct] | None = None, + template_parameters: tuple[str, ...] = (), + struct_definition_lookup: dict[str, DefinedStruct] | None = None, + ) -> int: + """Resolve a field's size using schema types and the target x64 ABI.""" + type_name = type_name.strip() + if type_name.endswith("*"): + return 8 + if type_name in template_parameters: + return 0 + + normalized_type = self._normalize_type(type_name) + # the short name lookup of the structs can match an unrelated struct + if normalized_type in enum_lookup: + return enum_lookup[normalized_type] + struct_size = self._get_struct_size(normalized_type, struct_lookup) + if struct_size is not None: + return struct_size + + primitive_size = self.get_size_from_string(type_name) + if primitive_size: + return primitive_size + if normalized_type.removeprefix("::") in self._external_layouts: + return self._external_layouts[normalized_type.removeprefix("::")][0] + + base_type, template_args = self._split_template_type(type_name) + if not template_args: + struct_definition_lookup = struct_definition_lookup or {} + struct_definition = struct_definition_lookup.get(normalized_type) + if struct_definition is not None and not struct_definition.template_types: + return self._get_template_struct_size( + struct_definition, + [], + struct_lookup, + enum_lookup, + template_struct_lookup or {}, + struct_definition_lookup, + ) + return 0 + + stl_type_sizes = { + "std::vector": 24, + "std::set": 16, + "std::map": 16, + "std::unordered_map": 64, + "std::string": 32, + "std::wstring": 32, + "std::basic_string": 32, + "std::list": 16, + "std::deque": 40, + } + if base_type in stl_type_sizes: + return stl_type_sizes[base_type] + + if base_type == "std::pair": + if len(template_args) != 2: + raise ValueError(f"std::pair requires two type arguments: {type_name}") + first_size = self.get_size_from_type( + template_args[0], + struct_lookup, + enum_lookup, + template_struct_lookup, + struct_definition_lookup=struct_definition_lookup, + ) + second_size = self.get_size_from_type( + template_args[1], + struct_lookup, + enum_lookup, + template_struct_lookup, + struct_definition_lookup=struct_definition_lookup, + ) + first_alignment = self._get_type_alignment( + template_args[0], + struct_lookup, + enum_lookup, + template_struct_lookup, + struct_definition_lookup, + ) + second_alignment = self._get_type_alignment( + template_args[1], + struct_lookup, + enum_lookup, + template_struct_lookup, + struct_definition_lookup, + ) + alignment = max(first_alignment, second_alignment) + second_offset = self._align_size(first_size, second_alignment) + return self._align_size(second_offset + second_size, alignment) + + template_struct_lookup = template_struct_lookup or {} + template_struct = template_struct_lookup.get(self._normalize_type(base_type)) + if template_struct is not None: + return self._get_template_struct_size( + template_struct, + template_args, + struct_lookup, + enum_lookup, + template_struct_lookup, + struct_definition_lookup, + ) + + raise ValueError(f"Cannot determine size for template type: {type_name}") + + def _get_type_alignment( + self, + type_name: str, + struct_lookup: dict[str, int], + enum_lookup: dict[str, int], + template_struct_lookup: dict[str, DefinedStruct] | None = None, + struct_definition_lookup: dict[str, DefinedStruct] | None = None, + ) -> int: + type_name = type_name.strip() + if type_name.endswith("*"): + return 8 + + normalized_type = self._normalize_type(type_name) + base_type, template_args = self._split_template_type(normalized_type.removeprefix("::")) + if template_args: + struct_definition = (template_struct_lookup or {}).get(base_type) + else: + struct_definition = (struct_definition_lookup or {}).get(base_type) + if base_type in self._external_layouts and not template_args: + return self._external_layouts[base_type][1] + if struct_definition is not None and len(template_args) == len(struct_definition.template_types): + alignment, packed = self._get_struct_layout( + struct_definition, + template_args, + struct_lookup, + enum_lookup, + template_struct_lookup, + struct_definition_lookup, + ) + return 1 if packed else alignment + + struct_size = self._get_struct_size(normalized_type, struct_lookup) + if normalized_type in enum_lookup: + size = enum_lookup[normalized_type] + elif struct_size is not None: + size = struct_size + else: + size = self.get_size_from_string(type_name) + if size == 0: + base_type, template_args = self._split_template_type(type_name) + if base_type in ("std::vector", "std::set", "std::map", "std::unordered_map", "std::string", "std::wstring", "std::basic_string", "std::list", "std::deque", "std::linkedlist"): + return 8 + if base_type == "std::pair": + if len(template_args) != 2: + raise ValueError(f"std::pair requires two type arguments: {type_name}") + return max( + self._get_type_alignment( + arg, + struct_lookup, + enum_lookup, + template_struct_lookup, + struct_definition_lookup, + ) + for arg in template_args + ) + size = self.get_size_from_type( + type_name, + struct_lookup, + enum_lookup, + template_struct_lookup, + struct_definition_lookup=struct_definition_lookup, + ) + return min(size & -size, 8) if size else 1 + + def _get_union_members( + self, field: DefinedStructField | DefinedStructFixedField + ) -> list[DefinedStructField]: + """The members of a union field as fields at its offset, empty when the field is not a union.""" + if isinstance(field, DefinedStructFixedField) or field.type == "__fastcall": + return [] + return [ + DefinedStructField(union_value.name, union_value.type, field.offset, False, [], []) + for union_value in field.union_values + ] + + def _get_template_parameters(self, struct: DefinedStruct) -> tuple[str, ...]: + """The template parameters of the struct and of the templates it is declared in.""" + return ( + *self._inherited_template_types.get(self._normalize_type(struct.type), ()), + *struct.template_types, + ) + + def _is_template_value_field(self, field: DefinedStructField, template_parameters: tuple[str, ...]) -> bool: + """A template parameter held by value, its size depends on the arguments.""" + return not field.type.strip().endswith("*") and field.type.strip() in template_parameters + + def _has_template_value_field(self, struct: DefinedStruct) -> bool: + """The size of the struct then depends on the arguments.""" + template_parameters = self._get_template_parameters(struct) + return any(self._is_template_value_field(field, template_parameters) for field in struct.fields) + + def _get_pad_alignment(self, prev_size: int, end: int) -> int: + """The alignment of the filler fields get_struct_pad writes from prev_size up to end.""" + alignment = 1 + while end > prev_size: + _, size = self.get_struct_pad(end - prev_size, prev_size) + if self.full_padding: + alignment = max(alignment, size) + prev_size += size + return alignment + + def _get_struct_layout( + self, + struct: DefinedStruct, + template_args: list[str], + struct_lookup: dict[str, int], + enum_lookup: dict[str, int], + template_struct_lookup: dict[str, DefinedStruct] | None = None, + struct_definition_lookup: dict[str, DefinedStruct] | None = None, + ) -> tuple[int, bool]: + """The natural alignment of the struct, and if it has to be packed to keep its field offsets and size.""" + cache_key = (self._normalize_type(struct.type), tuple(template_args)) + if cache_key in self._layout_cache: + return self._layout_cache[cache_key] + + substitutions = dict(zip(struct.template_types, template_args)) + has_base_at_zero = any( + field.base and field.offset == 0 for field in struct.fields + ) + alignment = 8 if struct.virtual_functions and not has_base_at_zero else 1 + packed = False + template_parameters = self._get_template_parameters(struct) + track_padding = not template_parameters and not struct.union + prev_size = 0 + base_types_used = set() + for field in struct.fields: + if field.base and field.offset == prev_size: + base_types_used.add(field.type) + prev_size += self.get_size_from_type( + self._get_field_type(field, template_struct_lookup or {}), + struct_lookup, + enum_lookup, + template_struct_lookup, + template_parameters, + struct_definition_lookup, + ) + if struct.virtual_functions and prev_size == 0: + prev_size = 8 + for field in struct.fields: + if field.type == "__fastcall": + field_alignment = field_size = 8 + else: + # a union is as large and as aligned as its largest and most aligned member + field_types = [ + self._substitute_template_types( + self._get_field_type(member, template_struct_lookup or {}), + substitutions, + ) + for member in self._get_union_members(field) or [field] + ] + field_alignment = max( + self._get_type_alignment( + field_type, + struct_lookup, + enum_lookup, + template_struct_lookup, + struct_definition_lookup, + ) + for field_type in field_types + ) + field_size = 0 + if track_padding and field.type not in base_types_used: + field_size = max( + self.get_size_from_type( + field_type, + struct_lookup, + enum_lookup, + template_struct_lookup, + template_parameters, + struct_definition_lookup, + ) + for field_type in field_types + ) + if isinstance(field, DefinedStructFixedField): + field_size *= field.size + alignment = max(alignment, field_alignment) + if field.offset % field_alignment != 0: + packed = True + if track_padding and field.type not in base_types_used: + if field.offset > prev_size: + alignment = max(alignment, self._get_pad_alignment(prev_size, field.offset)) + prev_size = field.offset + prev_size += field_size + if track_padding and struct.size: + alignment = max(alignment, self._get_pad_alignment(prev_size, struct.size)) + if struct.size and struct.size % alignment != 0: + packed = True + if template_parameters: + packed = False + self._layout_cache[cache_key] = (alignment, packed) + return alignment, packed + + @staticmethod + def _align_size(size: int, alignment: int) -> int: + return (size + alignment - 1) // alignment * alignment + + def _get_template_struct_size( + self, + struct: DefinedStruct, + template_args: list[str], + struct_lookup: dict[str, int], + enum_lookup: dict[str, int], + template_struct_lookup: dict[str, DefinedStruct], + struct_definition_lookup: dict[str, DefinedStruct] | None = None, + ) -> int: + if len(template_args) != len(struct.template_types): + raise ValueError( + f"Template type {struct.type} expects {len(struct.template_types)} " + f"arguments, got {len(template_args)}" + ) + if struct.size is not None and not self._has_template_value_field(struct): + return struct.size + + substitutions = dict(zip(struct.template_types, template_args)) + has_base_at_zero = any( + field.base and field.offset == 0 for field in struct.fields + ) + has_implicit_vtable = bool(struct.virtual_functions) and not has_base_at_zero + extent = 8 if has_implicit_vtable else 1 + alignment = 8 if has_implicit_vtable else 1 + for field in struct.fields: + if field.type == "__fastcall": + field_size = 8 + field_alignment = 8 + else: + field_size = 0 + field_alignment = 1 + for member in self._get_union_members(field) or [field]: + field_type = self._get_field_type(member, template_struct_lookup) + field_type = self._substitute_template_types(field_type, substitutions) + member_size = self.get_size_from_type( + field_type, + struct_lookup, + enum_lookup, + template_struct_lookup, + struct_definition_lookup=struct_definition_lookup, + ) + if member_size == 0: + raise ValueError( + f"Cannot determine size of {field_type} in template type {struct.type}" + ) + field_size = max(field_size, member_size) + field_alignment = max( + field_alignment, + self._get_type_alignment( + field_type, + struct_lookup, + enum_lookup, + template_struct_lookup, + struct_definition_lookup, + ), + ) + if isinstance(field, DefinedStructFixedField): + field_size *= field.size + + alignment = max(alignment, field_alignment) + field_end = field.offset + field_size + if struct.union: + extent = max(extent, field_size) + else: + extent = max(extent, field_end) + return self._align_size(extent, alignment) + + def get_string_define_and_size( + self, + field: DefinedStructField | DefinedStructFixedField, + namespace: str, + struct_lookup: dict[str, int], + enum_lookup: dict[str, int], + template_struct_lookup: dict[str, DefinedStruct] | None = None, + template_parameters: list[str] = [], + struct_definition_lookup: dict[str, DefinedStruct] | None = None, + field_name_override: str | None = None, + opaque_pointer_types: set[str] | None = None, + ) -> tuple[str, int]: + template_struct_lookup = template_struct_lookup or {} + field_type = self._get_field_type(field, template_struct_lookup) + opaque_pointer_types = opaque_pointer_types or set() + pointer_count = 0 + pointer_base = field_type + while pointer_base.endswith("*"): + pointer_base = pointer_base[:-1].strip() + pointer_count += 1 + if pointer_count and self._normalize_type(pointer_base) in opaque_pointer_types: + field_type = "void" + "*" * pointer_count + field_name = field_name_override or field.name + field_size = self.get_size_from_type( + field_type, + struct_lookup, + enum_lookup, + template_struct_lookup, + template_parameters, + struct_definition_lookup, + ) + display_type = self._qualify_template_arguments( + field_type, tuple(template_parameters) + ) + if display_type.startswith(namespace): + display_type = display_type.removeprefix(f"{namespace}::") + elif not self.is_base_type(display_type) and not display_type.rstrip('*') in template_parameters: + display_type = f"::{display_type}" + return (f"{display_type} {field_name};", field_size) + + def get_short_name(self, full_type: str) -> str: + depth = 0 + last_namespace_separator = -1 + index = 0 + while index < len(full_type) - 1: + if full_type[index] == "<": + depth += 1 + elif full_type[index] == ">": + depth -= 1 + elif full_type[index : index + 2] == "::" and depth == 0: + last_namespace_separator = index + 1 + index += 1 + index += 1 + return full_type[last_namespace_separator + 1 :] + + def build_vfunc_signature( + self, vfunc: DefinedStructVFunc, name_override: str | None = None + ) -> str: + function_name = name_override or vfunc.name + if not (vfunc.return_type and vfunc.parameters): + return f"void* {function_name};" + + return_type = vfunc.return_type if vfunc.return_type else "void" + + if not self.is_base_type(return_type): + return_type = f"::{return_type}" + + param_list = "" + if vfunc.parameters: + params = [] + for param in vfunc.parameters: + if param.type == "__fastcall": + # function pointer parameters are exported without their signature + params.append(f"void* {param.name}") + elif not self.is_base_type(param.type): + params.append(f"::{param.type} {param.name}") + else: + params.append(f"{param.type} {param.name}") + + param_list = ", ".join(params) + + return f"{return_type} (__fastcall *{function_name})({param_list});" + + def build_export_string( + self, + export: DefinedStructExport, + full_declspec_align: int = 0, + layout_parser: Callable[[str], dict[str, tuple[int, int]]] | None = None, + layout_header: str = "", + ): + """layout_parser measures the types the schema can not size, see get_layout_probe, layout_header is what it needs declared.""" + output = IndentWriter() + self._layout_cache = {} + self._external_layouts = {} + self._inherited_template_types = {} + + all_structs: list[DefinedStruct] = [] + inline_structs: list[DefinedStruct] = [] + inline_enums: list[DefinedStructEnum] = [] + inline_type_owners: dict[str, str] = {} + + def collect_inline_types(struct: DefinedStruct): + for inline_enum in struct.inline_enums: + inline_enums.append(inline_enum) + for inline_struct in struct.inline_structs: + inline_structs.append(inline_struct) + all_structs.append(inline_struct) + collect_inline_types(inline_struct) + + for struct in export.structs: + all_structs.append(struct) + collect_inline_types(struct) + + def collect_inherited_template_types(struct: DefinedStruct, template_parameters: tuple[str, ...]): + template_parameters = (*template_parameters, *struct.template_types) + for inline_struct in struct.inline_structs: + if template_parameters: + self._inherited_template_types[self._normalize_type(inline_struct.type)] = template_parameters + collect_inherited_template_types(inline_struct, template_parameters) + + for struct in export.structs: + collect_inherited_template_types(struct, ()) + + inline_struct_types = { + self._normalize_type(struct.type) for struct in inline_structs + } + + def collect_inline_type_owners(struct: DefinedStruct, owner_type: str): + for inline_struct in struct.inline_structs: + inline_type_owners.setdefault( + self._normalize_type(inline_struct.type), + self._normalize_type(owner_type), + ) + collect_inline_type_owners(inline_struct, owner_type) + + for struct in export.structs: + if self._normalize_type(struct.type) not in inline_struct_types: + collect_inline_type_owners(struct, struct.type) + + inline_enum_types = { + self._normalize_type(enum.type) for enum in inline_enums + } + + struct_lookup: dict[str, int] = {} + for struct in all_structs: + if struct.size is None: + continue + normalized_type = self._normalize_type(struct.type) + struct_lookup[normalized_type] = struct.size + struct_lookup.setdefault(self.get_short_name(normalized_type), struct.size) + enum_lookup: dict[str, int] = {} + for enum in [*export.enums, *inline_enums]: + enum_lookup[self._normalize_type(enum.type)] = self.get_size_from_string( + enum.underlying + ) + template_struct_lookup: dict[str, DefinedStruct] = {} + struct_definition_lookup: dict[str, DefinedStruct] = {} + for struct in all_structs: + normalized_type = self._normalize_type(struct.type) + struct_definition_lookup[normalized_type] = struct + if struct.template_types: + template_struct_lookup[normalized_type] = struct + + self._external_layouts = self.resolve_external_layouts( + self._collect_external_types( + [ + struct + for struct in export.structs + if self._normalize_type(struct.type) not in inline_struct_types + ], + struct_lookup, + enum_lookup, + template_struct_lookup, + struct_definition_lookup, + ), + layout_parser, + layout_header, + ) + + top_level_structs = [ + struct + for struct in export.structs + if self._normalize_type(struct.type) not in inline_struct_types + ] + + def iter_struct_tree(struct: DefinedStruct): + yield struct + for inline_struct in struct.inline_structs: + yield from iter_struct_tree(inline_struct) + + # CExporter writes the structs in dependency order + ordered_structs = top_level_structs + + ordered_type_indices = { + self._normalize_type(struct.type): index + for index, struct in enumerate(ordered_structs) + } + opaque_pointer_types_by_struct: dict[str, set[str]] = {} + for struct in ordered_structs: + struct_type = self._normalize_type(struct.type) + opaque_pointer_types = set() + for nested_struct in iter_struct_tree(struct): + for field in [ + member + for nested_field in nested_struct.fields + for member in [nested_field, *self._get_union_members(nested_field)] + ]: + field_type = self._normalize_type(field.type) + pointer_base = field_type + has_pointer = pointer_base.endswith("*") + while pointer_base.endswith("*"): + pointer_base = pointer_base[:-1].strip() + if not has_pointer: + continue + owner_type = inline_type_owners.get(pointer_base) + if ( + owner_type is not None + and owner_type != struct_type + and owner_type in ordered_type_indices + and ordered_type_indices[owner_type] + > ordered_type_indices[struct_type] + ): + opaque_pointer_types.add(pointer_base) + if opaque_pointer_types: + opaque_pointer_types_by_struct[struct_type] = opaque_pointer_types + + # Windows headers define macros such as DrawState (-> DrawStateA) that + # would silently rename the types and members sharing their name + identifiers: set[str] = set() + for enum in [*export.enums, *inline_enums]: + identifiers.add(self.get_short_name(enum.name)) + for struct in all_structs: + identifiers.add(self.get_short_name(struct.type)) + identifiers.update(field.name for field in struct.fields) + identifiers.update( + union_value.name + for field in struct.fields + for union_value in field.union_values + ) + identifiers.update( + virtual_function.name + for virtual_function in struct.virtual_functions or [] + ) + for identifier in sorted(identifiers): + if identifier.isidentifier(): + output.append(f"#undef {identifier}") + + def append_chunk(chunk: IndentWriter | str): + chunk_text = str(chunk) + for line in chunk_text.split("\n"): + if not line.strip(): + continue + leading = len(line) - len(line.lstrip(" ")) + chunk_indent = leading // 2 + prev_indent = output.indent_level + output.indent_level = prev_indent + chunk_indent + output.append(line.lstrip(" ")) + output.indent_level = prev_indent + + def append_ordered_chunks(chunks: list[tuple[str, IndentWriter]]): + open_namespaces: list[str] = [] + for namespace, chunk in chunks: + target_namespaces = namespace.split("::") if namespace else [] + shared_count = 0 + while ( + shared_count < len(open_namespaces) + and shared_count < len(target_namespaces) + and open_namespaces[shared_count] == target_namespaces[shared_count] + ): + shared_count += 1 + + while len(open_namespaces) > shared_count: + output.unindent() + output.append("}") + open_namespaces.pop() + + for namespace_name in target_namespaces[shared_count:]: + formatted_name = namespace_name.replace(".", "::") + output.append(f"namespace {formatted_name} {{") + output.indent() + open_namespaces.append(namespace_name) + + append_chunk(chunk) + + while open_namespaces: + output.unindent() + output.append("}") + open_namespaces.pop() + + enum_chunks = [] + forward_chunks: list[tuple[str, IndentWriter]] = [] + + for enum in export.enums: + if self._normalize_type(enum.type) in inline_enum_types: + continue + enum_forward = IndentWriter() + enum_forward += f"enum {enum.name} : {enum.underlying};" + forward_chunks.append((enum.namespace, enum_forward)) + enum_chunks.append((enum.namespace, self.build_enum_string(enum))) + + definition_chunks: list[tuple[str, IndentWriter]] = [] + inline_enum_definition_chunks: list[tuple[str, IndentWriter]] = [] + vtable_names: list[str] = [] + seen_vtable_names: set[str] = set() + + def collect_inline_enum_definitions(struct: DefinedStruct): + namespace_prefix = struct.namespace.replace(".", "::") + owner_type = struct.type + if namespace_prefix and owner_type.startswith(f"{namespace_prefix}::"): + owner_type = owner_type[len(namespace_prefix) + 2 :] + if struct.template_types: + owner_type += f"<{', '.join(struct.template_types)}>" + + for inline_enum in struct.inline_enums: + inline_enum_definition = self.build_enum_string( + inline_enum, owner_type, tuple(struct.template_types) + ) + inline_enum_definition_chunks.append( + (inline_enum.namespace, inline_enum_definition) + ) + for inline_struct in struct.inline_structs: + collect_inline_enum_definitions(inline_struct) + + def collect_vtable_names(struct: DefinedStruct): + # a template vtable only exists in IDA as its instantiations + if struct.virtual_functions and not struct.template_types: + vtable_name = f"{struct.type}_vtbl" if self.ida_vtables else f"{struct.type}::VTable" + if vtable_name not in seen_vtable_names: + seen_vtable_names.add(vtable_name) + vtable_names.append(vtable_name) + for inline_struct in struct.inline_structs: + collect_vtable_names(inline_struct) + + for struct in ordered_structs: + forward, define = self.build_struct_string( + struct, + struct_lookup, + enum_lookup, + template_struct_lookup, + full_declspec_align, + struct_definition_lookup, + opaque_pointer_types_by_struct.get( + self._normalize_type(struct.type), set() + ), + ) + collect_vtable_names(struct) + collect_inline_enum_definitions(struct) + forward_chunks.append((struct.namespace, forward)) + definition_chunks.append((struct.namespace, define)) + + def forward_chunk_key(key: tuple[str, IndentWriter]) -> str: + (namespace, _) = key + return namespace + + forward_chunks = sorted(forward_chunks, key=forward_chunk_key) + + append_ordered_chunks(forward_chunks) + append_ordered_chunks(enum_chunks) + append_ordered_chunks(definition_chunks) + append_ordered_chunks(inline_enum_definition_chunks) + + for alias, type_name in self._collect_template_aliases(export.structs).items(): + output.append(f"typedef ::{type_name} {alias};") + + return (str(output), vtable_names) + + def _collect_external_types( + self, + structs: list[DefinedStruct], + struct_lookup: dict[str, int], + enum_lookup: dict[str, int], + template_struct_lookup: dict[str, DefinedStruct], + struct_definition_lookup: dict[str, DefinedStruct], + ) -> list[str]: + """The types held by value that are not in the schema, ex. GUID or HANDLE from the windows headers.""" + type_names: set[str] = set() + + def collect(type_name: str, template_parameters: tuple[str, ...]): + type_name = type_name.strip() + if type_name.endswith("*") or type_name == "__fastcall": + return + _, template_args = self._split_template_type(type_name) + if template_args: + for template_arg in template_args: + collect(template_arg, template_parameters) + return + type_name = self._normalize_type(type_name).removeprefix("::") + if type_name in template_parameters or self.is_base_type(type_name): + return + try: + size = self.get_size_from_type( + type_name, + struct_lookup, + enum_lookup, + template_struct_lookup, + template_parameters, + struct_definition_lookup, + ) + except ValueError: + return + if size == 0: + type_names.add(type_name) + + def collect_struct(struct: DefinedStruct, template_parameters: tuple[str, ...]): + template_parameters = (*template_parameters, *struct.template_types) + for field in struct.fields: + for member in self._get_union_members(field) or [field]: + collect(self._get_field_type(member, template_struct_lookup), template_parameters) + for inline_struct in struct.inline_structs: + collect_struct(inline_struct, template_parameters) + + for struct in structs: + collect_struct(struct, ()) + return sorted(type_names) + + def get_layout_probe(self, type_names: list[str], header: str = "") -> str: + """A probe struct per type, a layout parser returns {probe name: (size of value, offset of value)} for it.""" + probes = "\n".join( + f"struct {layout_probe_prefix}{index} {{ char pad; ::{type_name} value; }};" + for index, type_name in enumerate(type_names) + ) + return f"{self.get_predef_header()}\n{header}\n{probes}\n" + + def resolve_external_layouts( + self, + type_names: list[str], + layout_parser: Callable[[str], dict[str, tuple[int, int]]] | None, + layout_header: str = "", + ) -> dict[str, tuple[int, int]]: + """Size and alignment of the types, measured by the parser of the disassembler.""" + if not type_names: + return {} + if layout_parser is None: + print(f"Warning: no layout parser, {len(type_names)} type(s) can not be sized: {', '.join(type_names)}") + return {} + + probe_layouts = layout_parser(self.get_layout_probe(type_names, layout_header)) + layouts: dict[str, tuple[int, int]] = {} + missing: list[str] = [] + for index, type_name in enumerate(type_names): + probe_layout = probe_layouts.get(f"{layout_probe_prefix}{index}") + if probe_layout is None or probe_layout[0] <= 0 or probe_layout[1] <= 0: + missing.append(type_name) + else: + layouts[type_name] = probe_layout + if missing: + print(f"Warning: the parser could not size {len(missing)} type(s): {', '.join(missing)}") + return layouts + + def _collect_template_aliases(self, structs: list[DefinedStruct]) -> dict[str, str]: + """The typedef name of every templated type in a function signature.""" + aliases: dict[str, str] = {} + + def collect(type_name: str | None): + type_name = (type_name or "").strip().rstrip("*").strip() + if "<" in type_name: + aliases.setdefault(get_template_alias(type_name), type_name) + + def collect_struct(struct: DefinedStruct): + # signatures of a template struct use its template parameters + if struct.template_types: + return + functions = [ + *(struct.virtual_functions or []), + *(struct.member_functions or []), + *(struct.static_member_functions or []), + ] + for function in functions: + collect(function.return_type) + for parameter in function.parameters or []: + collect(parameter.type) + for static_member in struct.static_members or []: + collect(static_member.return_type) + for inline_struct in struct.inline_structs: + collect_struct(inline_struct) + + for struct in structs: + collect_struct(struct) + return aliases + + def build_enum_string( + self, + enum: DefinedStructEnum, + enum_scope: str | None = None, + template_types: tuple[str, ...] = (), + ): + output = IndentWriter() + + enum_name = enum.name + if template_types: + output += f"template" + enum_type = f"{enum_scope}::{enum_name}" if enum_scope != None else enum_name + output += f"enum {enum_type} : {enum.underlying} {{" + output.indent() + + for value_name, value in enum.values.items(): + output += f"{enum_name}_{value_name} = {value}," + + output.unindent() + output += "};" + + return output + + def get_struct_pad(self, fill_size: int, prev_size: int): + if self.full_padding: + type, size = self.get_type_and_type_size_from_gap(prev_size) + if size > fill_size: + type, size = self.get_type_and_type_size_from_gap(fill_size, prev_size) + + return (f"{type} field_{prev_size:X};", size) + else: + return (f"char field_{prev_size:X}[{fill_size}];", fill_size) + + def build_struct_string( + self, + struct: DefinedStruct, + struct_lookup: dict[str, int], + enum_lookup: dict[str, int], + template_struct_lookup: dict[str, DefinedStruct] | None = None, + declspec_align: int = 0, + struct_definition_lookup: dict[str, DefinedStruct] | None = None, + opaque_pointer_types: set[str] | None = None, + ) -> tuple[IndentWriter, IndentWriter]: + """The forward declarations and the definition, the vtable inside as VTable or beside as Name_vtbl for IDA.""" + struct_string_forward: IndentWriter = IndentWriter() + struct_string_define: IndentWriter = IndentWriter() + template_struct_lookup = template_struct_lookup or {} + opaque_pointer_types = opaque_pointer_types or set() + template_declaration = "" + if struct.template_types: + template_declaration = ( + f"template" + ) + + short_name = self.get_short_name(struct.type) + template_parameters = self._get_template_parameters(struct) + + # pragma pack can not lower a declspec align, even align(1) + if declspec_align > 1: + declspec = f"__declspec(align({declspec_align})) " + else: + declspec = "" + + if template_declaration: + struct_string_forward += template_declaration + record_keyword = "union" if struct.union else "struct" + struct_string_forward += f"{record_keyword} {struct.name};" + + if template_declaration: + struct_string_define += template_declaration + struct_define_line = f"{record_keyword} {declspec}{short_name}" + struct_extend_lines = [] + + base_fields = [field for field in struct.fields if field.base] + base_fields_used = [] + prev_size = 0 + for base_field in base_fields: + if base_field.offset == prev_size: + base_fields_used.append(base_field.type) + field_type = self._get_field_type(base_field, template_struct_lookup) + struct_extend_lines.append(f"::{field_type}" if not self.is_base_type(field_type) else field_type) + prev_size += self.get_size_from_type( + field_type, + struct_lookup, + enum_lookup, + template_struct_lookup, + template_parameters, + struct_definition_lookup, + ) + + struct_extend_line = "" + if len(struct_extend_lines) > 0: + struct_extend_line = " : " + ", ".join(struct_extend_lines) + + struct_string_define += f"{struct_define_line}{struct_extend_line} {{" + struct_string_define.indent() + + for inline_enum in struct.inline_enums: + struct_string_define += f"enum {self.get_short_name(inline_enum.name)} : {inline_enum.underlying};" + + def append_chunk(chunk: IndentWriter | str): + for line in str(chunk).split("\n"): + if not line.strip(): + continue + leading = len(line) - len(line.lstrip(" ")) + previous_indent = struct_string_define.indent_level + struct_string_define.indent_level = previous_indent + leading // 2 + struct_string_define.append(line.lstrip(" ")) + struct_string_define.indent_level = previous_indent + + inline_struct_chunks = [ + self.build_struct_string( + inline_struct, + struct_lookup, + enum_lookup, + template_struct_lookup, + declspec_align, + struct_definition_lookup, + opaque_pointer_types, + ) + for inline_struct in struct.inline_structs + ] + + for inline_forward, _ in inline_struct_chunks: + append_chunk(inline_forward) + for _, inline_define in inline_struct_chunks: + append_chunk(inline_define) + + ida_vtable = bool(struct.virtual_functions) and self.ida_vtables + if ida_vtable: + if template_declaration: + struct_string_forward += template_declaration + struct_string_forward += f"struct {short_name}_vtbl;" + if prev_size == 0: + template_suffix = f"<{', '.join(struct.template_types)}>" if struct.template_types else "" + struct_string_define += f"{short_name}_vtbl{template_suffix}* __vftable;" + prev_size = 8 + elif struct.virtual_functions: + struct_string_define += "struct VTable {" + struct_string_define.indent() + for vfunc_line in self.build_vtable_lines(struct): + struct_string_define += vfunc_line + struct_string_define.unindent() + # with a base at offset 0 the pointer comes from the base + if prev_size == 0: + struct_string_define += "} *__vtable;" + prev_size = 8 + else: + struct_string_define += "};" + + field_name_counts: dict[str, int] = {} + field_name_last_indices: dict[str, int] = {} + for field_index, field in enumerate(struct.fields): + field_name_counts[field.name] = field_name_counts.get(field.name, 0) + 1 + field_name_last_indices[field.name] = field_index + reserved_field_names = set(field_name_counts) + emitted_field_names = set() + field_names = [] + for field_index, field in enumerate(struct.fields): + field_name = field.name + if ( + field_name_counts[field.name] > 1 + and field_name_last_indices[field.name] != field_index + ): + field_name = f"{field.name}_{field.offset:X}" + while field_name in reserved_field_names or field_name in emitted_field_names: + field_name += "_" + field_names.append(field_name) + emitted_field_names.add(field_name) + + offset = 0 + union_size = 0 + size_known = True + packed = not template_parameters and self._get_struct_layout( + struct, + [], + struct_lookup, + enum_lookup, + template_struct_lookup, + struct_definition_lookup, + )[1] + + for field_index, field in enumerate(struct.fields): + if field.type in base_fields_used: + continue + + if struct.union: + union_size = max(union_size, prev_size) + prev_size = 0 + + offset = field.offset + + if (not struct.is_sequential or packed) and size_known: + while offset > prev_size: + fill_size = offset - prev_size + line, size = self.get_struct_pad(fill_size, prev_size) + struct_string_define += line + prev_size += size + + field_name = field_names[field_index] + field_type = field.type + if isinstance(field, DefinedStructFixedField): + field_define, field_define_size = self.get_string_define_and_size( + field, + struct.namespace, + struct_lookup, + enum_lookup, + template_struct_lookup, + list(template_parameters), + struct_definition_lookup, + field_name, + opaque_pointer_types, + ) + struct_string_define += f"{field_define[:-1]}[{field.size}];" + prev_size += field_define_size * field.size + elif field_type == "__fastcall": + fastcall_field_define = "" + fastcall_field_define += ( + f"{field.return_type} ({field_type} *{field_name})(" + ) + for param in field.parameters: + fastcall_field_define += param.type + " " + param.name + ", " + if len(field.parameters) > 0: + fastcall_field_define = fastcall_field_define[:-2] + struct_string_define += fastcall_field_define + ");" + prev_size += 8 + elif union_members := self._get_union_members(field): + struct_string_define += "union {" + struct_string_define.indent() + union_define_size = 0 + for member in union_members: + member_define, member_define_size = self.get_string_define_and_size( + member, + struct.namespace, + struct_lookup, + enum_lookup, + template_struct_lookup, + list(template_parameters), + struct_definition_lookup, + None, + opaque_pointer_types, + ) + struct_string_define += member_define + union_define_size = max(union_define_size, member_define_size) + if self._is_template_value_field(member, template_parameters): + size_known = False + struct_string_define.unindent() + struct_string_define += f"}} {field_name};" + prev_size += union_define_size + else: + field_define, field_define_size = self.get_string_define_and_size( + field, + struct.namespace, + struct_lookup, + enum_lookup, + template_struct_lookup, + list(template_parameters), + struct_definition_lookup, + field_name, + opaque_pointer_types, + ) + struct_string_define += field_define + prev_size += field_define_size + if self._is_template_value_field(field, template_parameters): + size_known = False + + if struct.union: + prev_size = max(union_size, prev_size) + if struct.size and struct.size > prev_size: + struct_string_define += f"uint8_t field_0[{struct.size}];" + prev_size = struct.size + + if struct.size is not None and struct.size != 0 and not self._has_template_value_field(struct): + while struct.size > prev_size: + fill_size = struct.size - prev_size + line, size = self.get_struct_pad(fill_size, prev_size) + struct_string_define += line + prev_size += size + + struct_string_define.unindent() + struct_string_define += "};" + + if ida_vtable: + if template_declaration: + struct_string_define += template_declaration + struct_string_define += f"struct {short_name}_vtbl {{" + struct_string_define.indent() + for vfunc_line in self.build_vtable_lines(struct): + struct_string_define += vfunc_line + struct_string_define.unindent() + struct_string_define += "};" + + if packed: + struct_string_define.lines.insert(0, IndentWriterValue(0, "#pragma pack(push, 1)")) + struct_string_define += "#pragma pack(pop)" + + return (struct_string_forward, struct_string_define) + + def build_vtable_lines(self, struct: DefinedStruct) -> list[str]: + """One line per vtable slot.""" + lines: list[str] = [] + previous_offset = 0 + vfunc_name_counts: dict[str, int] = {} + vfunc_preferred_indices: dict[str, int] = {} + for vfunc_index, vfunc in enumerate(struct.virtual_functions): + vfunc_name_counts[vfunc.name] = vfunc_name_counts.get(vfunc.name, 0) + 1 + preferred_index = vfunc_preferred_indices.get(vfunc.name) + if preferred_index is None or ( + vfunc.return_type + and vfunc.parameters + and not ( + struct.virtual_functions[preferred_index].return_type + and struct.virtual_functions[preferred_index].parameters + ) + ): + vfunc_preferred_indices[vfunc.name] = vfunc_index + + emitted_vfunc_names = set() + for vfunc_index, vfunc in enumerate(struct.virtual_functions): + while previous_offset < vfunc.offset: + lines.append(f"void* vf{int(previous_offset/8)};") + previous_offset += 8 + + vfunc_name = vfunc.name + if ( + vfunc_name_counts[vfunc.name] > 1 + and vfunc_preferred_indices[vfunc.name] != vfunc_index + ): + vfunc_name = f"vf{vfunc.offset // 8}" + while vfunc_name in emitted_vfunc_names: + vfunc_name += "_" + emitted_vfunc_names.add(vfunc_name) + lines.append(self.build_vfunc_signature(vfunc, vfunc_name)) + previous_offset = vfunc.offset + 8 + return lines + + def get_predef_header(self): + return """#define _HAS_ITERATOR_DEBUGGING 0 +#define _ITERATOR_DEBUG_LEVEL 0 +#define _AMD64_ +#undef __unaligned +#define __unaligned +#define __IZoneIdentifier2_FWD_DEFINED__ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +""" \ No newline at end of file diff --git a/scripts/shared/structs_schema.py b/scripts/shared/structs_schema.py new file mode 100644 index 0000000000..9ca9cd459f --- /dev/null +++ b/scripts/shared/structs_schema.py @@ -0,0 +1,293 @@ +from dataclasses import dataclass +from typing import TypeAlias, Union +from yaml import load + +from yaml_loader import Loader + +YamlStructValuesExport: TypeAlias = list[ + dict[str, Union[str, int, "YamlStructValuesExport"]] +] +YamlStructExport: TypeAlias = dict[str, YamlStructValuesExport] +YamlExport: TypeAlias = dict[str, list[YamlStructExport]] + + +@dataclass +class DefinedStructBase: + name: str + type: str + namespace: str + + +@dataclass +class DefinedStructEnum(DefinedStructBase): + underlying: str + flags: bool + values: dict[str, int] + + +@dataclass +class DefinedStructFuncParam: + name: str + type: str + + +@dataclass +class DefinedStructVFunc: + name: str + return_type: str | None + offset: int + parameters: list[DefinedStructFuncParam] | None + + +@dataclass +class DefinedStructMemFunc: + signature: str + return_type: str + parameters: list[DefinedStructFuncParam] + name: str + + +@dataclass +class DefinedStructField(DefinedStructFuncParam): + offset: int + base: bool + template_args: list[str] + union_values: list[DefinedStructFuncParam] + + +@dataclass +class DefinedStructFuncField(DefinedStructField): + return_type: str | None + parameters: list[DefinedStructFuncParam] | None + + +@dataclass +class DefinedStructStaticMember: + name: str + signature: str + relative_offsets: list[int] + return_type: str + is_pointer: bool + + +@dataclass +class DefinedStructFixedField(DefinedStructField): + size: int + is_string: bool + + +Fields: TypeAlias = list[DefinedStructField | DefinedStructFixedField] + + +@dataclass +class DefinedStruct(DefinedStructBase): + fields: Fields + size: int + vtable_size: int | None + virtual_functions: list[DefinedStructVFunc] + member_functions: list[DefinedStructMemFunc] + union: bool + static_member_functions: list[DefinedStructMemFunc] | None + static_members: list[DefinedStructStaticMember] | None + template_types: list[str] + inline_structs: list["DefinedStruct"] + inline_enums: list[DefinedStructEnum] + is_sequential: bool + + +@dataclass +class DefinedStructExport: + enums: list[DefinedStructEnum] + structs: list[DefinedStruct] + + +def get_template_alias(type_name: str) -> str: + """The typedef name of a templated type, IDA spells template names differently.""" + alias = "".join(char if char.isalnum() else "_" for char in type_name.replace(" ", "").replace("*", "_ptr")) + return f"__cs_{alias}" + + +def get_yaml(stream, loader=Loader) -> DefinedStructExport: + dic: YamlExport = load(stream, loader) + enums = [] + structs = [] + for enum in dic["enums"]: + enums.append( + DefinedStructEnum( + enum["name"], + enum["type"], + enum["namespace"], + enum["underlying"], + enum["flags"], + enum["values"], + ) + ) + for struct in dic["structs"]: + structs.append(parse_struct(struct)) + return DefinedStructExport(enums, structs) + +def parse_struct(struct: YamlStructExport) -> DefinedStruct: + fields = [] + virtual_functions = [] + member_functions = [] + static_member_functions = None + static_members = None + template_types = [] + inline_structs = [] + inline_enums = [] + for field in struct["fields"]: + base = field["base"] if "base" in field else False + template_args = [] + union_values = [] + if "template_args" in field: + for template_arg in field["template_args"]: + template_args.append(template_arg) + if "union_values" in field: + for union_value in field["union_values"]: + union_values.append( + DefinedStructFuncParam(union_value["name"], union_value["type"]) + ) + if "size" in field: + fields.append( + DefinedStructFixedField( + field["name"], + field["type"], + field["offset"], + base, + template_args, + union_values, + field["size"], + field["is_string"], + ) + ) + elif "return_type" in field: + parameters = [] + for param in field["parameters"]: + parameters.append( + DefinedStructFuncParam(param["name"], param["type"]) + ) + fields.append( + DefinedStructFuncField( + field["name"], + field["type"], + field["offset"], + base, + template_args, + union_values, + field["return_type"], + parameters, + ) + ) + else: + fields.append( + DefinedStructField( + field["name"], + field["type"], + field["offset"], + base, + template_args, + union_values, + ) + ) + if "virtual_functions" in struct: + for vfunc in struct["virtual_functions"]: + parameters = ( + [ + DefinedStructFuncParam(param["name"], param["type"]) + for param in vfunc["parameters"] + ] + if "parameters" in vfunc + else None + ) + virtual_functions.append( + DefinedStructVFunc( + vfunc["name"], + vfunc["return_type"] if "return_type" in vfunc else None, + vfunc["offset"], + parameters, + ) + ) + if "member_functions" in struct: + for memfunc in struct["member_functions"]: + parameters = [] + for param in memfunc["parameters"]: + parameters.append( + DefinedStructFuncParam(param["name"], param["type"]) + ) + member_functions.append( + DefinedStructMemFunc( + memfunc["signature"], + memfunc["return_type"], + parameters, + memfunc["name"], + ) + ) + if "static_member_functions" in struct: + static_member_functions = [] + for smemfunc in struct["static_member_functions"]: + parameters = [] + for param in smemfunc["parameters"]: + parameters.append( + DefinedStructFuncParam(param["name"], param["type"]) + ) + static_member_functions.append( + DefinedStructMemFunc( + smemfunc["signature"], + smemfunc["return_type"], + parameters, + smemfunc["name"], + ) + ) + if "static_members" in struct: + static_members = [] + for sm in struct["static_members"]: + static_members.append( + DefinedStructStaticMember( + sm["name"], + sm["signature"], + sm["relative_follow_offsets"], + sm["return_type"], + sm["is_pointer"] if "is_pointer" in sm else False, + ) + ) + if "template_types" in struct: + for type in struct["template_types"]: + template_types.append(type) + if "inline_structs" in struct: + for inline_struct in struct["inline_structs"]: + inline_structs.append(parse_struct(inline_struct)) + if "inline_enums" in struct: + for enum in struct["inline_enums"]: + inline_enums.append( + DefinedStructEnum( + enum["name"], + enum["type"], + enum["namespace"], + enum["underlying"], + enum["flags"], + enum["values"], + ) + ) + size = None + if "size" in struct: + size = struct["size"] + vtable_size = None + if "vtable_size" in struct: + vtable_size = struct["vtable_size"] + return DefinedStruct( + struct["name"], + struct["type"], + struct["namespace"], + fields, + size, + vtable_size, + virtual_functions, + member_functions, + struct["union"], + static_member_functions, + static_members, + template_types, + inline_structs, + inline_enums, + struct["is_sequential"] if "is_sequential" in struct else False + ) diff --git a/scripts/shared/yaml_loader.py b/scripts/shared/yaml_loader.py new file mode 100644 index 0000000000..1cf7bac66a --- /dev/null +++ b/scripts/shared/yaml_loader.py @@ -0,0 +1,16 @@ +from yaml import load, SafeLoader, SafeDumper + +try: + from yaml import CSafeLoader + + # When a host re-imports the yaml package in the same interpreter (e.g. a script host unloading modules between runs), + # the cached C extension keeps the old node classes and the resolver fails with "tag None", so check it still works + load("a: 1", CSafeLoader) + Loader = CSafeLoader +except Exception: + Loader = SafeLoader + +try: + from yaml import CSafeDumper as Dumper +except ImportError: + Dumper = SafeDumper