WARNING: THIS SITE IS A MIRROR OF GITHUB.COM / IT CANNOT LOGIN OR REGISTER ACCOUNTS / THE CONTENTS ARE PROVIDED AS-IS / THIS SITE ASSUMES NO RESPONSIBILITY FOR ANY DISPLAYED CONTENT OR LINKS / IF YOU FOUND SOMETHING MAY NOT GOOD FOR EVERYONE, CONTACT ADMIN AT ilovescratch@foxmail.com
Skip to content

Mostly switch object writer to UTF-8 #122160

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
MichalStrehovsky merged 5 commits into from
Dec 8, 2025
Merged

Mostly switch object writer to UTF-8 #122160

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
787fcfa
Object writer
MichalStrehovsky Dec 2, 2025
fd0c4f2
Fix crossgen2
MichalStrehovsky Dec 5, 2025
cd66282
Review feedback
MichalStrehovsky Dec 5, 2025
860936a
Usings
MichalStrehovsky Dec 5, 2025
e578822
Review feedback
MichalStrehovsky Dec 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/coreclr/tools/Common/Compiler/DependencyAnalysis/MethodReadOnlyDataNode.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ public override ObjectNodeSection GetSection(NodeFactory factory)

public void AppendMangledName(NameMangler nameMangler, Utf8StringBuilder sb)
{
sb.Append("__readonlydata_" + nameMangler.GetMangledMethodName(_owningMethod));
sb.Append("__readonlydata_"u8).Append(nameMangler.GetMangledMethodName(_owningMethod));
}
public int Offset => 0;
public override bool IsShareable => true;
Expand Down
2 changes: 1 addition & 1 deletion src/coreclr/tools/Common/Compiler/NameMangler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ public NameMangler(NodeMangler nodeMangler)
public NodeMangler NodeMangler { get; private set; }
#endif

public abstract string CompilationUnitPrefix { get; set; }
public abstract Utf8String CompilationUnitPrefix { get; set; }

public abstract Utf8String SanitizeName(Utf8String s);

Expand Down
31 changes: 28 additions & 3 deletions src/coreclr/tools/Common/Compiler/NativeAotNameMangler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,13 @@ public NativeAotNameMangler(NodeMangler nodeMangler) : base(nodeMangler)
}
#endif

private string _compilationUnitPrefix;
private Utf8String _compilationUnitPrefix;

public override string CompilationUnitPrefix
public override Utf8String CompilationUnitPrefix
{
get
{
Debug.Assert(_compilationUnitPrefix != null);
Debug.Assert(!_compilationUnitPrefix.IsNull);
return _compilationUnitPrefix;
}
set { _compilationUnitPrefix = SanitizeNameWithHash(value); }
Expand Down Expand Up @@ -170,6 +170,31 @@ private string SanitizeNameWithHash(string literal)
return mangledName;
}

private Utf8String SanitizeNameWithHash(Utf8String literal)
{
Utf8String mangledName = SanitizeName(literal);
Copy link
Contributor

@ANahr ANahr Dec 8, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That code seems to be dangerous.
The result is not really an Utf8-String but ASCII - and it needs to be because the next few lines would be incorrect if it would be UTF8 and would contain any multi-byte chars (you can't simply cut off arbitrary Utf8 at byte position 30).
So if anybody ever changes SanitizeName to actually be Utf8 this will create hard-to-spot errors.

Copy link
Member Author

@MichalStrehovsky MichalStrehovsky Dec 9, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So if anybody ever changes SanitizeName to actually be Utf8 this will create hard-to-spot errors.

It's not likely we'd ever allow SanitizeName to return characters outside the basic ASCII set.

Copy link
Contributor

@MichalPetryka MichalPetryka Dec 9, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would a Debug.Assert(Ascii.IsValid(mangledName)) make sense here?


if (mangledName.Length > 30)
mangledName = new Utf8String(mangledName.AsSpan().Slice(0, 30).ToArray());

if (!mangledName.AsSpan().SequenceEqual(literal.AsSpan()))
{
byte[] hash;
lock (this)
{
// Use SHA256 hash here to provide a high degree of uniqueness to symbol names without requiring them to be long
// This hash function provides an exceedingly high likelihood that no two strings will be given equal symbol names
// This is not considered used for security purpose; however collisions would be highly unfortunate as they will cause compilation
// failure.
hash = SHA256.HashData(literal.AsSpan());
}

mangledName += "_" + Convert.ToHexString(hash);
}

return mangledName;
}

/// <summary>
/// Dictionary given a mangled name for a given <see cref="TypeDesc"/>
/// </summary>
Expand Down
34 changes: 17 additions & 17 deletions src/coreclr/tools/Common/Compiler/ObjectWriter/CoffObjectWriter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
using System.Text;
using ILCompiler.DependencyAnalysis;
using ILCompiler.DependencyAnalysisFramework;
using Internal.Text;
using Internal.TypeSystem;
using static ILCompiler.DependencyAnalysis.RelocType;
using static ILCompiler.ObjectWriter.CoffObjectWriter.CoffRelocationType;
Expand Down Expand Up @@ -46,16 +47,16 @@ namespace ILCompiler.ObjectWriter
/// </remarks>
internal partial class CoffObjectWriter : ObjectWriter
{
protected sealed record SectionDefinition(CoffSectionHeader Header, Stream Stream, List<CoffRelocation> Relocations, string ComdatName, string SymbolName);
protected sealed record SectionDefinition(CoffSectionHeader Header, Stream Stream, List<CoffRelocation> Relocations, Utf8String ComdatName, Utf8String SymbolName);

protected readonly Machine _machine;
protected readonly List<SectionDefinition> _sections = new();

// Symbol table
private readonly List<CoffSymbolRecord> _symbols = new();
private readonly Dictionary<string, uint> _symbolNameToIndex = new(StringComparer.Ordinal);
private readonly Dictionary<Utf8String, uint> _symbolNameToIndex = new();
private readonly Dictionary<int, CoffSectionSymbol> _sectionNumberToComdatAuxRecord = new();
private readonly HashSet<string> _referencedMethods = new();
private readonly HashSet<Utf8String> _referencedMethods = new();

private static readonly ObjectNodeSection GfidsSection = new ObjectNodeSection(".gfids$y", SectionType.ReadOnly);
private static readonly ObjectNodeSection DebugTypesSection = new ObjectNodeSection(".debug$T", SectionType.ReadOnly);
Expand All @@ -76,7 +77,7 @@ public CoffObjectWriter(NodeFactory factory, ObjectWritingOptions options, Outpu
};
}

private protected override void CreateSection(ObjectNodeSection section, string comdatName, string symbolName, int sectionIndex, Stream sectionStream)
private protected override void CreateSection(ObjectNodeSection section, Utf8String comdatName, Utf8String symbolName, int sectionIndex, Stream sectionStream)
{
var sectionHeader = new CoffSectionHeader
{
Expand Down Expand Up @@ -108,7 +109,7 @@ private protected override void CreateSection(ObjectNodeSection section, string
SectionCharacteristics.MemDiscardable;
}

if (comdatName is not null)
if (!comdatName.IsNull)
{
sectionHeader.SectionCharacteristics |= SectionCharacteristics.LinkerComdat;

Expand Down Expand Up @@ -140,7 +141,7 @@ private protected override void CreateSection(ObjectNodeSection section, string
});
_symbols.Add(auxRecord);

if (symbolName is not null)
if (!symbolName.IsNull)
{
_symbolNameToIndex.Add(symbolName, (uint)_symbols.Count);
_symbols.Add(new CoffSymbol
Expand Down Expand Up @@ -186,7 +187,7 @@ protected internal override unsafe void EmitRelocation(
long offset,
Span<byte> data,
RelocType relocType,
string symbolName,
Utf8String symbolName,
long addend)
{
if (relocType is IMAGE_REL_BASED_RELPTR32)
Expand All @@ -206,14 +207,14 @@ protected internal override unsafe void EmitRelocation(
base.EmitRelocation(sectionIndex, offset, data, relocType, symbolName, 0);
}

private protected override void EmitReferencedMethod(string symbolName)
private protected override void EmitReferencedMethod(Utf8String symbolName)
{
_referencedMethods.Add(symbolName);
}

private protected override void EmitSymbolTable(
IDictionary<string, SymbolDefinition> definedSymbols,
SortedSet<string> undefinedSymbols)
IDictionary<Utf8String, SymbolDefinition> definedSymbols,
SortedSet<Utf8String> undefinedSymbols)
{
Feat00Flags feat00Flags = _machine is Machine.I386 ? Feat00Flags.SafeSEH : 0;

Expand Down Expand Up @@ -734,7 +735,7 @@ private enum CoffSymbolClass : byte

private sealed class CoffSymbol : CoffSymbolRecord
{
public string Name { get; set; }
public Utf8String Name { get; set; }
public uint Value { get; set; }
public uint SectionIndex { get; set; }
public ushort Type { get; set; }
Expand Down Expand Up @@ -763,13 +764,12 @@ public override void Write(Stream stream, CoffStringTable stringTable, bool isBi
{
Span<byte> buffer = stackalloc byte[isBigObj ? BigObjSize : RegularSize];

int nameBytes = Encoding.UTF8.GetByteCount(Name);
if (nameBytes <= NameSize)
if (Name.Length <= NameSize)
{
Encoding.UTF8.GetBytes(Name, buffer);
if (nameBytes < NameSize)
Name.AsSpan().CopyTo(buffer);
if (Name.Length < NameSize)
{
buffer.Slice(nameBytes, 8 - nameBytes).Clear();
buffer.Slice(Name.Length, 8 - Name.Length).Clear();
}
}
else
Expand Down Expand Up @@ -853,7 +853,7 @@ protected sealed class CoffStringTable : StringTableBuilder
{
public new uint Size => (uint)(base.Size + 4);

public new uint GetStringOffset(string text)
public new uint GetStringOffset(Utf8String text)
{
return base.GetStringOffset(text) + 4;
}
Expand Down
35 changes: 18 additions & 17 deletions src/coreclr/tools/Common/Compiler/ObjectWriter/ElfObjectWriter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
using System.Reflection;
using ILCompiler.DependencyAnalysis;
using ILCompiler.DependencyAnalysisFramework;
using Internal.Text;
using Internal.TypeSystem;
using static ILCompiler.DependencyAnalysis.RelocType;
using static ILCompiler.ObjectWriter.EabiNative;
Expand Down Expand Up @@ -40,10 +41,10 @@ internal sealed partial class ElfObjectWriter : UnixObjectWriter
private readonly List<ElfSectionDefinition> _sections = new();
private readonly List<ElfSymbol> _symbols = new();
private uint _localSymbolCount;
private readonly Dictionary<string, ElfSectionDefinition> _comdatNameToElfSection = new(StringComparer.Ordinal);
private readonly Dictionary<Utf8String, ElfSectionDefinition> _comdatNameToElfSection = new();

// Symbol table
private readonly Dictionary<string, uint> _symbolNameToIndex = new();
private readonly Dictionary<Utf8String, uint> _symbolNameToIndex = new();

private static readonly ObjectNodeSection ArmAttributesSection = new ObjectNodeSection(".ARM.attributes", SectionType.ReadOnly);
private static readonly ObjectNodeSection ArmTextThunkSection = new ObjectNodeSection(".text.thunks", SectionType.Executable);
Expand All @@ -69,7 +70,7 @@ public ElfObjectWriter(NodeFactory factory, ObjectWritingOptions options)
_symbols.Add(new ElfSymbol {});
}

private protected override void CreateSection(ObjectNodeSection section, string comdatName, string symbolName, int sectionIndex, Stream sectionStream)
private protected override void CreateSection(ObjectNodeSection section, Utf8String comdatName, Utf8String symbolName, int sectionIndex, Stream sectionStream)
{
string sectionName =
section.Name == "rdata" ? ".rodata" :
Expand Down Expand Up @@ -114,7 +115,7 @@ private protected override void CreateSection(ObjectNodeSection section, string
};
}

if (comdatName is not null)
if (!comdatName.IsNull)
{
flags |= SHF_GROUP;
if (!_comdatNameToElfSection.TryGetValue(comdatName, out groupSection))
Expand Down Expand Up @@ -154,7 +155,7 @@ private protected override void CreateSection(ObjectNodeSection section, string
});

// Emit section symbol into symbol table (for COMDAT the defining symbol is section symbol)
if (comdatName is null)
if (comdatName.IsNull)
{
_symbolNameToIndex[sectionName] = (uint)_symbols.Count;
_symbols.Add(new ElfSymbol
Expand All @@ -174,7 +175,7 @@ private protected override void CreateSection(ObjectNodeSection section, string
});
}

base.CreateSection(section, comdatName, symbolName ?? sectionName, sectionIndex, sectionStream);
base.CreateSection(section, comdatName, symbolName.IsNull ? sectionName : symbolName, sectionIndex, sectionStream);
}

protected internal override void UpdateSectionAlignment(int sectionIndex, int alignment)
Expand All @@ -188,7 +189,7 @@ protected internal override unsafe void EmitRelocation(
long offset,
Span<byte> data,
RelocType relocType,
string symbolName,
Utf8String symbolName,
long addend)
{
fixed (byte *pData = data)
Expand Down Expand Up @@ -265,11 +266,11 @@ protected internal override unsafe void EmitRelocation(
private static string GetRiscV64SymbolNameForPcrelRelocation(int sectionIndex, long offset) => $".Lpcrel_hi{sectionIndex}_{offset:x}";

private protected override void EmitSymbolTable(
IDictionary<string, SymbolDefinition> definedSymbols,
SortedSet<string> undefinedSymbols)
IDictionary<Utf8String, SymbolDefinition> definedSymbols,
SortedSet<Utf8String> undefinedSymbols)
{
List<ElfSymbol> sortedSymbols = new(definedSymbols.Count + undefinedSymbols.Count);
foreach ((string name, SymbolDefinition definition) in definedSymbols)
foreach ((Utf8String name, SymbolDefinition definition) in definedSymbols)
{
var section = _sections[definition.SectionIndex];
var type =
Expand All @@ -290,7 +291,7 @@ private protected override void EmitSymbolTable(
int thunkSectionIndex = useArmThunks ? GetOrCreateSection(ArmTextThunkSection).SectionIndex : 0;
int thunkSymbolsIndex = 0;

foreach (string externSymbol in undefinedSymbols)
foreach (Utf8String externSymbol in undefinedSymbols)
{
if (!_symbolNameToIndex.ContainsKey(externSymbol))
{
Expand All @@ -316,7 +317,7 @@ private protected override void EmitSymbolTable(
}
}

sortedSymbols.Sort((symA, symB) => string.CompareOrdinal(symA.Name, symB.Name));
sortedSymbols.Sort((symA, symB) => Comparer<Utf8String>.Default.Compare(symA.Name, symB.Name));
_localSymbolCount = (uint)_symbols.Count;
_symbols.AddRange(sortedSymbols);
uint symbolIndex = _localSymbolCount;
Expand All @@ -337,7 +338,7 @@ private protected override void EmitSymbolTable(
Span<byte> relocationEntry = stackalloc byte[8];
var relocationStream = new MemoryStream(8 * undefinedSymbols.Count);
_sections[thunkSectionWriter.SectionIndex].RelocationStream = relocationStream;
foreach (string externSymbol in undefinedSymbols)
foreach (Utf8String externSymbol in undefinedSymbols)
{
if (_symbolNameToIndex.TryGetValue($"{externSymbol}$thunk", out uint thunkSymbolIndex))
{
Expand All @@ -356,7 +357,7 @@ private protected override void EmitSymbolTable(
}

// Update group sections links
foreach ((string comdatName, ElfSectionDefinition groupSection) in _comdatNameToElfSection)
foreach ((Utf8String comdatName, ElfSectionDefinition groupSection) in _comdatNameToElfSection)
{
groupSection.SectionHeader.Info = (uint)_symbolNameToIndex[comdatName];
}
Expand Down Expand Up @@ -669,7 +670,7 @@ private void EmitObjectFile<TSize>(Stream outputFileStream)
// Reserve all symbol names
foreach (var symbol in _symbols)
{
if (symbol.Name is not null)
if (!symbol.Name.IsNull)
{
_stringTable.ReserveString(symbol.Name);
}
Expand Down Expand Up @@ -1039,7 +1040,7 @@ public void Write<TSize>(Stream stream)

private sealed class ElfSymbol
{
public string Name { get; init; }
public Utf8String Name { get; init; }
public ulong Value { get; init; }
public ulong Size { get; init; }
public ElfSectionDefinition Section { get; init; }
Expand All @@ -1062,7 +1063,7 @@ public void Write<TSize>(Stream stream, ElfStringTable stringTable)
(ushort)SHN_XINDEX :
(Section is not null ? (ushort)Section.SectionIndex : (ushort)0u);

BinaryPrimitives.WriteUInt32LittleEndian(buffer, Name is not null ? stringTable.GetStringOffset(Name) : 0);
BinaryPrimitives.WriteUInt32LittleEndian(buffer, !Name.IsNull ? stringTable.GetStringOffset(Name) : 0);
if (typeof(TSize) == typeof(uint))
{
TSize.CreateChecked(Value).WriteLittleEndian(buffer.Slice(4));
Expand Down
Loading
Loading