Skip to content

Commit 5eeae08

Browse files
wdx727lifengxiang1025zcfhrlavaee
authored
Adding Matching and Inference Functionality to Propeller (#160706)
We have optimized the implementation of introducing the "matching and inference" technique into Propeller. In this new implementation, we have made every effort to avoid introducing new compilation parameters while ensuring compatibility with Propeller's current usage. Instead of creating a new profile format, we reused the existing one employed by Propeller. This new implementation is fully compatible with Propeller's current usage patterns and reduces the amount of code changes. For detailed information, please refer to the following RFC: https://discourse.llvm.org/t/rfc-adding-matching-and-inference-functionality-to-propeller/86238. We plan to submit the relevant changes in several pull requests (PRs). The current one is the first PR, which adds the basic block hash to the SHT_LLVM_BB_ADDR_MAP section. co-authors: lifengxiang1025 <lifengxiang@kuaishou.com>; zcfh <wuminghui03@kuaishou.com> Co-authored-by: lifengxiang1025 <lifengxiang@kuaishou.com> Co-authored-by: zcfh <wuminghui03@kuaishou.com> Co-authored-by: Rahman Lavaee <rahmanl@google.com>
1 parent 2e11538 commit 5eeae08

File tree

13 files changed

+322
-62
lines changed

13 files changed

+322
-62
lines changed

‎llvm/include/llvm/Object/ELFTypes.h‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -833,6 +833,7 @@ struct BBAddrMap {
833833
bool MultiBBRange : 1;
834834
bool OmitBBEntries : 1;
835835
bool CallsiteEndOffsets : 1;
836+
bool BBHash : 1;
836837

837838
bool hasPGOAnalysis() const { return FuncEntryCount || BBFreq || BrProb; }
838839

@@ -845,7 +846,8 @@ struct BBAddrMap {
845846
(static_cast<uint8_t>(BrProb) << 2) |
846847
(static_cast<uint8_t>(MultiBBRange) << 3) |
847848
(static_cast<uint8_t>(OmitBBEntries) << 4) |
848-
(static_cast<uint8_t>(CallsiteEndOffsets) << 5);
849+
(static_cast<uint8_t>(CallsiteEndOffsets) << 5) |
850+
(static_cast<uint8_t>(BBHash) << 6);
849851
}
850852

851853
// Decodes from minimum bit width representation and validates no
@@ -854,7 +856,8 @@ struct BBAddrMap {
854856
Features Feat{
855857
static_cast<bool>(Val & (1 << 0)), static_cast<bool>(Val & (1 << 1)),
856858
static_cast<bool>(Val & (1 << 2)), static_cast<bool>(Val & (1 << 3)),
857-
static_cast<bool>(Val & (1 << 4)), static_cast<bool>(Val & (1 << 5))};
859+
static_cast<bool>(Val & (1 << 4)), static_cast<bool>(Val & (1 << 5)),
860+
static_cast<bool>(Val & (1 << 6))};
858861
if (Feat.encode() != Val)
859862
return createStringError(
860863
std::error_code(), "invalid encoding for BBAddrMap::Features: 0x%x",
@@ -864,10 +867,10 @@ struct BBAddrMap {
864867

865868
bool operator==(const Features &Other) const {
866869
return std::tie(FuncEntryCount, BBFreq, BrProb, MultiBBRange,
867-
OmitBBEntries, CallsiteEndOffsets) ==
870+
OmitBBEntries, CallsiteEndOffsets, BBHash) ==
868871
std::tie(Other.FuncEntryCount, Other.BBFreq, Other.BrProb,
869872
Other.MultiBBRange, Other.OmitBBEntries,
870-
Other.CallsiteEndOffsets);
873+
Other.CallsiteEndOffsets, Other.BBHash);
871874
}
872875
};
873876

@@ -920,17 +923,19 @@ struct BBAddrMap {
920923
false}; // Metdata for this basic block.
921924
// Offsets of end of call instructions, relative to the basic block start.
922925
SmallVector<uint32_t, 1> CallsiteEndOffsets;
926+
uint64_t Hash = 0; // Hash for this basic block.
923927

924928
BBEntry(uint32_t ID, uint32_t Offset, uint32_t Size, Metadata MD,
925-
SmallVector<uint32_t, 1> CallsiteEndOffsets)
929+
SmallVector<uint32_t, 1> CallsiteEndOffsets, uint64_t Hash)
926930
: ID(ID), Offset(Offset), Size(Size), MD(MD),
927-
CallsiteEndOffsets(std::move(CallsiteEndOffsets)) {}
931+
CallsiteEndOffsets(std::move(CallsiteEndOffsets)), Hash(Hash) {}
928932

929933
UniqueBBID getID() const { return {ID, 0}; }
930934

931935
bool operator==(const BBEntry &Other) const {
932936
return ID == Other.ID && Offset == Other.Offset && Size == Other.Size &&
933-
MD == Other.MD && CallsiteEndOffsets == Other.CallsiteEndOffsets;
937+
MD == Other.MD && CallsiteEndOffsets == Other.CallsiteEndOffsets &&
938+
Hash == Other.Hash;
934939
}
935940

936941
bool hasReturn() const { return MD.HasReturn; }

‎llvm/include/llvm/ObjectYAML/ELFYAML.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,7 @@ struct BBAddrMapEntry {
163163
llvm::yaml::Hex64 Size;
164164
llvm::yaml::Hex64 Metadata;
165165
std::optional<std::vector<llvm::yaml::Hex64>> CallsiteEndOffsets;
166+
std::optional<llvm::yaml::Hex64> Hash;
166167
};
167168
uint8_t Version;
168169
llvm::yaml::Hex8 Feature;

‎llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1437,7 +1437,8 @@ getBBAddrMapFeature(const MachineFunction &MF, int NumMBBSectionRanges,
14371437
BrProbEnabled,
14381438
MF.hasBBSections() && NumMBBSectionRanges > 1,
14391439
static_cast<bool>(BBAddrMapSkipEmitBBEntries),
1440-
HasCalls};
1440+
HasCalls,
1441+
false};
14411442
}
14421443

14431444
void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {

‎llvm/lib/Object/ELF.cpp‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -837,7 +837,7 @@ decodeBBAddrMapImpl(const ELFFile<ELFT> &EF,
837837
Version = Data.getU8(Cur);
838838
if (!Cur)
839839
break;
840-
if (Version < 2 || Version > 3)
840+
if (Version < 2 || Version > 4)
841841
return createError("unsupported SHT_LLVM_BB_ADDR_MAP version: " +
842842
Twine(static_cast<int>(Version)));
843843
Feature = Data.getU8(Cur); // Feature byte
@@ -852,6 +852,11 @@ decodeBBAddrMapImpl(const ELFFile<ELFT> &EF,
852852
"callsite offsets feature is enabled: version = " +
853853
Twine(static_cast<int>(Version)) +
854854
" feature = " + Twine(static_cast<int>(Feature)));
855+
if (FeatEnable.BBHash && Version < 4)
856+
return createError("version should be >= 4 for SHT_LLVM_BB_ADDR_MAP when "
857+
"basic block hash feature is enabled: version = " +
858+
Twine(static_cast<int>(Version)) +
859+
" feature = " + Twine(static_cast<int>(Feature)));
855860
uint32_t NumBlocksInBBRange = 0;
856861
uint32_t NumBBRanges = 1;
857862
typename ELFFile<ELFT>::uintX_t RangeBaseAddress = 0;
@@ -907,14 +912,15 @@ decodeBBAddrMapImpl(const ELFFile<ELFT> &EF,
907912
uint32_t Size = readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr) +
908913
LastCallsiteEndOffset;
909914
uint32_t MD = readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr);
915+
uint64_t Hash = FeatEnable.BBHash ? Data.getU64(Cur) : 0;
910916
Expected<BBAddrMap::BBEntry::Metadata> MetadataOrErr =
911917
BBAddrMap::BBEntry::Metadata::decode(MD);
912918
if (!MetadataOrErr) {
913919
MetadataDecodeErr = MetadataOrErr.takeError();
914920
break;
915921
}
916922
BBEntries.push_back({ID, Offset + PrevBBEndOffset, Size,
917-
*MetadataOrErr, CallsiteEndOffsets});
923+
*MetadataOrErr, CallsiteEndOffsets, Hash});
918924
PrevBBEndOffset += Offset + Size;
919925
}
920926
TotalNumBlocks += BBEntries.size();

‎llvm/lib/ObjectYAML/ELFEmitter.cpp‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1465,7 +1465,7 @@ void ELFState<ELFT>::writeSectionContent(
14651465
for (const auto &[Idx, E] : llvm::enumerate(*Section.Entries)) {
14661466
// Write version and feature values.
14671467
if (Section.Type == llvm::ELF::SHT_LLVM_BB_ADDR_MAP) {
1468-
if (E.Version > 3)
1468+
if (E.Version > 4)
14691469
WithColor::warning() << "unsupported SHT_LLVM_BB_ADDR_MAP version: "
14701470
<< static_cast<int>(E.Version)
14711471
<< "; encoding using the most recent version";
@@ -1526,6 +1526,12 @@ void ELFState<ELFT>::writeSectionContent(
15261526
}
15271527
SHeader.sh_size += CBA.writeULEB128(BBE.Size);
15281528
SHeader.sh_size += CBA.writeULEB128(BBE.Metadata);
1529+
if (FeatureOrErr->BBHash || BBE.Hash.has_value()) {
1530+
uint64_t Hash =
1531+
BBE.Hash.has_value() ? BBE.Hash.value() : llvm::yaml::Hex64(0);
1532+
CBA.write<uint64_t>(Hash, ELFT::Endianness);
1533+
SHeader.sh_size += 8;
1534+
}
15291535
}
15301536
}
15311537
if (!PGOAnalyses)

‎llvm/lib/ObjectYAML/ELFYAML.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1887,6 +1887,7 @@ void MappingTraits<ELFYAML::BBAddrMapEntry::BBEntry>::mapping(
18871887
IO.mapRequired("Size", E.Size);
18881888
IO.mapRequired("Metadata", E.Metadata);
18891889
IO.mapOptional("CallsiteEndOffsets", E.CallsiteEndOffsets);
1890+
IO.mapOptional("Hash", E.Hash);
18901891
}
18911892

18921893
void MappingTraits<ELFYAML::PGOAnalysisMapEntry>::mapping(

‎llvm/test/tools/llvm-readobj/ELF/bb-addr-map.test‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
# CHECK-NEXT: {
3535
# CHECK-NEXT: ID: 0
3636
# CHECK-NEXT: Offset: 0x0
37+
# CHECK-NEXT: Hash: 0x0
3738
# CHECK-NEXT: Size: 0x1
3839
# CHECK-NEXT: HasReturn: No
3940
# CHECK-NEXT: HasTailCall: Yes
@@ -50,6 +51,7 @@
5051
# CHECK-NEXT: ID: 2
5152
# CHECK-NEXT: Offset: 0x3
5253
# CHECK-NEXT: Callsite End Offsets: [1, 3]
54+
# CHECK-NEXT: Hash: 0x123
5355
# CHECK-NEXT: Size: 0x7
5456
# CHECK-NEXT: HasReturn: Yes
5557
# CHECK-NEXT: HasTailCall: No
@@ -144,8 +146,8 @@ Sections:
144146
ShSize: [[SIZE=<none>]]
145147
Link: .text
146148
Entries:
147-
- Version: 3
148-
Feature: 0x28
149+
- Version: 4
150+
Feature: 0x68
149151
BBRanges:
150152
- BaseAddress: [[ADDR=0x11111]]
151153
BBEntries:
@@ -160,6 +162,7 @@ Sections:
160162
Size: 0x4
161163
Metadata: 0x15
162164
CallsiteEndOffsets: [ 0x1 , 0x2 ]
165+
Hash: 0x123
163166
- Version: 2
164167
BBRanges:
165168
- BaseAddress: 0x22222

‎llvm/test/tools/obj2yaml/ELF/bb-addr-map.yaml‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,92 @@ Sections:
162162
BBRanges:
163163
- BaseAddress: 0x20
164164

165+
## Check that obj2yaml can dump basic block hash in the .llvm_bb_addr_map section.
166+
167+
# RUN: yaml2obj --docnum=4 %s -o %t4
168+
# RUN: obj2yaml %t4 | FileCheck %s --check-prefix=BBHASH
169+
170+
# BBHASH: --- !ELF
171+
# BBHASH-NEXT: FileHeader:
172+
# BBHASH-NEXT: Class: ELFCLASS64
173+
# BBHASH-NEXT: Data: ELFDATA2LSB
174+
# BBHASH-NEXT: Type: ET_EXEC
175+
# BBHASH-NEXT: Sections:
176+
# BBHASH-NEXT: - Name: .llvm_bb_addr_map
177+
# BBHASH-NEXT: Type: SHT_LLVM_BB_ADDR_MAP
178+
# BBHASH-NEXT: Entries:
179+
# BBHASH-NEXT: - Version: 4
180+
# BBHASH-NEXT: Feature: 0x40
181+
# BBHASH-NEXT: BBRanges:
182+
# BBHASH-NEXT: - BBEntries:
183+
# BBHASH-NEXT: - ID: 0
184+
# BBHASH-NEXT: AddressOffset: 0x1
185+
# BBHASH-NEXT: Size: 0x2
186+
# BBHASH-NEXT: Metadata: 0x3
187+
# BBHASH-NEXT: Hash: 0x1
188+
# BBHASH-NEXT: - ID: 2
189+
# BBHASH-NEXT: AddressOffset: 0x4
190+
# BBHASH-NEXT: Size: 0x5
191+
# BBHASH-NEXT: Metadata: 0x6
192+
# BBHASH-NEXT: Hash: 0x2
193+
# BBHASH-NEXT: - ID: 4
194+
# BBHASH-NEXT: AddressOffset: 0xFFFFFFFFFFFFFFF7
195+
# BBHASH-NEXT: Size: 0xFFFFFFFFFFFFFFF8
196+
# BBHASH-NEXT: Metadata: 0xFFFFFFFFFFFFFFF9
197+
# BBHASH-NEXT: Hash: 0x3
198+
# BBHASH-NEXT: - Version: 4
199+
# BBHASH-NEXT: Feature: 0x68
200+
# BBHASH-NEXT: BBRanges:
201+
# BBHASH-NEXT: - BaseAddress: 0xFFFFFFFFFFFFFF20
202+
# BBHASH-NEXT: BBEntries:
203+
# BBHASH-NEXT: - ID: 6
204+
# BBHASH-NEXT: AddressOffset: 0xA
205+
# BBHASH-NEXT: Size: 0xB
206+
# BBHASH-NEXT: Metadata: 0xC
207+
# BBHASH-NEXT: CallsiteEndOffsets: [ 0x1, 0x2 ]
208+
# BBHASH-NEXT: Hash: 0x123
209+
210+
--- !ELF
211+
FileHeader:
212+
Class: ELFCLASS64
213+
Data: ELFDATA2LSB
214+
Type: ET_EXEC
215+
Sections:
216+
- Name: .llvm_bb_addr_map
217+
Type: SHT_LLVM_BB_ADDR_MAP
218+
Entries:
219+
- Version: 4
220+
Feature: 0x40
221+
BBRanges:
222+
- BaseAddress: 0x0
223+
BBEntries:
224+
- ID: 0
225+
AddressOffset: 0x1
226+
Size: 0x2
227+
Metadata: 0x3
228+
Hash: 0x1
229+
- ID: 2
230+
AddressOffset: 0x4
231+
Size: 0x5
232+
Metadata: 0x6
233+
Hash: 0x2
234+
- ID: 4
235+
AddressOffset: 0xFFFFFFFFFFFFFFF7
236+
Size: 0xFFFFFFFFFFFFFFF8
237+
Metadata: 0xFFFFFFFFFFFFFFF9
238+
Hash: 0x3
239+
- Version: 4
240+
Feature: 0x68
241+
BBRanges:
242+
- BaseAddress: 0xFFFFFFFFFFFFFF20
243+
BBEntries:
244+
- ID: 6
245+
AddressOffset: 0xA
246+
Size: 0xB
247+
Metadata: 0xC
248+
CallsiteEndOffsets: [ 0x1, 0x2 ]
249+
Hash: 0x123
250+
165251
## Check that obj2yaml uses the "Content" tag to describe an .llvm_bb_addr_map section
166252
## when it can't extract the entries, for example, when the section is truncated, or
167253
## when an invalid 'NumBlocks' or 'NumBBRanges` field is specified.

‎llvm/test/tools/yaml2obj/ELF/bb-addr-map.yaml‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,13 @@
7272
# CHECK-NEXT: 0000: 03202000 00000000 0000010E 01000203
7373
# CHECK-NEXT: )
7474

75+
# Case 10: Specify basic block hash.
76+
# CHECK: Name: .llvm_bb_addr_map (1)
77+
# CHECK: SectionData (
78+
# CHECK-NEXT: 0000: 04602000 00000000 0000010E 01000203
79+
# CHECK-NEXT: 0010: 23010000 00000000
80+
# CHECK-NEXT: )
81+
7582

7683
--- !ELF
7784
FileHeader:
@@ -176,6 +183,22 @@ Sections:
176183
Metadata: 0x00000003
177184
CallsiteEndOffsets: []
178185

186+
## 10) We can produce a SHT_LLVM_BB_ADDR_MAP section with basic block hash.
187+
- Name: '.llvm_bb_addr_map (10)'
188+
Type: SHT_LLVM_BB_ADDR_MAP
189+
Entries:
190+
- Version: 4
191+
Feature: 0x60
192+
BBRanges:
193+
- BaseAddress: 0x0000000000000020
194+
BBEntries:
195+
- ID: 14
196+
AddressOffset: 0x00000001
197+
Size: 0x00000002
198+
Metadata: 0x00000003
199+
CallsiteEndOffsets: []
200+
Hash: 0x123
201+
179202
## Check we can't use Entries at the same time as either Content or Size.
180203
# RUN: not yaml2obj --docnum=2 -DCONTENT="00" %s 2>&1 | FileCheck %s --check-prefix=INVALID
181204
# RUN: not yaml2obj --docnum=2 -DSIZE="0" %s 2>&1 | FileCheck %s --check-prefix=INVALID
@@ -197,7 +220,7 @@ Sections:
197220

198221
## Check that yaml2obj generates a warning when we use unsupported versions.
199222
# RUN: yaml2obj --docnum=3 %s 2>&1 | FileCheck %s --check-prefix=INVALID-VERSION
200-
# INVALID-VERSION: warning: unsupported SHT_LLVM_BB_ADDR_MAP version: 4; encoding using the most recent version
223+
# INVALID-VERSION: warning: unsupported SHT_LLVM_BB_ADDR_MAP version: 5; encoding using the most recent version
201224

202225
--- !ELF
203226
FileHeader:
@@ -209,4 +232,4 @@ Sections:
209232
Type: SHT_LLVM_BB_ADDR_MAP
210233
Entries:
211234
## Specify unsupported version
212-
- Version: 4
235+
- Version: 5

‎llvm/tools/llvm-readobj/ELFDumper.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8155,6 +8155,8 @@ void LLVMELFDumper<ELFT>::printBBAddrMaps(bool PrettyPGOAnalysis) {
81558155
W.printHex("Offset", BBE.Offset);
81568156
if (!BBE.CallsiteEndOffsets.empty())
81578157
W.printList("Callsite End Offsets", BBE.CallsiteEndOffsets);
8158+
if (PAM.FeatEnable.BBHash)
8159+
W.printHex("Hash", BBE.Hash);
81588160
W.printHex("Size", BBE.Size);
81598161
W.printBoolean("HasReturn", BBE.hasReturn());
81608162
W.printBoolean("HasTailCall", BBE.hasTailCall());

0 commit comments

Comments
 (0)