Skip to content

Missed Optimization: Missed Unlikely Optimization #167117

@atomic-kernel

Description

@atomic-kernel

look at this emample:
https://godbolt.org/z/jrrK8q3Ex

#include <stdbool.h>

# define likely(x)      __builtin_expect(!!(x), 1)
# define unlikely(x)    __builtin_expect(!!(x), 0)

extern int xxx;

void temp(void)
{
        if (({
                    const bool __ret_do_once = unlikely(xxx != 2);  // define unlikely here
                    static bool __attribute__((__section__(".data..once"))) __already_done;
                    if (__ret_do_once && !__already_done)
                        __already_done = true; // -> unlikely path

                    __ret_do_once;
                }))
                __asm__ volatile ("nop":::); // -> unlikely path
}

The assemble of clang is:

temp:                                   # @temp
        .cfi_startproc
        movq    xxx@GOTPCREL(%rip), %rax
        movl    (%rax), %eax
        cmpl    $2, %eax
        je      .LBB0_3                            //  eax ==2 is likely, so here jmp likely
        testb   $1, temp.__already_done(%rip)
        jne     .LBB0_3
        movb    $1, temp.__already_done(%rip)
        jmp     .LBB0_4
.LBB0_3: 
        cmpl    $2, %eax                        // eax is not changed, so jmp again
        je      .LBB0_5
.LBB0_4:
        #APP
        nop
        #NO_APP
.LBB0_5:
        retq

The gcc assemble is :

temp:
        cmpl    $2, xxx(%rip)
        jne     .L5                            // jump unlikely right
        ret                                      // likely run here, return fast
.L5:
        cmpb    $0, __already_done.0(%rip)
        jne     .L3
        movb    $1, __already_done.0(%rip)
.L3:
        nop
        ret

This results in all the WARN_ONCE branches in the Linux kernel source code being judged twice, causing significant performance loss:
https://elixir.bootlin.com/linux/v6.18-rc4/source/include/linux/once_lite.h#L19

Metadata

Metadata

Assignees

No one assigned

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions