Skip to content

vllm_gaudi.extension.features

get_experimental_flags 

get_experimental_flags()
Source code in vllm_gaudi/extension/features.py 
68
69
70
71
72
73
74
75
76
77
78
def get_experimental_flags():
    flags = [
        Env('VLLM_PT_PROFILE', str),
        Env('VLLM_PROFILE_PROMPT', str),
        Env('VLLM_PROFILE_DECODE', str),
        Env('VLLM_PROFILE_STEPS', list_of(int)),
        Env('VLLM_DEFRAG_THRESHOLD', int),
        Env('VLLM_DEFRAG_WITH_GRAPHS', boolean),
        Env('VLLM_DEBUG', list_of(str), check=for_all(choice('steps', 'defrag', 'fwd'))),
    ]
    return to_dict(flags)

get_features 

get_features()
Source code in vllm_gaudi/extension/features.py 
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
def get_features():
    supported_attn_impls = ['flex_impl', 'fsdpa_impl', 'naive_impl']
    features = [
        Value('fp32_alibi_biases', True, env_var='VLLM_ALIBI_USE_FLOAT32_BIASES'),
        Value('fp32_softmax', Any(ModelType('qwen2'), ModelType('qwen2_5_vl'))),
        Value(
            'fused_block_softmax_adjustment',
            All(VersionRange(">=1.22.0.494"), Hardware('gaudi3'), Kernel(block_softmax_adjustment),
                Not(ModelType('qwen2')))),
        Value('fused_block_softmax', False),
        Value('flex_impl', False, env_var='VLLM_PROMPT_USE_FLEX_ATTENTION'),
        Value('fsdpa_impl', All(Kernel(fsdpa), Not(ModelType('mllama'))), env_var='VLLM_PROMPT_USE_FUSEDSDPA'),
        Value('naive_impl', True),
        ValueFromList('prompt_attn_impl', supported_attn_impls),
        Value('skip_warmup', False),
        Value('merged_prefill', False),
        Value('use_contiguous_pa', Disabled('prefix_caching'), env_var='VLLM_CONTIGUOUS_PA'),
        Value('use_bucketing', True, env_var='VLLM_ENABLE_BUCKETING'),
        Value('bucketing_strategy',
              'exp',
              env_var='VLLM_BUCKETING_STRATEGY',
              env_var_type=str,
              check=choice('exp', 'lin', 'pad')),
        Value('defrag', Enabled('use_contiguous_pa')),
        Value('regional_compilation', True, env_var='VLLM_T_COMPILE_REGIONAL_COMPILATION', env_var_type=boolean),
        Value('dynamic_shapes_compilation', True, env_var='VLLM_T_COMPILE_DYNAMIC_SHAPES', env_var_type=boolean),
        Value('fullgraph_compilation', False, env_var='VLLM_T_COMPILE_FULLGRAPH', env_var_type=boolean),
        Value('scale_adjustment', True, env_var='VLLM_SCALE_ADJUSTMENT', env_var_type=boolean),
        Value('flatten_input', Any(ModelType('qwen3_moe'), ModelType('granitemoe'), ModelType('glm4_moe'))),
        Value('high_level_profiler_enabled', False, env_var='VLLM_PROFILER_ENABLED', env_var_type=boolean),
        Value('track_graph_compilation', False, env_var='PT_HPU_METRICS_GC_DETAILS', env_var_type=boolean),
        Value('per_token_kv_scaling_support',
              All(VersionRange(">=1.24.0.350"), MinPackageVersion("neural_compressor_pt", "3.7")),
              env_var_type=boolean),
        Value('moe_chunk', "", env_var='VLLM_MOE_CHUNK', env_var_type=list_of(int)),
        Value('moe_token_boundary', "", env_var='VLLM_MOE_TOKEN_BOUNDARY', env_var_type=list_of(int)),
        Value('row_parallel_chunks', 1, env_var='VLLM_ROW_PARALLEL_CHUNKS', env_var_type=int),
        Value('row_parallel_chunk_threshold', 8192, env_var='VLLM_ROW_PARALLEL_CHUNK_THRESHOLD', env_var_type=int),
        Value('use_dispatch_fn',
              All(VersionRange(">=1.24.0.460"), MinPackageVersion("neural_compressor_pt", "3.7")),
              env_var_type=boolean),
        Value('use_hpu_aligned_scale', False, env_var='HPU_ALIGNED_SCALE', env_var_type=boolean),
        Value('enable_fsdpa_slicing',
              All(Eq('use_bucketing', True), Eq('bucketing_strategy', 'pad'), Disabled('merged_prefill'),
                  Kernel(fsdpa)),
              env_var='VLLM_HPU_FSDPA_SLICE_ENABLED',
              env_var_type=boolean),
    ]
    return split_values_and_flags(features)

get_user_flags 

get_user_flags()
Source code in vllm_gaudi/extension/features.py 
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
def get_user_flags():
    flags = [
        Env('VLLM_DEVELOPER_MODE', boolean),
        Env('VLLM_PROMPT_BS_BUCKET_MIN', int),
        Env('VLLM_PROMPT_BS_BUCKET_STEP', int),
        Env('VLLM_PROMPT_BS_BUCKET_MAX', int),
        Env('VLLM_PROMPT_BS_BUCKET_PAD_MAX', int),
        Env('VLLM_PROMPT_BS_BUCKET_PAD_PERCENT', int),
        Env('VLLM_PROMPT_QUERY_BUCKET_MIN', int),
        Env('VLLM_PROMPT_QUERY_BUCKET_STEP', int),
        Env('VLLM_PROMPT_QUERY_BUCKET_MAX', int),
        Env('VLLM_PROMPT_QUERY_BUCKET_PAD_MAX', int),
        Env('VLLM_PROMPT_QUERY_BUCKET_PAD_PERCENT', int),
        Env('VLLM_PROMPT_SEQ_BUCKET_MIN', int),
        Env('VLLM_PROMPT_SEQ_BUCKET_STEP', int),
        Env('VLLM_PROMPT_SEQ_BUCKET_MAX', int),
        Env('VLLM_PROMPT_SEQ_BUCKET_PAD_MAX', int),
        Env('VLLM_PROMPT_SEQ_BUCKET_PAD_PERCENT', int),
        Env('VLLM_PROMPT_CTX_BUCKET_MIN', int),
        Env('VLLM_PROMPT_CTX_BUCKET_STEP', int),
        Env('VLLM_PROMPT_CTX_BUCKET_MAX', int),
        Env('VLLM_PROMPT_CTX_BUCKET_PAD_MAX', int),
        Env('VLLM_PROMPT_CTX_BUCKET_PAD_PERCENT', int),
        Env('VLLM_DECODE_BS_BUCKET_MIN', int),
        Env('VLLM_DECODE_BS_BUCKET_STEP', int),
        Env('VLLM_DECODE_BS_BUCKET_MAX', int),
        Env('VLLM_DECODE_BS_BUCKET_PAD_MAX', int),
        Env('VLLM_DECODE_BS_BUCKET_PAD_PERCENT', int),
        Env('VLLM_DECODE_BLOCK_BUCKET_MIN', int),
        Env('VLLM_DECODE_BLOCK_BUCKET_STEP', int),
        Env('VLLM_DECODE_BLOCK_BUCKET_MAX', int),
        Env('VLLM_DECODE_BLOCK_BUCKET_PAD_MAX', int),
        Env('VLLM_DECODE_BLOCK_BUCKET_PAD_PERCENT', int),
        Env('VLLM_BUCKETING_STRATEGY', str),
        Env('VLLM_BUCKETING_FROM_FILE', str),

        # Non-vllm flags that are also important to print
        Env('EXPERIMENTAL_WEIGHT_SHARING', str),
        Env('PT_HPU_WEIGHT_SHARING', str),
        Env('RUNTIME_SCALE_PATCHING', str),

        # Sliding window flags
        Env('PT_HPU_SDPA_QKV_SLICE_MODE_FWD', boolean),
        Env('PT_HPU_SDPA_BC_FACTOR', int),
        Env('VLLM_FUSEDSDPA_SLIDE_THLD', int),

        # FusedSDPA slicing flags
        Env('VLLM_HPU_FSDPA_SLICE_ENABLED', boolean),
        Env('VLLM_HPU_FSDPA_SLICE_SEQ_LEN_THLD', int),
        Env('VLLM_HPU_FSDPA_SLICE_CHUNK_SIZE', int),
        Env('VLLM_HPU_FSDPA_SLICE_WITH_GRAPH_BREAKS', boolean),
    ]
    return to_dict(flags)