xref: /aosp_15_r20/external/pytorch/benchmarks/dynamo/huggingface.yaml (revision da0073e96a02ea20f0ac840b70461e3646d07c45) 
1skip:
2  all:
3    # Difficult to setup accuracy test because .eval() not supported
4    - Reformer
5    # Fails deepcopy
6    - BlenderbotForConditionalGeneration
7    - GPTNeoForCausalLM
8    - GPTNeoForSequenceClassification
9    # Fails with even batch size = 1
10    - GPTJForCausalLM
11    - GPTJForQuestionAnswering
12
13  device:
14    cpu: []
15
16  control_flow:
17    - AllenaiLongformerBase
18
19batch_size:
20  # TODO - Fails even after fake tensors
21  divisors:
22    AlbertForMaskedLM: 2
23    AlbertForQuestionAnswering: 2
24    AllenaiLongformerBase: 2
25    BartForCausalLM: 2
26    BartForConditionalGeneration: 2
27    BertForMaskedLM: 2
28    BertForQuestionAnswering: 2
29    BlenderbotForCausalLM: 8
30    # BlenderbotForConditionalGeneration : 16
31    BlenderbotSmallForCausalLM: 4
32    BlenderbotSmallForConditionalGeneration: 2
33    CamemBert: 2
34    DebertaForMaskedLM: 4
35    DebertaForQuestionAnswering: 2
36    DebertaV2ForMaskedLM: 4
37    DebertaV2ForQuestionAnswering: 8
38    DistilBertForMaskedLM: 2
39    DistilBertForQuestionAnswering: 2
40    DistillGPT2: 2
41    ElectraForCausalLM: 2
42    ElectraForQuestionAnswering: 2
43    GPT2ForSequenceClassification: 2
44    # GPTJForCausalLM : 2
45    # GPTJForQuestionAnswering : 2
46    # GPTNeoForCausalLM : 32
47    # GPTNeoForSequenceClassification : 2
48    GoogleFnet: 2
49    LayoutLMForMaskedLM: 2
50    LayoutLMForSequenceClassification: 2
51    M2M100ForConditionalGeneration: 4
52    MBartForCausalLM: 2
53    MBartForConditionalGeneration: 2
54    MT5ForConditionalGeneration: 2
55    MegatronBertForCausalLM: 4
56    MegatronBertForQuestionAnswering: 2
57    MobileBertForMaskedLM: 2
58    MobileBertForQuestionAnswering: 2
59    OPTForCausalLM: 2
60    PLBartForCausalLM: 2
61    PLBartForConditionalGeneration: 2
62    PegasusForCausalLM: 4
63    PegasusForConditionalGeneration: 2
64    RobertaForCausalLM: 2
65    RobertaForQuestionAnswering: 2
66    Speech2Text2ForCausalLM: 4
67    T5ForConditionalGeneration: 2
68    T5Small: 2
69    TrOCRForCausalLM: 2
70    XGLMForCausalLM: 4
71    XLNetLMHeadModel: 2
72    YituTechConvBert: 2
73
74
75tolerance:
76  higher_training:
77    - MT5ForConditionalGeneration
78    # AlbertForQuestionAnswering fails in CI GCP A100 but error does not seem
79    # harmful.
80    - AlbertForQuestionAnswering
81
82  higher_max_autotune_training:
83    # DebertaForQuestionAnswering needs higher tolerance in Max-Autotune mode
84    - DebertaForQuestionAnswering
85
86  higher_inference:
87    - GPT2ForSequenceClassification
88    - RobertaForQuestionAnswering
89
90  higher_inference_cpu:
91    - LayoutLMForSequenceClassification
92
93  cosine: []
94
95
96accuracy:
97  skip:
98    large_models:
99      # Models too large to have eager, dynamo and fp64_numbers simultaneously
100      # even for 40 GB machine.
101      - DebertaV2ForMaskedLM
102      - BlenderbotForCausalLM
103
104only_inference:
105  # Fails with dynamo for train mode
106  - M2M100ForConditionalGeneration
107
108only_fp32:
109  - GoogleFnet
110