bigcode/starcoderplus

like0

pipeline_tag: text-generation
inference: true
widget:
- text: 'def print_hello_world():'
  example_title: Hello world
  group: Python
- text: 'Gradient descent is'
  example_title: Machine Learning
  group: English
- license: bigcode-openrail-m
datasets:
- bigcode/the-stack-dedup
- tiiuae/falcon-refinedweb
metrics:
- code_eval
- mmlu
- arc
- hellaswag
- truthfulqa
library_name: transformers
tags:
- code
model-index:
- name: StarCoderPlus
  results:
  - task:
      type: text-generation
    dataset:
      type: openai_humaneval
      name: HumanEval (Prompted)
    metrics:
    - name: pass@1
      type: pass@1
      value: 26.7
      verified: false
  - task:
      type: text-generation
    dataset:
      type: MMLU (5-shot)
      name: MMLU
    metrics:
    - name: Accuracy
      type: Accuracy
      value: 45.1
      verified: false
  - task:
      type: text-generation
    dataset:
      type: HellaSwag (10-shot)
      name: HellaSwag
    metrics:
    - name: Accuracy
      type: Accuracy
      value: 77.3
      verified: false
  - task:
      type: text-generation
    dataset:
      type: ARC (25-shot)
      name: ARC
    metrics:
    - name: Accuracy
      type: Accuracy
      value: 48.9
      verified: false
  - task:
      type: text-generation
    dataset:
      type: ThrutfulQA (0-shot)
      name: ThrutfulQA
    metrics:
    - name: Accuracy
      type: Accuracy
      value: 37.9
      verified: false
extra_gated_prompt: >-
  ## Model License Agreement

  Please read the BigCode [OpenRAIL-M
  license](https://huggingface.co/spaces/bigcode/bigcode-model-license-agreement)
  agreement before accepting it.
    
extra_gated_fields:
  I accept the above license agreement, and will use the Model complying with the set of use restrictions and sharing requirements: checkbox