1#!/bin/bash 2# Copyright (c) Meta Platforms, Inc. and affiliates. 3# All rights reserved. 4# 5# This source code is licensed under the BSD-style license found in the 6# LICENSE file in the root directory of this source tree. 7 8set -exu 9 10if [[ -z "${PYTHON_EXECUTABLE:-}" ]]; then 11 PYTHON_EXECUTABLE=python3 12fi 13 14# Download and prepare stories model artifacts 15prepare_model_artifacts() { 16 echo "Preparing stories model artifacts" 17 wget -O stories110M.pt "https://huggingface.co/karpathy/tinyllamas/resolve/main/stories110M.pt" 18 wget -O tokenizer.model "https://raw.githubusercontent.com/karpathy/llama2.c/master/tokenizer.model" 19 echo '{"dim": 768, "multiple_of": 32, "n_heads": 12, "n_layers": 12, "norm_eps": 1e-05, "vocab_size": 32000}' > params.json 20} 21 22run_and_verify() { 23 NOW=$(date +"%H:%M:%S") 24 echo "Starting to run eval_llama at ${NOW}" 25 if [[ ! -f "stories110M.pt" ]]; then 26 echo "stories110M.pt is missing." 27 exit 1 28 fi 29 if [[ ! -f "tokenizer.model" ]]; then 30 echo "tokenizer.model is missing." 31 exit 1 32 fi 33 if [[ ! -f "params.json" ]]; then 34 echo "params.json is missing." 35 exit 1 36 fi 37 $PYTHON_EXECUTABLE -m examples.models.llama.eval_llama \ 38 -c stories110M.pt \ 39 -p params.json \ 40 -t tokenizer.model \ 41 -kv \ 42 -d fp32 \ 43 --max_seq_length 2048 \ 44 --limit 5 > result.txt 45 46 # Verify result.txt 47 RESULT=$(cat result.txt) 48 EXPECTED_TASK="wikitext" 49 EXPECTED_RESULT="word_perplexity" 50 if [[ "${RESULT}" == "${EXPECTED_TASK}: {"*"${EXPECTED_RESULT}"* ]]; then 51 echo "Actual result: ${RESULT}" 52 echo "Success" 53 exit 0 54 else 55 echo "Actual result: ${RESULT}" 56 echo "Failure; results not the same" 57 exit 1 58 fi 59} 60 61prepare_model_artifacts 62run_and_verify 63