I am attempting to fine-tune a HuggingFace model using Low Rank Adaptation (LoRA). I want to evaluate the model for each epoch while implementing 5-fold cross validation. With the LoRA feature, the eval_loss value is not being returned to evaluate each epoch.

I have tried modifying my compute_metrics function, and adding print statements to determine if the tensor shape is correct. However, I cannot get any logits to be returned. Here is my code:

`# Define compute metrics
def compute_metrics(eval_pred):
    logits, labels = eval_pred
    print(f"Logits shape: {logits.shape}")
    print(f"Type of logits: {type(logits)}")
    print(f"First row of logits: {logits[0]}")
    predictions = logits.argmax(axis=-1)
    
    # Calculate accuracy
    acc = accuracy_score(labels, predictions)
    
    # Convert logits and labels to tensors
    logits = torch.tensor(logits)
    labels = torch.tensor(labels)

    # Calculate loss
    eval_loss = F.cross_entropy(logits, labels).item()  

    print(f"eval_loss: {eval_loss}")

    return {
        'accuracy': acc,
        'eval_loss': eval_loss   
    }

best_eval_loss = float('inf')
best_model = None
best_fold = None

for fold in range(5):
    print(f'Fold {fold + 1}')

    # Load train and validation data for the current fold
    train_df = pd.read_csv(f'./data/fold_{fold + 1}_train.csv')
    val_df = pd.read_csv(f'./data/fold_{fold + 1}_val.csv')
    
    # Convert to Huggingface dataset format
    train_dataset = Dataset(pa.Table.from_pandas(train_df))
    val_dataset = Dataset(pa.Table.from_pandas(val_df))

    # Import mental RoBERTa from Hugging Face
    model_name = "mental/mental-roberta-base"
    model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=2)
    tokenizer = AutoTokenizer.from_pretrained(model_name)

    def tokenize_function(examples):
        return tokenizer(examples["text"], padding="max_length", truncation=True)

    tokenized_train = train_dataset.map(tokenize_function, batched=True)
    tokenized_val = val_dataset.map(tokenize_function, batched=True)

    # Set the format for PyTorch
    tokenized_train.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label'])
    tokenized_val.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label'])

    # lora_config = LoraConfig(r=8, lora_alpha=16, lora_dropout=0.1)
    # model = get_peft_model(model, lora_config)

    # Training
    training_args = TrainingArguments(
        output_dir='./results',            
        evaluation_strategy='epoch',       
        per_device_train_batch_size=4,    
        per_device_eval_batch_size=4,      
        num_train_epochs=10,
        learning_rate=3e-5,
        weight_decay=0.01,
        load_best_model_at_end=True,       
        save_total_limit=1,                
        save_strategy="epoch",             
        logging_dir='./logs',              
        logging_steps=10,
        metric_for_best_model="eval_loss", 
    )

    trainer = Trainer(
        model=model,
        args=training_args,
        train_dataset=tokenized_train,
        eval_dataset=tokenized_val,
        compute_metrics=compute_metrics,
    )

    trainer.train()

    # Evaluate the model after training 
    results = trainer.evaluate(eval_dataset=tokenized_val)
    print(f'Results for fold {fold + 1}:', results)`

Here is the error that I'm receiving:

KeyError: "The `metric_for_best_model` training argument is set to 'eval_loss', which is not found in the evaluation metrics. The available evaluation metrics are: ['eval_runtime', 'eval_samples_per_second', 'eval_steps_per_second', 'epoch']. Please ensure that the `compute_metrics` function returns a dictionary that includes 'eval_loss' or consider changing the `metric_for_best_model` via the TrainingArguments."