I'm currently learning machine learning and have moved from basic models and neural networks to reinforcement learning (RL). After watching a tutorial by Nicholas Renotte (link here), I tried to follow along with the code, but I'm encountering a few issues due to the tutorial being a bit outdated i guess. Specifically, some parts of the code are causing bugs that I cannot resolve.

Before posting this, I attempted to solve the problem by searching for answers, including referencing this StackOverflow question: Keras symbolic inputs/outputs do not implement __len__, but I still couldn't resolve the issue.

I am using the following versions of the libraries:

• TensorFlow: 2.10.1
• Keras-RL: 0.4.2

Here is the code I'm working with:

import numpy as np
import random
import pygame
import gym
from rl.memory import SequentialMemory
from rl.policy import BoltzmannQPolicy
from rl.agents.dqn import DQNAgent
from keras.layers import Dense, Flatten
import tensorflow as tf

env = gym.make("CartPole-v1", render_mode="rgb_array")
states = env.observation_space.shape[0]
actions = env.action_space.n

def build_model(states, actions):
    model = tf.keras.Sequential()
    model.add(Dense(24, activation='relu', input_shape=(states,)))
    model.add(Dense(24, activation='relu'))
    model.add(Dense(actions, activation='linear'))
    model.build((None, states))
    return model

def buildAgent(model, actions):
    policy = BoltzmannQPolicy()
    memory = SequentialMemory(limit=50000, window_length=1)
    dqn = DQNAgent(model, memory=memory, policy=policy, nb_actions=actions, nb_steps_warmup=10,
                   target_model_update=1e-2)
    return dqn

model = build_model(states, actions)

DQN = buildAgent(model, actions)

DQNpile(tf.keras.optimizers.Adam(learning_rate=1e-3), metrics=['mae'])
DQN.fit(env, nb_steps=50000, visualize=False, verbose=1)
scores = DQN.test(env, nb_episodes=100, visualize=True)
print(np.mean(scores.history['episode_reward']))
model.save('model.h5')

Here is my error message:

{
    "name": "TypeError",
    "message": "Keras symbolic inputs/outputs do not implement `__len__`. You may be trying to pass Keras symbolic inputs/outputs to a TF API that does not register dispatching, preventing Keras from automatically converting the API call to a lambda layer in the Functional Model. This error will also get raised if you try asserting a symbolic input/output directly.",
    "stack": "---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
~\\AppData\\Local\\Temp\\ipykernel_9048\\467332075.py in <module>
     30 model = build_model(states, actions)
     31 
---> 32 DQN = buildAgent(model, actions)
     33 
     34 DQNpile(tf.keras.optimizers.Adam(learning_rate=1e-3), metrics=['mae'])

~\\AppData\\Local\\Temp\\ipykernel_9048\\467332075.py in buildAgent(model, actions)
     25     memory = SequentialMemory(limit=50000, window_length=1)
     26     dqn = DQNAgent(model, memory=memory, policy=policy, nb_actions=actions, nb_steps_warmup=10,
---> 27                    target_model_update=1e-2)
     28     return dqn
     29 

c:\\Users\\Cyril\\miniconda3\\envs\\dsl\\lib\\site-packages\\rl\\agents\\dqn.py in __init__(self, model, policy, test_policy, enable_double_dqn, enable_dueling_network, dueling_type, *args, **kwargs)
    106 
    107         # Validate (important) input.
--> 108         if hasattr(model.output, '__len__') and len(model.output) > 1:
    109             raise ValueError('Model \"{}\" has more than one output. DQN expects a model that has a single output.'.format(model))
    110         if model.output._keras_shape != (None, self.nb_actions):

c:\\Users\\Cyril\\miniconda3\\envs\\dsl\\lib\\site-packages\\keras\\engine\\keras_tensor.py in __len__(self)
    243     def __len__(self):
    244         raise TypeError(
--> 245             \"Keras symbolic inputs/outputs do not \"
    246             \"implement `__len__`. You may be \"
    247             \"trying to pass Keras symbolic inputs/outputs \"

TypeError: Keras symbolic inputs/outputs do not implement `__len__`. You may be trying to pass Keras symbolic inputs/outputs to a TF API that does not register dispatching, preventing Keras from automatically converting the API call to a lambda layer in the Functional Model. This error will also get raised if you try asserting a symbolic input/output directly."
}