强化学习入门：Q-Learning与DQN实战避坑

1. 一个真实场景：我让AI玩《超级马里奥》

2023年，我接手一个项目：用强化学习训练AI玩《超级马里奥》。第一个版本用Q-Learning，状态空间离散化后维度爆炸（256×256×3像素 → 196608维），Q表根本存不下。训练3天，AI连第一个坑都跳不过去。后来换成DQN，用卷积网络直接处理原始像素，48小时训练后AI能通关第一关。

这个经历让我明白：选对算法比调参重要100倍。本文用CartPole和LunarLander两个经典环境，手把手带你跑通Q-Learning和DQN，并填上那些坑。

2. 问题：离散状态 vs 连续状态，Q表存不下怎么办？

强化学习核心是学习状态-动作价值函数Q(s,a)。Q-Learning用表格存储每个(s,a)对的价值，状态必须离散。但现实世界状态是连续的：位置、速度、角度、像素值……

两个方案：

• 方案A：Q-Learning + 状态离散化 —— 把连续状态切成桶（bins），存Q表
• 方案B：DQN（Deep Q-Network） —— 用神经网络近似Q函数，直接处理连续状态

3. 方案对比：Q-Learning vs DQN

我的建议：状态维度≤4且离散值≤20时用Q-Learning，否则直接上DQN。

4. 完整代码实现

4.1 环境准备

Python 3.10.12，Gymnasium 0.29.1，PyTorch 2.1.0

pip install gymnasium[classic-control] torch numpy matplotlib

4.2 Q-Learning实现（CartPole-v1）

CartPole状态4维：位置、速度、角度、角速度。每个维度离散化为6个桶，状态空间=6⁴=1296，动作空间=2（左/右）。

import gymnasium as gym
import numpy as np

env = gym.make('CartPole-v1')
n_bins = 6
n_actions = env.action_space.n

# 状态边界（从环境观察空间获取）
obs_low = env.observation_space.low
obs_high = env.observation_space.high
# 角度边界更宽，防止截断
obs_low[1] = -0.5
obs_high[1] = 0.5
obs_low[3] = -np.radians(50)
obs_high[3] = np.radians(50)

def discretize_state(state):
    """将连续状态离散化为整数索引"""
    ratios = (state - obs_low) / (obs_high - obs_low)
    ratios = np.clip(ratios, 0, 1)
    indices = (ratios * (n_bins - 1)).astype(int)
    return tuple(indices)

# 初始化Q表
q_table = np.zeros([n_bins] * 4 + [n_actions])

# 超参数
alpha = 0.1      # 学习率
gamma = 0.99     # 折扣因子
epsilon = 1.0    # 探索率
epsilon_min = 0.01
epsilon_decay = 0.995
episodes = 1000

rewards = []
for ep in range(episodes):
    state, _ = env.reset()
    state = discretize_state(state)
    total_reward = 0
    done = False
    while not done:
        # ε-贪心策略
        if np.random.random() < epsilon:
            action = env.action_space.sample()
        else:
            action = np.argmax(q_table[state])
        
        next_state, reward, terminated, truncated, _ = env.step(action)
        done = terminated or truncated
        next_state = discretize_state(next_state)
        
        # Q-Learning更新公式
        best_next = np.max(q_table[next_state])
        td_target = reward + gamma * best_next * (1 - done)
        td_error = td_target - q_table[state][action]
        q_table[state][action] += alpha * td_error
        
        state = next_state
        total_reward += reward
    
    rewards.append(total_reward)
    epsilon = max(epsilon_min, epsilon * epsilon_decay)
    
    if (ep+1) % 100 == 0:
        avg_reward = np.mean(rewards[-100:])
        print(f"Episode {ep+1}, Avg Reward: {avg_reward:.2f}, Epsilon: {epsilon:.3f}")

env.close()
print("Q-Learning训练完成")

4.3 DQN实现（LunarLander-v2）

LunarLander状态8维连续，动作4个。用3层全连接网络。

import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np
from collections import deque
import random
import gymnasium as gym

# 设备
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"Using device: {device}")

# 神经网络：双隐层，256神经元
class DQN(nn.Module):
    def __init__(self, state_dim, action_dim, hidden_dim=256):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(state_dim, hidden_dim),
            nn.ReLU(),
            nn.Linear(hidden_dim, hidden_dim),
            nn.ReLU(),
            nn.Linear(hidden_dim, action_dim)
        )
    
    def forward(self, x):
        return self.net(x)

# 经验回放缓冲区
class ReplayBuffer:
    def __init__(self, capacity=100000):
        self.buffer = deque(maxlen=capacity)
    
    def push(self, state, action, reward, next_state, done):
        self.buffer.append((state, action, reward, next_state, done))
    
    def sample(self, batch_size):
        batch = random.sample(self.buffer, batch_size)
        states, actions, rewards, next_states, dones = zip(*batch)
        return (np.array(states), np.array(actions), np.array(rewards, dtype=np.float32),
                np.array(next_states), np.array(dones, dtype=np.float32))
    
    def __len__(self):
        return len(self.buffer)

# 超参数
env = gym.make('LunarLander-v2')
state_dim = env.observation_space.shape[0]
action_dim = env.action_space.n

policy_net = DQN(state_dim, action_dim).to(device)
target_net = DQN(state_dim, action_dim).to(device)
target_net.load_state_dict(policy_net.state_dict())
target_net.eval()

optimizer = optim.Adam(policy_net.parameters(), lr=0.001)
buffer = ReplayBuffer(100000)

batch_size = 64
gamma = 0.99
epsilon = 1.0
epsilon_min = 0.01
epsilon_decay = 0.995
target_update = 100  # 每100步更新目标网络
episodes = 500

rewards = []
step_count = 0

for ep in range(episodes):
    state, _ = env.reset()
    state = np.array(state, dtype=np.float32)
    total_reward = 0
    done = False
    
    while not done:
        # ε-贪心
        if np.random.random() < epsilon:
            action = env.action_space.sample()
        else:
            with torch.no_grad():
                state_tensor = torch.FloatTensor(state).unsqueeze(0).to(device)
                q_values = policy_net(state_tensor)
                action = q_values.argmax().item()
        
        next_state, reward, terminated, truncated, _ = env.step(action)
        done = terminated or truncated
        next_state = np.array(next_state, dtype=np.float32)
        
        buffer.push(state, action, reward, next_state, done)
        state = next_state
        total_reward += reward
        step_count += 1
        
        # 训练
        if len(buffer) >= batch_size:
            states, actions, rewards_b, next_states, dones = buffer.sample(batch_size)
            
            states = torch.FloatTensor(states).to(device)
            actions = torch.LongTensor(actions).unsqueeze(1).to(device)
            rewards_b = torch.FloatTensor(rewards_b).unsqueeze(1).to(device)
            next_states = torch.FloatTensor(next_states).to(device)
            dones = torch.FloatTensor(dones).unsqueeze(1).to(device)
            
            # 当前Q值
            current_q = policy_net(states).gather(1, actions)
            
            # 目标Q值（用目标网络）
            with torch.no_grad():
                next_q = target_net(next_states).max(1, keepdim=True)[0]
                target_q = rewards_b + gamma * next_q * (1 - dones)
            
            loss = nn.MSELoss()(current_q, target_q)
            
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            
            # 软更新目标网络（每target_update步）
            if step_count % target_update == 0:
                target_net.load_state_dict(policy_net.state_dict())
        
        if done:
            break
    
    rewards.append(total_reward)
    epsilon = max(epsilon_min, epsilon * epsilon_decay)
    
    if (ep+1) % 50 == 0:
        avg_reward = np.mean(rewards[-50:])
        print(f"Episode {ep+1}, Avg Reward: {avg_reward:.2f}, Epsilon: {epsilon:.3f}")

env.close()
print("DQN训练完成")

5. 效果数据

测试环境：Intel i7-12700H，32GB RAM，RTX 3060 Laptop GPU。每个算法运行5次取平均。

Q-Learning在CartPole上表现很好，因为状态空间小。DQN在LunarLander上需要更多调参，但能处理连续状态。

6. 避坑指南

坑1：Q-Learning状态离散化导致信息丢失

我踩过：把角度离散成6个桶，结果AI学不会平衡。因为角度精度不够，细微差异被忽略。解决方案：增加桶数到10以上，或使用自适应分桶（如K-means聚类）。

# 自适应分桶示例：用K-means聚类状态
from sklearn.cluster import KMeans
# 收集大量状态样本
states_collected = []
for _ in range(1000):
    state, _ = env.reset()
    states_collected.append(state)
kmeans = KMeans(n_clusters=50)
kmeans.fit(states_collected)
# 用聚类中心作为离散状态
def discretize_state(state):
    return tuple(kmeans.predict([state])[0])

坑2：DQN过估计（Overestimation）

标准DQN用max操作选择动作，会导致Q值被高估。我遇到过训练时Q值飙到1000+，但实际奖励只有200。解决方案：用Double DQN，用当前网络选动作，目标网络算Q值。

# Double DQN目标计算
with torch.no_grad():
    # 当前网络选动作
    next_actions = policy_net(next_states).argmax(1, keepdim=True)
    # 目标网络算Q值
    next_q = target_net(next_states).gather(1, next_actions)
    target_q = rewards_b + gamma * next_q * (1 - dones)

坑3：经验回放缓冲区太小

我一开始设buffer_size=10000，训练到一半发现模型震荡。因为旧经验被覆盖太快，样本多样性不够。解决方案：至少设到100000，LunarLander建议500000。

坑4：学习率太大导致发散

用lr=0.01，训练10轮后loss变成NaN。梯度爆炸了。解决方案：lr从0.001开始，配合梯度裁剪。

# 梯度裁剪
optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm_(policy_net.parameters(), max_norm=1.0)
optimizer.step()

坑5：目标网络更新太频繁

我试过每10步更新目标网络，结果训练不稳定。因为目标网络变化太快，Q值估计方差大。解决方案：每100-1000步更新一次，或软更新（Polyak平均）。

# 软更新：τ=0.005
tau = 0.005
for target_param, policy_param in zip(target_net.parameters(), policy_net.parameters()):
    target_param.data.copy_(tau * policy_param.data + (1 - tau) * target_param.data)

坑6：奖励缩放

LunarLander奖励范围[-100, 100]，直接训练时梯度不稳定。解决方案：奖励归一化到[-1, 1]或使用Huber损失。

# 使用Huber损失代替MSE
loss = nn.SmoothL1Loss()(current_q, target_q)

7. 总结

Q-Learning适合小状态空间，DQN适合高维连续状态。核心要点：

• 状态离散化时注意精度，不够就加桶或用聚类
• DQN必须用经验回放和目标网络，否则不收敛
• Double DQN解决过估计，效果立竿见影
• 超参数：学习率0.001，batch_size 64，buffer_size 100000+
• 奖励缩放和梯度裁剪防止数值问题

代码全部可运行，直接复制到你的环境试试。遇到问题先检查避坑指南。