01 - 什么是强化学习¶

import numpy as np
import gymnasium as gym

# 创建环境
env = gym.make('CartPole-v1')

# 查看环境信息
print(f"状态空间: {env.observation_space}")
print(f"动作空间: {env.action_space}")

# 随机策略的完整episode
def random_policy_episode():
    state, info = env.reset()
    total_reward = 0
    done = False

    while not done:
        # 随机选择动作
        action = env.action_space.sample()

        # 执行动作
        next_state, reward, terminated, truncated, info = env.step(action)
        done = terminated or truncated

        total_reward += reward
        state = next_state

    return total_reward

# 测试随机策略
rewards = [random_policy_episode() for _ in range(100)]
print(f"随机策略平均奖励: {np.mean(rewards):.2f}")

env.close()

import numpy as np
import random

class SimpleGridWorld:
    """
    简单的网格世界环境

    4x4网格：
    S . . .
    . . . .
    . . . .
    . . . G

    S = 起点 (0,0)
    G = 终点 (3,3)，到达后获得奖励+1
    其他格子奖励为0
    """

    def __init__(self, size=4):
        self.size = size
        self.start_pos = (0, 0)
        self.goal_pos = (size-1, size-1)
        self.reset()

    def reset(self):
        """重置环境，返回初始状态和info"""
        self.agent_pos = self.start_pos
        return self._get_state(), {}

    def _get_state(self):
        """将位置转换为状态编号"""
        return self.agent_pos[0] * self.size + self.agent_pos[1]

    def step(self, action):
        """
        执行动作，返回 (next_state, reward, terminated, truncated, info)

        动作: 0=上, 1=下, 2=左, 3=右
        """
        x, y = self.agent_pos

        # 根据动作移动
        if action == 0:    # 上
            x = max(0, x - 1)
        elif action == 1:  # 下
            x = min(self.size - 1, x + 1)
        elif action == 2:  # 左
            y = max(0, y - 1)
        elif action == 3:  # 右
            y = min(self.size - 1, y + 1)

        self.agent_pos = (x, y)

        # 计算奖励
        if self.agent_pos == self.goal_pos:
            reward = 1.0
            done = True
        else:
            reward = 0.0
            done = False

        return self._get_state(), reward, done, False, {}

    def render(self):
        """可视化当前状态"""
        grid = np.full((self.size, self.size), '.')
        grid[self.goal_pos] = 'G'
        grid[self.agent_pos] = 'A'
        print('\n'.join([' '.join(row) for row in grid]))
        print()

# 测试环境
env = SimpleGridWorld(size=4)
print("初始状态:")
env.render()

# 随机策略运行一个episode
state, _ = env.reset()
total_reward = 0
done = False
steps = 0

print("随机策略运行:")
while not done and steps < 20:
    action = random.randint(0, 3)  # 随机选择动作
    next_state, reward, terminated, truncated, _ = env.step(action)
    done = terminated or truncated
    total_reward += reward
    steps += 1

    action_names = ['上', '下', '左', '右']
    print(f"步骤 {steps}: 动作={action_names[action]}, 奖励={reward}")
    env.render()

print(f"总奖励: {total_reward}, 总步数: {steps}")

def greedy_policy(state, env):
    """
    贪婪策略：总是向终点方向移动
    """
    x, y = env.agent_pos
    goal_x, goal_y = env.goal_pos

    # 计算应该移动的方向
    if x < goal_x:
        return 1  # 下
    elif x > goal_x:
        return 0  # 上
    elif y < goal_y:
        return 3  # 右
    else:
        return 2  # 左

# 测试贪婪策略
env = SimpleGridWorld(size=4)
state, _ = env.reset()
total_reward = 0
done = False
steps = 0

print("贪婪策略运行:")
while not done and steps < 20:
    action = greedy_policy(state, env)
    next_state, reward, terminated, truncated, _ = env.step(action)
    done = terminated or truncated
    total_reward += reward
    steps += 1

    action_names = ['上', '下', '左', '右']
    print(f"步骤 {steps}: 动作={action_names[action]}")
    env.render()

print(f"贪婪策略总步数: {steps}")

class GridWorldWithObstacles(SimpleGridWorld):
    def __init__(self, size=4):
        super().__init__(size)  # super()调用父类方法
        self.obstacles = [(1, 1), (1, 2), (2, 1)]

    def step(self, action):
        next_state, reward, done, truncated, info = super().step(action)

        # 检查是否碰到障碍物
        if self.agent_pos in self.obstacles:
            reward = -1.0
            self.agent_pos = self.start_pos  # 回到起点
            done = False

        return self._get_state(), reward, done, False, {}

    def render(self):
        """可视化（包含障碍物）"""
        grid = np.full((self.size, self.size), '.')
        for obs in self.obstacles:
            grid[obs] = 'X'
        grid[self.goal_pos] = 'G'
        grid[self.agent_pos] = 'A'
        print('\n'.join([' '.join(row) for row in grid]))
        print()