# Accept the Atari ROM licence — required before first use
AutoROM --accept-license

# Create and activate the environment (one-time setup)
conda env create -f environment.yml
conda activate rl-python

# Accept the Atari ROM licence (required once per environment)
AutoROM --accept-license

import warnings

import ale_py
import cv2
import gymnasium as gym
import matplotlib.pyplot as plt
import minigrid
import mlflow
import numpy as np
import pettingzoo
import pygame
import ray
import sb3_contrib
import shimmy
import stable_baselines3
import supersuit
import tensorboard
import torch
import torchvision
from torch.utils.tensorboard import SummaryWriter

def pprint(statement, max_length=85):
    fmt = int(0.5 * (max_length - len(statement)))
    print(f"{'-'*fmt} {statement} {'-'*fmt}\n")

pprint("All imports successful!")

------------------------------- All imports successful! -------------------------------

# Check if CUDA/MPS are available
if torch.cuda.is_available():
    pprint("CUDA is available.")
elif torch.backends.mps.is_available():
    pprint("MPS is available.")
else:
    pprint("CUDA/MPS are not available.")

----------------------------- CUDA/MPS are not available. -----------------------------

env_warning_flag = False

# Classic control environments
try:
    env = gym.make("CartPole-v1")
    obs, _ = env.reset()
    env.close()
except Exception as e:
    raise RuntimeError("Error installing Gymnasium.") from e

# ALE environments
try:
    gym.register_envs(ale_py)
    env = gym.make("ALE/Breakout-v5")
    obs, _ = env.reset()
    env.close()
except Exception:
    env_warning_flag = True
    warnings.warn("ALE environments are not installed.")

# MuJoco environments
try:
    env = gym.make("Reacher-v5")
    obs, _ = env.reset()
    env.close()
except Exception:
    env_warning_flag = True
    warnings.warn("MuJoco environments are not installed.")

# Box2D environments
try:
    env = gym.make("CarRacing-v3")
    obs, _ = env.reset()
    env.close()
except Exception:
    env_warning_flag = True
    warnings.warn("Box2D environments are not installed.")

# MiniGrid environments
try:
    env = gym.make("MiniGrid-Empty-Random-5x5-v0")
    obs, _ = env.reset()
    env.close()
except Exception:
    env_warning_flag = True
    warnings.warn("MiniGrid environments are not installed.")

if not env_warning_flag:
    pprint("All environments are working correctly!")

----------------------- All environments are working correctly! -----------------------

A.L.E: Arcade Learning Environment (version 0.11.2+ecc1138)
[Powered by Stella]

import gymnasium as gym
import matplotlib.pyplot as plt
from IPython import display

env = gym.make("FrozenLake-v1", is_slippery=True, render_mode="rgb_array")

print("Environment:", env.spec.id)
print()
print("Observation space:", env.observation_space)
print("n (grid cells):", env.observation_space.n)
print("Interpretation: integer 0-15 representing position on the 4x4 grid")
print()
print("Action space:", env.action_space)
print("n (actions):", env.action_space.n)
print("Meanings: 0 = left, 1 = down, 2 = right, 3 = up")

Environment: FrozenLake-v1

Observation space: Discrete(16)
n (grid cells): 16
Interpretation: integer 0-15 representing position on the 4x4 grid

Action space: Discrete(4)
n (actions): 4
Meanings: 0 = left, 1 = down, 2 = right, 3 = up

obs, info = env.reset(seed=42)
print(f"Initial observation (grid cell index): {obs}")

frame = env.render()

plt.imshow(frame)
plt.axis('off')

Initial observation (grid cell index): 0

(np.float64(-0.5), np.float64(255.5), np.float64(255.5), np.float64(-0.5))

obs, reward, terminated, truncated, info = env.step(action)
done = terminated or truncated

def display_env(env):
    frame = env.render()
    plt.imshow(frame)
    plt.axis("off")
    plt.show()


obs, info = env.reset(seed=0)
print("Step-by-step episode (first 6 steps shown)\n")
display_env(env)

for step in range(6):
    action = env.action_space.sample()
    action_name = ["left", "down", "right", "up"][action]
    obs, reward, terminated, truncated, info = env.step(action)
    print(f"Step {step + 1}: action={action_name}, cell={obs}, reward={reward}, "
          f"terminated={terminated}, prob={info['prob']:.2f}")
    display_env(env)
    if terminated or truncated:
        print("Episode over.")
        break

env.close()

Step-by-step episode (first 6 steps shown)

Step 1: action=up, cell=1, reward=0, terminated=False, prob=0.33

Step 2: action=up, cell=2, reward=0, terminated=False, prob=0.33

Step 3: action=right, cell=6, reward=0, terminated=False, prob=0.33

Step 4: action=up, cell=5, reward=0, terminated=True, prob=0.33

Episode over.

def run_episodes(env, n=1000):
    """Return the fraction of episodes that reached the goal."""
    successes = 0
    for ep in range(n):
        obs, _ = env.reset(seed=ep)
        done = False
        while not done:
            obs, reward, terminated, truncated, _ = env.step(env.action_space.sample())
            done = terminated or truncated
        if reward == 1.0:
            successes += 1
    return successes / n

slippery = gym.make("FrozenLake-v1", is_slippery=True)
deterministic = gym.make("FrozenLake-v1", is_slippery=False)

slip_rate = run_episodes(slippery)
det_rate = run_episodes(deterministic)

slippery.close()
deterministic.close()

print(f"Random policy success rate - slippery: {slip_rate:.1%}")
print(f"Random policy success rate - deterministic: {det_rate:.1%}")

Random policy success rate - slippery: 1.5%
Random policy success rate - deterministic: 1.2%

Package	Purpose
PyTorch	Deep learning backend for neural-network policies
Gymnasium	Standard single-agent RL environment API
ale-py	Atari 2600 game environments (see legal notice above)
Pygame	Rendering backend used by many environments
PettingZoo	Multi-agent RL environments
Stable-Baselines3	Production-quality single-agent RL algorithms
SB3-Contrib	Extra algorithms for Stable-Baselines3 (TQC, TRPO, MaskablePPO, …)
Ray RLlib	Scalable, distributed RL framework
MLflow	Experiment tracking, model registry, and run comparison
SuperSuit	Environment wrappers for PettingZoo + SB3
Shimmy	Gymnasium compatibility shims
TensorBoard	Training metrics visualisation

`render_mode`	Effect
`None` (default)	No rendering - fastest, use for training
`"human"`	Opens a window - use for interactive demos
`"ansi"`	Returns a text string - useful for grid worlds in notebooks
`"rgb_array"`	Returns pixel arrays - use for video recording

Return value	Type	Meaning
`obs`	int	New grid cell index (0–15)
`reward`	float	`1.0` if goal reached, `0.0` otherwise
`terminated`	bool	`True` when the agent reaches the goal or falls in a hole
`truncated`	bool	`True` when the episode time limit is hit
`info`	dict	`{"prob": p}` — transition probability of the step that was taken

00 - Getting Started¶

Legal Notice - Atari ROM License¶

What the licence allows¶

What the licence prohibits¶

Accepting the licence¶

1. Packages¶

2. Installation¶

Option A - Conda (recommended for local development)¶

Option B - pip (virtualenv / system Python)¶

Option C - Docker (reproducible, isolated, no host dependencies)¶

3. Installation Checks¶

4. Gymnasium Tutorial¶

4.1 Creating an environment¶

4.2 Resetting an episode¶

4.3 The step loop¶

4.4 Slippery vs deterministic¶