I was trying the Pacman game with Q-Learning(Reinforcement learning) in java.
However, I could see the game was pausing automatically for a few seconds and then again running. I just wanted to know the reason for this.
Youtube Video link
https://www.youtube.com/watch?v=bmDTAX13so4&feature=youtu.be
Github link for code https://github.com/iamarkaj/pacman-Q_learning
Board.java :
package com.zetcode;
import java.util.Random;
import java.awt.AWTException;
import java.awt.BasicStroke;
import java.awt.Color;
import java.awt.Dimension;
import java.awt.Font;
import java.awt.Graphics;
import java.awt.Graphics2D;
import java.awt.Image;
import java.awt.Robot;
import java.awt.Toolkit;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.awt.event.KeyAdapter;
import java.awt.event.KeyEvent;
import javax.swing.ImageIcon;
import javax.swing.JPanel;
import javax.swing.Timer;
public class Board extends JPanel implements ActionListener {
/**
*
*/
private static final long serialVersionUID = 1L;
private Dimension d;
private final Font smallFont = new Font("Helvetica", Font.BOLD, 14);
private Image ii;
private final Color dotColor = new Color(192, 192, 0);
private Color mazeColor;
private boolean inGame = true;
private boolean dying = false;
private final int BLOCK_SIZE = 24;
private final int N_BLOCKS = 15;
private final int SCREEN_SIZE = N_BLOCKS * BLOCK_SIZE;
private final int PAC_ANIM_DELAY = 2;
private final int PACMAN_ANIM_COUNT = 4;
private final int MAX_GHOSTS = 12;
private final int PACMAN_SPEED = 6;
private int pacAnimCount = PAC_ANIM_DELAY;
private int pacAnimDir = 1;
private int pacmanAnimPos = 0;
private int N_GHOSTS = 6;
private int score;
private int[] dx, dy;
private int[] ghost_x, ghost_y, ghost_dx, ghost_dy, ghostSpeed;
private Image ghost;
private Image pacman1, pacman2up, pacman2left, pacman2right, pacman2down;
private Image pacman3up, pacman3down, pacman3left, pacman3right;
private Image pacman4up, pacman4down, pacman4left, pacman4right;
private int pacman_x, pacman_y, pacmand_x, pacmand_y;
private int req_dx, req_dy, view_dx, view_dy;
private final short levelData[] = {
19, 26, 26, 26, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 22,
21, 0, 0, 0, 17, 16, 16, 16, 16, 16, 16, 16, 16, 16, 20,
21, 0, 0, 0, 17, 16, 16, 16, 16, 16, 16, 16, 16, 16, 20,
21, 0, 0, 0, 17, 16, 16, 24, 16, 16, 16, 16, 16, 16, 20,
17, 18, 18, 18, 16, 16, 20, 0, 17, 16, 16, 16, 16, 16, 20,
17, 16, 16, 16, 16, 16, 20, 0, 17, 16, 16, 16, 16, 24, 20,
25, 16, 16, 16, 24, 24, 28, 0, 25, 24, 24, 16, 20, 0, 21,
1, 17, 16, 20, 0, 0, 0, 0, 0, 0, 0, 17, 20, 0, 21,
1, 17, 16, 16, 18, 18, 22, 0, 19, 18, 18, 16, 20, 0, 21,
1, 17, 16, 16, 16, 16, 20, 0, 17, 16, 16, 16, 20, 0, 21,
1, 17, 16, 16, 16, 16, 20, 0, 17, 16, 16, 16, 20, 0, 21,
1, 17, 16, 16, 16, 16, 16, 18, 16, 16, 16, 16, 20, 0, 21,
1, 17, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 20, 0, 21,
1, 25, 24, 24, 24, 24, 24, 24, 24, 24, 16, 16, 16, 18, 20,
9, 8, 8, 8, 8, 8, 8, 8, 8, 8, 25, 24, 24, 24, 28
};
private final int validSpeeds[] = {1, 2, 3, 4, 6, 8};
private final int maxSpeed = 6;
private int currentSpeed = 3;
private short[] screenData;
private Timer timer;
private int[][][] Q = new int[60][60][6];
private int action;
private double lr;
private int step = 0;
private int x_state;
private int y_state;
private int key = 1;
private int seed = 0;
public Board() {
loadImages();
initVariables();
initBoard();
}
private void initBoard() {
addKeyListener(new TAdapter());
setFocusable(true);
setBackground(Color.black);
}
private void initVariables() {
screenData = new short[N_BLOCKS * N_BLOCKS];
mazeColor = new Color(5, 100, 5);
d = new Dimension(400, 400);
ghost_x = new int[MAX_GHOSTS];
ghost_dx = new int[MAX_GHOSTS];
ghost_y = new int[MAX_GHOSTS];
ghost_dy = new int[MAX_GHOSTS];
ghostSpeed = new int[MAX_GHOSTS];
dx = new int[4];
dy = new int[4];
timer = new Timer(40, this);
timer.start();
}
@Override
public void addNotify() {
super.addNotify();
initGame();
}
private void doAnim() {
pacAnimCount--;
if (pacAnimCount <= 0) {
pacAnimCount = PAC_ANIM_DELAY;
pacmanAnimPos = pacmanAnimPos + pacAnimDir;
if (pacmanAnimPos == (PACMAN_ANIM_COUNT - 1) || pacmanAnimPos == 0) {
pacAnimDir = -pacAnimDir;
}
}
}
private void playGame(Graphics2D g2d) {
if (dying) {
death();
} else {
movePacman();
drawPacman(g2d);
moveGhosts(g2d);
checkMaze();
}
}
private void drawScore(Graphics2D g) {
String s;
g.setFont(smallFont);
g.setColor(new Color(96, 128, 255));
s = "Score: " + score;
g.drawString(s, SCREEN_SIZE / 2 + 96, SCREEN_SIZE + 16);
}
private void checkMaze() {
short i = 0;
boolean finished = true;
while (i < N_BLOCKS * N_BLOCKS && finished) {
if ((screenData[i] & 48) != 0) {
finished = false;
}
i++;
}
if (finished) {
score += 50;
if (N_GHOSTS < MAX_GHOSTS) {
N_GHOSTS++;
}
if (currentSpeed < maxSpeed) {
currentSpeed++;
}
initLevel();
}
}
private void death() {
inGame = true;
initLevel();
}
private void moveGhosts(Graphics2D g2d) {
short i;
int pos;
int count;
for (i = 0; i < N_GHOSTS; i++) {
if (ghost_x[i] % BLOCK_SIZE == 0 && ghost_y[i] % BLOCK_SIZE == 0) {
pos = ghost_x[i] / BLOCK_SIZE + N_BLOCKS * (int) (ghost_y[i] / BLOCK_SIZE);
count = 0;
if ((screenData[pos] & 1) == 0 && ghost_dx[i] != 1) {
dx[count] = -1;
dy[count] = 0;
count++;
}
if ((screenData[pos] & 2) == 0 && ghost_dy[i] != 1) {
dx[count] = 0;
dy[count] = -1;
count++;
}
if ((screenData[pos] & 4) == 0 && ghost_dx[i] != -1) {
dx[count] = 1;
dy[count] = 0;
count++;
}
if ((screenData[pos] & 8) == 0 && ghost_dy[i] != -1) {
dx[count] = 0;
dy[count] = 1;
count++;
}
if (count == 0) {
if ((screenData[pos] & 15) == 15) {
ghost_dx[i] = 0;
ghost_dy[i] = 0;
} else {
ghost_dx[i] = -ghost_dx[i];
ghost_dy[i] = -ghost_dy[i];
}
} else {
Random rand = new Random(seed);
seed += 1;
double prob2 = rand.nextInt(10)+2;
double prob2_ = 1/prob2;
count = (int) (prob2_ * count);
if (count > 3) {
count = 3;
}
ghost_dx[i] = dx[count];
ghost_dy[i] = dy[count];
}
}
ghost_x[i] = ghost_x[i] + (ghost_dx[i] * ghostSpeed[i]);
ghost_y[i] = ghost_y[i] + (ghost_dy[i] * ghostSpeed[i]);
drawGhost(g2d, ghost_x[i] + 1, ghost_y[i] + 1);
if (pacman_x > (ghost_x[i] - 12) && pacman_x < (ghost_x[i] + 12)
&& pacman_y > (ghost_y[i] - 12) && pacman_y < (ghost_y[i] + 12)
&& inGame) {
dying = true;
}
}
}
private void drawGhost(Graphics2D g2d, int x, int y) {
g2d.drawImage(ghost, x, y, this);
}
private void movePacman() {
int pos;
short ch;
if (req_dx == -pacmand_x && req_dy == -pacmand_y) {
pacmand_x = req_dx;
pacmand_y = req_dy;
view_dx = pacmand_x;
view_dy = pacmand_y;
}
if (pacman_x % BLOCK_SIZE == 0 && pacman_y % BLOCK_SIZE == 0) {
pos = pacman_x / BLOCK_SIZE + N_BLOCKS * (int) (pacman_y / BLOCK_SIZE);
ch = screenData[pos];
if ((ch & 16) != 0) {
screenData[pos] = (short) (ch & 15);
score++;
}
if (req_dx != 0 || req_dy != 0) {
if (!((req_dx == -1 && req_dy == 0 && (ch & 1) != 0)
|| (req_dx == 1 && req_dy == 0 && (ch & 4) != 0)
|| (req_dx == 0 && req_dy == -1 && (ch & 2) != 0)
|| (req_dx == 0 && req_dy == 1 && (ch & 8) != 0))) {
pacmand_x = req_dx;
pacmand_y = req_dy;
view_dx = pacmand_x;
view_dy = pacmand_y;
}
}
// Check for standstill
if ((pacmand_x == -1 && pacmand_y == 0 && (ch & 1) != 0)
|| (pacmand_x == 1 && pacmand_y == 0 && (ch & 4) != 0)
|| (pacmand_x == 0 && pacmand_y == -1 && (ch & 2) != 0)
|| (pacmand_x == 0 && pacmand_y == 1 && (ch & 8) != 0)) {
pacmand_x = 0;
pacmand_y = 0;
}
}
pacman_x = pacman_x + PACMAN_SPEED * pacmand_x;
pacman_y = pacman_y + PACMAN_SPEED * pacmand_y;
x_state = pacman_x*60/380;
y_state = pacman_y*60/420;
step += 1;
if (step % 500 == 0) {
System.out.println(step);
}
lr = Math.pow(0.85, (step/100));
Q[x_state][y_state][action] += lr * (score + Q[x_state][y_state][action]);
}
private void drawPacman(Graphics2D g2d) {
if (view_dx == -1) {
drawPacmanLeft(g2d);
} else if (view_dx == 1) {
drawPacmanRight(g2d);
} else if (view_dy == -1) {
drawPacmanUp(g2d);
} else {
drawPacmanDown(g2d);
}
}
private void drawPacmanUp(Graphics2D g2d) {
switch (pacmanAnimPos) {
case 1:
g2d.drawImage(pacman2up, pacman_x + 1, pacman_y + 1, this);
break;
case 2: