Change vertices of a plane by rotating

I have a plan on the screen and I want to rotate it by 5 degrees, without using glRotate , I have the code:

#include <GL/glut.h>
#include <math.h>
#include <stdio.h>

#define HEIGHT 500
#define WIDTH 500
#define POINTS 5

#define TRANSLATE 5
#define ROTATION 5.0
#define SCALE_M 0.5
#define SCALE_P 1.5

void display();
void specialKeys();

typedef struct {
  float x;
  float y;
  float z;
} Dot;

Dot dots[5];
int i, translate = 0, rotate = 0, scale = 0;
char typeRotate = '
for (i = 0; i < POINTS; i++) {
    dots[i].x = (dots[i].x * cos(-ROTATION)) - (dots[i].y * sin(-ROTATION));
    dots[i].y = (dots[i].x * sin(-ROTATION)) + (dots[i].y * cos(-ROTATION));
}
';

Dot point(float x, float y, float z) {
  Dot p;
  p.x = x;
  p.y = y;
  p.z = z;

  return p;
}

void createPoints() {
  dots[0] = point(00, 00, 00);
  dots[1] = point(20, 00, 00);
  dots[2] = point(20, 20, 00);
  dots[3] = point(00, 20, 00);
}

void drawDot(float x, float y, float z) { glVertex3f(x / 30, y / 30, z / 30); }

void display() {
  glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
  glLoadIdentity();

  for (i = 0; i < POINTS; i++) {
    dots[i].x = (dots[i].x * cos(-ROTATION)) - (dots[i].y * sin(-ROTATION));
    dots[i].y = (dots[i].x * sin(-ROTATION)) + (dots[i].y * cos(-ROTATION));
  }

  glBegin(GL_POLYGON);
  glColor3f(1.0, 1.0, 0.0);
  for (i = 0; i < POINTS; i++)
    drawDot(dots[i].x, dots[i].y, dots[i].z);

  glEnd();

  glFlush();
  glutSwapBuffers();
}

int main(int argc, char *argv[]) {
  createPoints();
  glutInit(&argc, argv);
  glutInitDisplayMode(GLUT_RGB | GLUT_DEPTH);
  glutInitWindowSize(WIDTH, HEIGHT);
  glutInitWindowPosition(150, 150);
  glutCreateWindow("Exercicio Lista");
  glEnable(GL_DEPTH_TEST);
  glutDisplayFunc(display);
  glutMainLoop();

  return 0;
}

The part on which I try to rotate the plane is:

gcc -o lista lista.c -lm -lglut -lGL -lGLU && ./lista

Where do I try to rotate the points by following a formula I saw in US Geometrical Transformations . But the result was not a 5-degree rotation on the Z axis, you can see when running the file.

Q.: Compilation is:

#include <GL/glut.h>
#include <math.h>
#include <stdio.h>

#define HEIGHT 500
#define WIDTH 500
#define POINTS 5

#define TRANSLATE 5
#define ROTATION 5.0
#define SCALE_M 0.5
#define SCALE_P 1.5

void display();
void specialKeys();

typedef struct {
  float x;
  float y;
  float z;
} Dot;

Dot dots[5];
int i, translate = 0, rotate = 0, scale = 0;
char typeRotate = '
for (i = 0; i < POINTS; i++) {
    dots[i].x = (dots[i].x * cos(-ROTATION)) - (dots[i].y * sin(-ROTATION));
    dots[i].y = (dots[i].x * sin(-ROTATION)) + (dots[i].y * cos(-ROTATION));
}
';

Dot point(float x, float y, float z) {
  Dot p;
  p.x = x;
  p.y = y;
  p.z = z;

  return p;
}

void createPoints() {
  dots[0] = point(00, 00, 00);
  dots[1] = point(20, 00, 00);
  dots[2] = point(20, 20, 00);
  dots[3] = point(00, 20, 00);
}

void drawDot(float x, float y, float z) { glVertex3f(x / 30, y / 30, z / 30); }

void display() {
  glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
  glLoadIdentity();

  for (i = 0; i < POINTS; i++) {
    dots[i].x = (dots[i].x * cos(-ROTATION)) - (dots[i].y * sin(-ROTATION));
    dots[i].y = (dots[i].x * sin(-ROTATION)) + (dots[i].y * cos(-ROTATION));
  }

  glBegin(GL_POLYGON);
  glColor3f(1.0, 1.0, 0.0);
  for (i = 0; i < POINTS; i++)
    drawDot(dots[i].x, dots[i].y, dots[i].z);

  glEnd();

  glFlush();
  glutSwapBuffers();
}

int main(int argc, char *argv[]) {
  createPoints();
  glutInit(&argc, argv);
  glutInitDisplayMode(GLUT_RGB | GLUT_DEPTH);
  glutInitWindowSize(WIDTH, HEIGHT);
  glutInitWindowPosition(150, 150);
  glutCreateWindow("Exercicio Lista");
  glEnable(GL_DEPTH_TEST);
  glutDisplayFunc(display);
  glutMainLoop();

  return 0;
}

I hope they help you figure out the error in the calculation, I did not try to do with the PDF arrays because I did not understand.