Message format websocket (Arduino + Esp8266)

Question

Update Almost there I can receive messages I think. When the code is readable, I will put it in. Trying to send also..

Original question

I'm trying to connect my esp8266 (@38400 baud) ($3.50 wifi chip :)), to a Websocket. The chip is connected with a Arduino pro mini. This setup is OK and it works.

I am able to do a handshake, thanks to some code (https://github.com/ejeklint/ArduinoWebsocketServer).

So this is what the program has to do:

• Handle handshake V
• Receive message Received some unknown chars
• Sending (when I'm able to receive, I will find out how to send)

I'm testing websocket with: http://www.websocket.org/echo.html

connecting with my wifi module ws://192.168.1.104:8000

When I send 3 x the message "aaaa" to my Arduino I receive this:

+IPD,0,10: | | | q | | b | k | | c | | |

+IPD,0,10: | | | ¦ | ¡ | 0 | P | Ç | À | Q | 1 |

+IPD,0,10: | | | _ | ò | ± | ? | > | | Ð | ^ | |

How can I decode this?

#include "sha1.h"
#include "Base64.h"
#include <SoftwareSerial.h>
#include <MemoryFree.h>
SoftwareSerial debug(8, 9); // RX, TX

void setup() {
  Serial.begin(38400);
  debug.begin(38400);
  delay(50);
  debug.println("start");
  Serial.println("AT+RST");
  delay(5000);
  Serial.println("AT+CWMODE=1");  // NO CHANGE
  delay(1500);
  Serial.find("OK");
  Serial.println("AT+CIPMUX=1");
  Serial.find("OK");
  delay(3000);
  Serial.println("AT+CIPSERVER=1,8000");
  boolean server = Serial.find("OK");
  delay(3000);
  Serial.println("AT+CIFSR");   // Display the ip please
  boolean r = readLines(4);
  debug.println("eind setup");
  debug.println(server);
  boolean found = false;

  while(!found)   // wait for the link
    found = Serial.find("Link");
  debug.println("link builded, end setup");
}


void loop() {
  String key = "";
  boolean isKey = Serial.find("Key: ");
  if(isKey) {
    debug.println("Key found!");
      while(true) {
        if(Serial.available()) {
          char c = (char)Serial.read();
          if(c == '=') {
            doHandshake(key + "==");
            key = "";
            break;
          } 
          if(c != '\r' || c != '\n') {
            key = key + c;
          }
        }
      }
// _________________________ PROBLEMO ____________________________________
while(true) {  // So far so good. Handshake done Now wait for the message
         if(Serial.available()) {
          char c = (char)Serial.read();
          debug.print(c);
          debug.print(" | ");
         }
      }
  }
// _________________________ /PROBLEMO ____________________________________
}

boolean readLines(int lines) {
  boolean found = false;
  int count = 0;
  while(count < lines) {
    if(Serial.available()) {
      char c = (char)Serial.read();
      if(c != '\r') {
        debug.write(c);
      } else {
        count++;
      } 
    }
  }
  return true;
}

bool doHandshake(String k) {
    debug.println("do handshake: " + k);
    char bite;
    char temp[128];
    char key[80];
    memset(temp, '\0', sizeof(temp));
    memset(key, '\0', sizeof(key));

    byte counter = 0;
    int myCo = 0;
    while ((bite = k.charAt(myCo++)) != 0) {
      key[counter++] = bite;
    }
    strcat(key, "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"); // Add the omni-valid GUID
    Sha1.init();
    Sha1.print(key);
    uint8_t *hash = Sha1.result();
    base64_encode(temp, (char*)hash, 20);
    debug.print(temp);

    int cc = -1;
    while(temp[cc++] != '\0') {} // cc is length return key
    cc = 165 + cc; // length return key + 165 keys for rest of header

    Serial.print("AT+CIPSEND=0,");
    Serial.println(129);  // +30   // was 129
    boolean found = false;
    while(!found)
      found = Serial.find(">");  // Wait until I can send
    Serial.print("HTTP/1.1 101 Switching Protocols\r\n");
    Serial.print("Upgrade: websocket\r\n");
    Serial.print("Connection: Upgrade\r\n");
    Serial.print("Sec-WebSocket-Accept: ");
    Serial.print(temp);
    Serial.print("\r\n\r\n");
    return true;
}

Answer 1

What you are looking at is the first Web Socket frame ( | | | q | | b | k | | c | | |) that has been concatenated with the HTTP header. (+IPD,0,10:) The data that your delimiting with pipes (|) is unintelligible because it's not ASCII, nor is it UTF8. You must display the data after the last colon (:) as BINARY. Then it should make complete sense. I was doing exactly the same thing. It was only when I displayed the total data as binary that I "Got it". I was using the "Web Sockets rock" demo from the web. It's an echo that just sends "Web Sockets rock" to a server that you nominate. I changed the Server address to the I.P. of my ESP8266 and started to look at the frames. I did a little analysis for myself (same as you did) to see what the ESP8266 would send back after a successful handshake. (I got the hand shake working first) Here is the 'post handshake' listing straight of TeraTerm-

+IPD,0,21:r¨$v%ÍF%ËVÃW    (NOTE: Garbage after the :) 

I expected to find "Web Sockets rock" somewhere in there.

Here is the listing converted to Binary, that I extracted from my receive buffer-

0 2B    0010 1011
1 49    0100 1001
2 50    0101 0000
3 44    0100 0100
4 2C    0010 1100
5 30    0011 0000
6 2C    0010 1100
7 32    0011 0010  (Ascii for 21 bytes to follow)
8 31    0011 0001  (Ascii for 21 bytes to follow)
9 3A    0011 1010  (Colon) 
10 -7F  1000 0001  (Start of actual FRAME) 
11 -71  1000 1111
12 72   0111 0010
13 -58  1010 1000
14 24   0010 0100
15 76   0111 0110
16 25   0010 0101
17 -33  1100 1101
18 46   0100 0110  
19 25   0010 0101
20 1D   0001 1101
21 -35  1100 1011
22 4F   0100 1111
23 13   0001 0011
24 6    0000 0110
25 -78  1000 1000
26 56   0101 0110
27 19   0001 1001
28 11   0001 0001
29 -3D  1100 0011
30 57   0101 0111

Description of the fields- (Starting from the first byte after the Colon. 81)

// First byte has FIN bit and frame type opcode = text
// Second byte mask and payload length
// next four bytes for masking key
// So total of 6 bytes for the overhead
// The size of the payload in this case is "F" = 15  (the 4th nibble) 
// So total of bytes are (6+15) = 21
// The first  byte is saying> FIN bit is set. This is last frame in sequence. The OP code is 1 = TEXT data.
// The second byte is saying> MASK bit is set. The following data will be masked. The data length is "F" = 15
// The 3rd, 4th, 5th, 6th bytes is the masking key. In this case 72, A8, 24, 76.

Answer 2

I can send messages now from the websocket >> arduino. But sending is not working :(.

boolean getFrame() {
    debug.println("getFrame()");
    byte bite;
    unsigned short payloadLength = 0;

    bite = Serial.read();        
    frame.opcode = bite & 0xf; // Opcode
    frame.isFinal = bite & 0x80; // Final frame?
    bite = Serial.read();
    frame.length = bite & 0x7f; // Length of payload        
    frame.isMasked = bite & 0x80;

    // Frame complete!
    if (!frame.isFinal) {
        return false;
    }
    // First check if the frame size is within our limits.
    if (frame.length > 126) {       
        return false;
    }

    // If the length part of the header is 126, it means it contains an extended length field.
    // Next two bytes contain the actual payload size, so we need to get the "true" length.
     if (frame.length == 126) {
        byte exLengthByte1 = Serial.read();
        byte exLengthByte2 = Serial.read();
        payloadLength = (exLengthByte1 << 8) + exLengthByte2;
     } 
     // If frame length is less than 126, that is the size of the payload.
     else {
        payloadLength = frame.length;        
     }

     // Check if our buffer can store the payload.
     if (payloadLength > MAX_RECEIVE_MESSAGE_SIZE) {
        debug.println("te groot");
        return false;
     }

    // Client should always send mask, but check just to be sure    
    if (frame.isMasked) {
        frame.mask[0] = Serial.read();
        frame.mask[1] = Serial.read();
        frame.mask[2] = Serial.read();
        frame.mask[3] = Serial.read();
    }

    // Get message bytes and unmask them if necessary
    for (int i = 0; i < payloadLength; i++) {
        if (frame.isMasked) {
            frame.data[i] = Serial.read() ^ frame.mask[i % 4];
        } else {
            frame.data[i] = Serial.read();
        }
    }

    for (int i = 0; i < payloadLength; i++) {
        debug.print(frame.data[i]);
        if(frame.data[i] == '/r')
          break;
    } 
    return true;
}

// !!!!!!!!!! NOT WORKING
boolean sendMessage(char *data, byte length) {    
      Serial.print((uint8_t) 0x1); // Txt frame opcode
      Serial.print((uint8_t) length); // Length of data
      for (int i = 0; i < length ; i++) {
          Serial.print(data[i]);
      }
      delay(1);
      return true;
}

See https://github.com/zoutepopcorn/esp8266-Websocket/blob/master/arduino_websocket.ino

The only problem now is the websocket format from arduino > websocket is not OK :(. But I think this is another issue / question. WebSocket connection to 'ws://192.168.1.101:8000/?encoding=text' failed: One or more reserved bits are on: reserved1 = 0, reserved2 = 1, reserved3 = 1

Answer 3

I have no experience with websockets, but I think websockets uses UTF-8 while the Arduino terminal uses ASCII. I do not see in your code conversion between UTF-8 and ASCII.