MeatPack serial encoding (#20802)

Co-authored-by: Scott Lahteine <thinkyhead@users.noreply.github.com>

Marlin/Configuration_adv.h

@@ -3298,6 +3298,8 @@
   //#define GCODE_QUOTED_STRINGS  // Support for quoted string parameters
 #endif
 
+//#define MEATPACK                // Support for MeatPack G-code compression (https://github.com/scottmudge/OctoPrint-MeatPack)
+
 //#define GCODE_CASE_INSENSITIVE  // Accept G-code sent to the firmware in lowercase
 
 //#define REPETIER_GCODE_M360     // Add commands originally from Repetier FW

Marlin/src/feature/meatpack.cpp

@@ -0,0 +1,254 @@
+/**
+ * Marlin 3D Printer Firmware
+ * Copyright (c) 2020 MarlinFirmware [https://github.com/MarlinFirmware/Marlin]
+ *
+ * Based on Sprinter and grbl.
+ * Copyright (c) 2011 Camiel Gubbels / Erik van der Zalm
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+/**
+ * MeatPack G-code Compression
+ *
+ * Algorithm & Implementation: Scott Mudge - mail@scottmudge.com
+ * Date: Dec. 2020
+ *
+ * Character Frequencies from ~30 MB of comment-stripped gcode:
+ *  '1' -> 4451136    '4' -> 1353273   '\n' -> 1087683    '-' ->   90242
+ *  '0' -> 4253577    '9' -> 1352147    'G' -> 1075806    'Z' ->   34109
+ *  ' ' -> 3053297    '3' -> 1262929    'X' ->  975742    'M' ->   11879
+ *  '.' -> 3035310    '5' -> 1189871    'E' ->  965275    'S' ->    9910
+ *  '2' -> 1523296    '6' -> 1127900    'Y' ->  965274
+ *  '8' -> 1366812    '7' -> 1112908    'F' ->   99416
+ *
+ * When space is omitted the letter 'E' is used in its place
+ */
+
+#include "../inc/MarlinConfig.h"
+
+#if ENABLED(MEATPACK)
+
+#include "meatpack.h"
+MeatPack meatpack;
+
+#define MeatPack_ProtocolVersion "PV01"
+//#define MEATPACK_LOOKUP_TABLE
+//#define MP_DEBUG
+
+#define DEBUG_OUT ENABLED(MP_DEBUG)
+#include "../core/debug_out.h"
+
+bool MeatPack::cmd_is_next = false;       // A command is pending
+uint8_t MeatPack::state = 0;              // Configuration state OFF
+uint8_t MeatPack::second_char = 0;        // The unpacked 2nd character from an out-of-sequence packed pair
+uint8_t MeatPack::cmd_count = 0,          // Counts how many command bytes are received (need 2)
+        MeatPack::full_char_count = 0,    // Counts how many full-width characters are to be received
+        MeatPack::char_out_count = 0;     // Stores number of characters to be read out.
+uint8_t MeatPack::char_out_buf[2];        // Output buffer for caching up to 2 characters
+
+#if ENABLED(MEATPACK_LOOKUP_TABLE)
+  // The 15 most-common characters used in G-code, ~90-95% of all G-code uses these characters
+  // Stored in SRAM for performance.
+  static const uint8_t meatPackLookupTable[16] = {
+    '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
+    '.', ' ', '\n', 'G', 'X',
+    '\0' // Unused. 0b1111 indicates a literal character
+  };
+#endif
+
+uint8_t MeatPack::unpacked_char(register const uint8_t in) {
+  #if ENABLED(MEATPACK_LOOKUP_TABLE)
+
+    return meatPackLookupTable[in];
+
+  #else
+
+      switch (in) {
+        case 0b0000 ... 0b1001: return '0' + in;
+        case 0b1010: return '.';
+        case 0b1011: return (state & MPConfig_Bit_NoSpaces) ? kSpaceCharReplace : ' ';
+        case 0b1100: return '\n';
+        case 0b1101: return 'G';
+        case 0b1110: return 'X';
+      }
+      return 0;
+
+  #endif
+}
+
+TERN_(MP_DEBUG, uint8_t chars_decoded = 0); // Log the first 64 bytes after each reset
+
+void MeatPack::reset_state() {
+  state = 0;
+  cmd_is_next = false;
+  second_char = 0;
+  cmd_count = full_char_count = char_out_count = 0;
+  TERN_(MP_DEBUG, chars_decoded = 0);
+  report_state();
+}
+
+/**
+ * Unpack one or two characters from a packed byte into a buffer.
+ * Return flags indicating whether any literal bytes follow.
+ */
+uint8_t MeatPack::unpack_chars(const uint8_t pk, uint8_t* __restrict const chars_out) {
+  uint8_t out = 0;
+
+  // If lower nybble is 1111, the higher nybble is unused, and next char is full.
+  if ((pk & kFirstNotPacked) == kFirstNotPacked)
+    out = kFirstCharIsLiteral;
+  else {
+    const uint8_t chr = pk & 0x0F;
+    chars_out[0] = unpacked_char(chr); // Set the first char
+  }
+
+  // Check if upper nybble is 1111... if so, we don't need the second char.
+  if ((pk & kSecondNotPacked) == kSecondNotPacked)
+    out |= kSecondCharIsLiteral;
+  else {
+    const uint8_t chr = (pk >> 4) & 0x0F;
+    chars_out[1] = unpacked_char(chr); // Set the second char
+  }
+
+  return out;
+}
+
+/**
+ * Interpret a single (non-command) character
+ * according to the current MeatPack state.
+ */
+void MeatPack::handle_rx_char_inner(const uint8_t c) {
+  if (TEST(state, MPConfig_Bit_Active)) {                   // Is MeatPack active?
+    if (!full_char_count) {                                 // No literal characters to fetch?
+      uint8_t buf[2] = { 0, 0 };
+      register const uint8_t res = unpack_chars(c, buf);    // Decode the byte into one or two characters.
+      if (res & kFirstCharIsLiteral) {                      // The 1st character couldn't be packed.
+        ++full_char_count;                                  // So the next stream byte is a full character.
+        if (res & kSecondCharIsLiteral) ++full_char_count;  // The 2nd character couldn't be packed. Another stream byte is a full character.
+        else second_char = buf[1];                          // Retain the unpacked second character.
+      }
+      else {
+        handle_output_char(buf[0]);                           // Send the unpacked first character out.
+        if (buf[0] != '\n') {                                 // After a newline the next char won't be set
+          if (res & kSecondCharIsLiteral) ++full_char_count;  // The 2nd character couldn't be packed. The next stream byte is a full character.
+          else handle_output_char(buf[1]);                    // Send the unpacked second character out.
+        }
+      }
+    }
+    else {
+      handle_output_char(c);                                // Pass through the character that couldn't be packed...
+      if (second_char) {
+        handle_output_char(second_char);                    // ...and send an unpacked 2nd character, if set.
+        second_char = 0;
+      }
+      --full_char_count;                                    // One literal character was consumed
+    }
+  }
+  else // Packing not enabled, just copy character to output
+    handle_output_char(c);
+}
+
+/**
+ * Buffer a single output character which will be picked up in
+ * GCodeQueue::get_serial_commands via calls to get_result_char
+ */
+void MeatPack::handle_output_char(const uint8_t c) {
+  char_out_buf[char_out_count++] = c;
+
+  #if ENABLED(MP_DEBUG)
+    if (chars_decoded < 1024) {
+      ++chars_decoded;
+      DEBUG_ECHOPGM("RB: ");
+      MYSERIAL.print((char)c);
+      DEBUG_EOL();
+    }
+  #endif
+}
+
+/**
+ * Process a MeatPack command byte to update the state.
+ * Report the new state to serial.
+ */
+void MeatPack::handle_command(const MeatPack_Command c) {
+  switch (c) {
+    case MPCommand_EnablePacking:   SBI(state, MPConfig_Bit_Active);   DEBUG_ECHOLNPGM("[MPDBG] ENA REC");   break;
+    case MPCommand_DisablePacking:  CBI(state, MPConfig_Bit_Active);   DEBUG_ECHOLNPGM("[MPDBG] DIS REC");   break;
+    case MPCommand_TogglePacking:   TBI(state, MPConfig_Bit_Active);   DEBUG_ECHOLNPGM("[MPDBG] TGL REC");   break;
+    case MPCommand_ResetAll:        reset_state();                     DEBUG_ECHOLNPGM("[MPDBG] RESET REC"); break;
+    case MPCommand_EnableNoSpaces:  SBI(state, MPConfig_Bit_NoSpaces); DEBUG_ECHOLNPGM("[MPDBG] ENA NSP");
+                                    TERN_(USE_LOOKUP_TABLE, MeatPackLookupTbl[kSpaceCharIdx] = kSpaceCharReplace);
+                                    break;
+    case MPCommand_DisableNoSpaces: CBI(state, MPConfig_Bit_NoSpaces); DEBUG_ECHOLNPGM("[MPDBG] DIS NSP");
+                                    TERN_(USE_LOOKUP_TABLE, MeatPackLookupTbl[kSpaceCharIdx] = ' ');
+                                    break;
+    default:                                                           DEBUG_ECHOLNPGM("[MPDBG] UNK CMD REC");
+    case MPCommand_QueryConfig: break;
+  }
+  report_state();
+}
+
+void MeatPack::report_state() {
+  // NOTE: if any configuration vars are added below, the outgoing sync text for host plugin
+  // should not contain the "PV' substring, as this is used to indicate protocol version
+  SERIAL_ECHOPGM("[MP] ");
+  SERIAL_ECHOPGM(MeatPack_ProtocolVersion);
+  serialprint_onoff(TEST(state, MPConfig_Bit_Active));
+  SERIAL_CHAR(' ');
+  serialprintPGM(TEST(state, MPConfig_Bit_NoSpaces) ? PSTR("NSP") : PSTR("ESP"));
+  SERIAL_EOL();
+}
+
+/**
+ * Interpret a single character received from serial
+ * according to the current meatpack state.
+ */
+void MeatPack::handle_rx_char(const uint8_t c) {
+  if (c == kCommandByte) {                // A command (0xFF) byte?
+    if (cmd_count) {                      // In fact, two in a row?
+      cmd_is_next = true;                 // Then a MeatPack command follows
+      cmd_count = 0;
+    }
+    else
+      ++cmd_count; // cmd_count = 1       // One command byte received so far...
+    return;
+  }
+
+  if (cmd_is_next) {                      // Were two command bytes received?
+    handle_command((MeatPack_Command)c);  // Then the byte is a MeatPack command
+    cmd_is_next = false;
+    return;
+  }
+
+  if (cmd_count) {                        // Only a single 0xFF was received
+    handle_rx_char_inner(kCommandByte);   // A single 0xFF is passed on literally so it can be interpreted as kFirstNotPacked|kSecondNotPacked
+    cmd_count = 0;
+  }
+
+  handle_rx_char_inner(c);                // Other characters are passed on for MeatPack decoding
+}
+
+uint8_t MeatPack::get_result_char(char* const __restrict out) {
+  uint8_t res = 0;
+  if (char_out_count) {
+    res = char_out_count;
+    char_out_count = 0;
+    for (register uint8_t i = 0; i < res; ++i)
+      out[i] = (char)char_out_buf[i];
+  }
+  return res;
+}
+
+#endif // MEATPACK

Marlin/src/feature/meatpack.h

@@ -0,0 +1,124 @@
+/**
+ * Marlin 3D Printer Firmware
+ * Copyright (c) 2020 MarlinFirmware [https://github.com/MarlinFirmware/Marlin]
+ *
+ * Based on Sprinter and grbl.
+ * Copyright (c) 2011 Camiel Gubbels / Erik van der Zalm
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+/*
+ * MeatPack G-code Compression
+ *
+ * Algorithm & Implementation: Scott Mudge - mail@scottmudge.com
+ * Date: Dec. 2020
+ *
+ * Specifically optimized for 3D printing G-Code, this is a zero-cost data compression method
+ * which packs ~180-190% more data into the same amount of bytes going to the CNC controller.
+ * As a majority of G-Code can be represented by a restricted alphabet, I performed histogram
+ * analysis on a wide variety of 3D printing gcode samples, and found ~93% of all gcode could
+ * be represented by the same 15-character alphabet.
+ *
+ * This allowed me to design a system of packing 2 8-bit characters into a single byte, assuming
+ * they fall within this limited 15-character alphabet. Using a 4-bit lookup table, these 8-bit
+ * characters can be represented by a 4-bit index.
+ *
+ * Combined with some logic to allow commingling of full-width characters outside of this 15-
+ * character alphabet (at the cost of an extra 8-bits per full-width character), and by stripping
+ * out unnecessary comments, the end result is gcode which is roughly half the original size.
+ *
+ * Why did I do this? I noticed micro-stuttering and other data-bottleneck issues while printing
+ * objects with high curvature, especially at high speeds. There is also the issue of the limited
+ * baud rate provided by Prusa's Atmega2560-based boards, over the USB serial connection. So soft-
+ * ware like OctoPrint would also suffer this same micro-stuttering and poor print quality issue.
+ *
+ */
+#pragma once
+
+#include <stdint.h>
+
+/**
+ * Commands sent to MeatPack to control its behavior.
+ * They are sent by first sending 2x MeatPack_CommandByte (0xFF) in sequence,
+ *      followed by one of the command bytes below.
+ * Provided that 0xFF is an exceedingly rare character that is virtually never
+ * present in G-code naturally, it is safe to assume 2 in sequence should never
+ * happen naturally, and so it is used as a signal here.
+ *
+ * 0xFF *IS* used in "packed" G-code (used to denote that the next 2 characters are
+ * full-width), however 2 in a row will never occur, as the next 2 bytes will always
+ * some non-0xFF character.
+ */
+enum MeatPack_Command : uint8_t {
+  MPCommand_None            = 0,
+  MPCommand_TogglePacking   = 0xFD,
+  MPCommand_EnablePacking   = 0xFB,
+  MPCommand_DisablePacking  = 0xFA,
+  MPCommand_ResetAll        = 0xF9,
+  MPCommand_QueryConfig     = 0xF8,
+  MPCommand_EnableNoSpaces  = 0xF7,
+  MPCommand_DisableNoSpaces = 0xF6
+};
+
+enum MeatPack_ConfigStateBits : uint8_t {
+  MPConfig_Bit_Active   = 0,
+  MPConfig_Bit_NoSpaces = 1
+};
+
+class MeatPack {
+private:
+  friend class GCodeQueue;
+
+  // Utility definitions
+  static const uint8_t kCommandByte         = 0b11111111,
+                       kFirstNotPacked      = 0b00001111,
+                       kSecondNotPacked     = 0b11110000,
+                       kFirstCharIsLiteral  = 0b00000001,
+                       kSecondCharIsLiteral = 0b00000010;
+
+  static const uint8_t kSpaceCharIdx = 11;
+  static const char kSpaceCharReplace = 'E';
+
+  static bool cmd_is_next;        // A command is pending
+  static uint8_t state;           // Configuration state
+  static uint8_t second_char;     // Buffers a character if dealing with out-of-sequence pairs
+  static uint8_t cmd_count,       // Counter of command bytes received (need 2)
+                 full_char_count, // Counter for full-width characters to be received
+                 char_out_count;  // Stores number of characters to be read out.
+  static uint8_t char_out_buf[2]; // Output buffer for caching up to 2 characters
+
+  // Pass in a character rx'd by SD card or serial. Automatically parses command/ctrl sequences,
+  // and will control state internally.
+  static void handle_rx_char(const uint8_t c);
+
+  /**
+   * After passing in rx'd char using above method, call this to get characters out.
+   * Can return from 0 to 2 characters at once.
+   * @param out [in] Output pointer for unpacked/processed data.
+   * @return Number of characters returned. Range from 0 to 2.
+   */
+  static uint8_t get_result_char(char* const __restrict out);
+
+  static void reset_state();
+  static void report_state();
+  static uint8_t unpacked_char(register const uint8_t in);
+  static uint8_t unpack_chars(const uint8_t pk, uint8_t* __restrict const chars_out);
+  static void handle_command(const MeatPack_Command c);
+  static void handle_output_char(const uint8_t c);
+  static void handle_rx_char_inner(const uint8_t c);
+};
+
+extern MeatPack meatpack;

Marlin/src/gcode/host/M115.cpp

@@ -141,6 +141,9 @@ void GcodeSuite::M115() {
     // CHAMBER_TEMPERATURE (M141, M191)
     cap_line(PSTR("CHAMBER_TEMPERATURE"), ENABLED(HAS_HEATED_CHAMBER));
 
+    // MEATPACK Compresson
+    cap_line(PSTR("MEATPACK"), ENABLED(MEATPACK));
+
     // Machine Geometry
     #if ENABLED(M115_GEOMETRY_REPORT)
       const xyz_pos_t dmin = { X_MIN_POS, Y_MIN_POS, Z_MIN_POS },

Marlin/src/gcode/queue.cpp

@@ -48,6 +48,10 @@ GCodeQueue queue;
   #include "../feature/binary_stream.h"
 #endif
 
+#if ENABLED(MEATPACK)
+  #include "../feature/meatpack.h"
+#endif
+
 #if ENABLED(POWER_LOSS_RECOVERY)
   #include "../feature/powerloss.h"
 #endif
@@ -474,7 +478,16 @@ void GCodeQueue::get_serial_commands() {
       const int c = read_serial(i);
       if (c < 0) continue;
 
-      const char serial_char = c;
+      #if ENABLED(MEATPACK)
+        meatpack.handle_rx_char(uint8_t(c));
+        char c_res[2] = { 0, 0 };
+        const uint8_t char_count = meatpack.get_result_char(c_res);
+      #else
+        constexpr uint8_t char_count = 1;
+      #endif
+
+      LOOP_L_N(char_index, char_count) {
+        const char serial_char = TERN(MEATPACK, c_res[char_index], c);
 
         if (ISEOL(serial_char)) {
 
@@ -565,7 +578,9 @@ void GCodeQueue::get_serial_commands() {
         else
           process_stream_char(serial_char, serial_input_state[i], serial_line_buffer[i], serial_count[i]);
 
-    } // for NUM_SERIAL
+      } // char_count loop
+
+    } // NUM_SERIAL loop
   } // queue has space, serial has data
 }
 

Marlin/src/inc/SanityCheck.h

@@ -3303,6 +3303,14 @@ static_assert(   _ARR_TEST(3,0) && _ARR_TEST(3,1) && _ARR_TEST(3,2)
   #endif
 #endif
 
+
+/**
+ * Sanity Check for MEATPACK and BINARY_FILE_TRANSFER Features
+ */
+#if BOTH(MEATPACK, BINARY_FILE_TRANSFER)
+  #error "Either enable MEATPACK or enable BINARY_FILE_TRANSFER."
+#endif
+
 /**
  * Sanity check for valid stepper driver types
  */

buildroot/tests/FYSETC_S6-tests

@@ -9,6 +9,7 @@ set -e
 # Build examples
 restore_configs
 use_example_configs FYSETC/S6
+opt_enable MEATPACK
 opt_set Y_DRIVER_TYPE TMC2209
 opt_set Z_DRIVER_TYPE TMC2130
 exec_test $1 $2 "FYSETC S6 Example" "$3"

platformio.ini

@@ -97,6 +97,7 @@ default_src_filter = +<src/*> -<src/config> -<src/HAL> +<src/HAL/shared>
   -<src/feature/leds/printer_event_leds.cpp>
   -<src/feature/leds/tempstat.cpp>
   -<src/feature/max7219.cpp>
+  -<src/feature/meatpack.cpp>
   -<src/feature/mixing.cpp>
   -<src/feature/mmu/mmu.cpp>
   -<src/feature/mmu/mmu2.cpp> -<src/gcode/feature/prusa_MMU2>
@@ -319,6 +320,7 @@ PCA9632                 = src_filter=+<src/feature/leds/pca9632.cpp>
 PRINTER_EVENT_LEDS      = src_filter=+<src/feature/leds/printer_event_leds.cpp>
 TEMP_STAT_LEDS          = src_filter=+<src/feature/leds/tempstat.cpp>
 MAX7219_DEBUG           = src_filter=+<src/feature/max7219.cpp> +<src/gcode/feature/leds/M7219.cpp>
+MEATPACK                = src_filter=+<src/feature/meatpack.cpp>
 MIXING_EXTRUDER         = src_filter=+<src/feature/mixing.cpp> +<src/gcode/feature/mixing/M163-M165.cpp>
 HAS_PRUSA_MMU1          = src_filter=+<src/feature/mmu/mmu.cpp>
 HAS_PRUSA_MMU2          = src_filter=+<src/feature/mmu/mmu2.cpp> +<src/gcode/feature/prusa_MMU2>
@@ -706,7 +708,7 @@ extra_scripts     = ${common.extra_scripts}
 src_filter        = ${common.default_src_filter} +<src/HAL/LPC1768> +<src/HAL/shared/backtrace>
 lib_deps          = ${common.lib_deps}
   Servo
-custom_marlin.USES_LIQUIDCRYSTAL = LiquidCrystal@1.0.0
+custom_marlin.USES_LIQUIDCRYSTAL = LiquidCrystal~1.0.7
 custom_marlin.NEOPIXEL_LED = Adafruit NeoPixel=https://github.com/p3p/Adafruit_NeoPixel/archive/1.5.0.zip
 build_flags       = ${common.build_flags} -DU8G_HAL_LINKS -IMarlin/src/HAL/LPC1768/include -IMarlin/src/HAL/LPC1768/u8g
   # debug options for backtrace