zaubentrucker/muele-marlin
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

SPI Flash data compression (#18879)

Browse source
This commit is contained in:
Victor Oliveira 2020-08-04 06:50:05 -03:00 committed by GitHub
parent 3b1f84a9b7
commit 04b12dde49
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 489 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

317
Marlin/src/lcd/extui/lib/mks_ui/SPIFlashStorage.cpp Normal file
View file

@ -0,0 +1,317 @@
/**
* Marlin 3D Printer Firmware
* Copyright (c) 2020 MarlinFirmware [https://github.com/MarlinFirmware/Marlin]
*
* Based on Sprinter and grbl.
* Copyright (c) 2011 Camiel Gubbels / Erik van der Zalm
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
#include "../../../../inc/MarlinConfigPre.h"
#if HAS_TFT_LVGL_UI
#include "SPIFlashStorage.h"
uint8_t SPIFlashStorage::::m_pageData[SPI_FLASH_PageSize];
uint32_t SPIFlashStorage::::m_currentPage;
uint16_t SPIFlashStorage::::m_pageDataUsed;
uint32_t SPIFlashStorage::::m_startAddress;
#if HAS_SPI_FLASH_COMPRESSION
uint8_t SPIFlashStorage::m_compressedData[SPI_FLASH_PageSize];
uint16_t SPIFlashStorage::m_compressedDataUsed;
template <typename T>
static uint32_t rle_compress(T *output, uint32_t outputLength, T *input, uint32_t inputLength, uint32_t& inputProcessed) {
uint32_t count = 0, out = 0, index, i;
T pixel;
//32767 for uint16_t
//127 for uint16_t
//calculated at compile time
constexpr T max = (0xFFFFFFFF >> (8 * (4 - sizeof(T)))) / 2;
inputProcessed = 0;
while (count < inputLength && out < outputLength) {
index = count;
pixel = input[index++];
while (index < inputLength && index - count < max && input[index] == pixel)
index++;
if (index - count == 1) {
/*
* Failed to "replicate" the current pixel. See how many to copy.
* Avoid a replicate run of only 2-pixels after a literal run. There
* is no gain in this, and there is a risK of loss if the run after
* the two identical pixels is another literal run. So search for
* 3 identical pixels.
*/
while (index < inputLength && index - count < max && (input[index] != input[index - 1] || (index > 1 && input[index] != input[index - 2])))
index++;
/*
* Check why this run stopped. If it found two identical pixels, reset
* the index so we can add a run. Do this twice: the previous run
* tried to detect a replicate run of at least 3 pixels. So we may be
* able to back up two pixels if such a replicate run was found.
*/
while (index < inputLength && input[index] == input[index - 1])
index--;
// If the output buffer could overflow, stop at the remaining bytes
NOMORE(index, count + outputLength - out - 1);
output[out++] = (uint16_t)(count - index);
for (i = count; i < index; i++)
output[out++] = input[i];
}
else {
// Need at least more 2 spaces
if (out > outputLength - 2) break;
output[out++] = (uint16_t)(index - count);
output[out++] = pixel;
}
count = index;
}
inputProcessed = count;
// Padding
if (out == outputLength - 1) output[out++] = 0;
return out;
}
template <typename UT, typename T>
static uint32_t rle_uncompress(UT *output, uint32_t outputLength, UT *input, uint32_t inputLength, uint32_t &outputFilled) {
T count;
UT i;
uint32_t processedBytes = 0;
outputFilled = 0;
while (outputLength > 0 && inputLength > 0) {
processedBytes++;
count = static_cast<T>(*input++);
inputLength--;
if (count > 0) { // Replicate run
for (i = 0; i < count && outputLength > i; i++)
output[i] = *input;
outputFilled += i;
// If copy incomplete, change the input buffer to start with remaining data in the next call
if (i < count) {
// Change to process the difference in the next call
*(input - 1) = static_cast<UT>(count - i);
return processedBytes - 1;
}
input++;
inputLength--;
processedBytes++;
}
else if (count < 0) { // literal run
count = static_cast<T>(-count);
// Copy, validating if the output have enough space
for (i = 0; i < count && outputLength > i; i++)
output[i] = input[i];
outputFilled += i;
// If copy incomplete, change the input buffer to start with remaining data in the next call
if (i < count) {
input[i - 1] = static_cast<UT>((count - i) * -1);
// Back one
return processedBytes + i - 1;
}
input += count;
inputLength -= count;
processedBytes += count;
}
output += count;
outputLength -= count;
}
return processedBytes;
}
#endif // HAS_SPI_FLASH_COMPRESSION
void SPIFlashStorage::beginWrite(uint32_t startAddress) {
m_pageDataUsed = 0;
m_currentPage = 0;
m_startAddress = startAddress;
#if HAS_SPI_FLASH_COMPRESSION
// Restart the compressed buffer, keep the pointers of the uncompressed buffer
m_compressedDataUsed = 0;
#endif
}
void SPIFlashStorage::endWrite() {
// Flush remaining data
#if HAS_SPI_FLASH_COMPRESSION
if (m_compressedDataUsed > 0) {
flushPage();
savePage(m_compressedData);
}
#else
if (m_pageDataUsed > 0) flushPage();
#endif
}
void SPIFlashStorage::savePage(uint8_t* buffer) {
W25QXX.SPI_FLASH_BufferWrite(buffer, m_startAddress + (SPI_FLASH_PageSize * m_currentPage), SPI_FLASH_PageSize);
// Test env
// char fname[256];
// snprintf(fname, sizeof(fname), "./pages/page-%03d.data", m_currentPage);
// FILE *fp = fopen(fname, "wb");
// fwrite(buffer, 1, m_compressedDataUsed, fp);
// fclose(fp);
}
void SPIFlashStorage::loadPage(uint8_t* buffer) {
W25QXX.SPI_FLASH_BufferRead(buffer, m_startAddress + (SPI_FLASH_PageSize * m_currentPage), SPI_FLASH_PageSize);
// Test env
// char fname[256];
// memset(buffer, 0, SPI_FLASH_PageSize);
// snprintf(fname, sizeof(fname), "./pages/page-%03d.data", m_currentPage);
// FILE *fp = fopen(fname, "rb");
// if (fp != NULL) {
// fread(buffer, 1, SPI_FLASH_PageSize, fp);
// fclose(fp);
// }
}
void SPIFlashStorage::flushPage() {
#if HAS_SPI_FLASH_COMPRESSION
// Work com with compressed in memory
uint32_t inputProcessed;
uint32_t compressedSize = rle_compress<uint16_t>((uint16_t *)(m_compressedData + m_compressedDataUsed), compressedDataFree() / 2, (uint16_t *)m_pageData, m_pageDataUsed / 2, inputProcessed) * 2;
inputProcessed *= 2;
m_compressedDataUsed += compressedSize;
// Space remaining in the compressed buffer?
if (compressedDataFree() > 0) {
// Free the uncompressed buffer
m_pageDataUsed = 0;
return;
}
// Part of the m_pageData was compressed, so ajust the pointers, freeing what was processed, shift the buffer
// TODO: To avoid this copy, use a circular buffer
memmove(m_pageData, m_pageData + inputProcessed, m_pageDataUsed - inputProcessed);
m_pageDataUsed -= inputProcessed;
// No? So flush page with compressed data!!
uint8_t *buffer = m_compressedData;
#else
uint8_t *buffer = m_pageData;
#endif
savePage(buffer);
#if HAS_SPI_FLASH_COMPRESSION
// Restart the compressed buffer, keep the pointers of the uncompressed buffer
m_compressedDataUsed = 0;
#elif
m_pageDataUsed = 0;
#endif
m_currentPage++;
}
void SPIFlashStorage::readPage() {
#if HAS_SPI_FLASH_COMPRESSION
if (compressedDataFree() == 0) {
loadPage(m_compressedData);
m_currentPage++;
m_compressedDataUsed = 0;
}
// Need to uncompress data
if (pageDataFree() == 0) {
m_pageDataUsed = 0;
uint32_t outpuProcessed = 0;
uint32_t inputProcessed = rle_uncompress<uint16_t, int16_t>((uint16_t *)(m_pageData + m_pageDataUsed), pageDataFree() / 2, (uint16_t *)(m_compressedData + m_compressedDataUsed), compressedDataFree() / 2, outpuProcessed);
inputProcessed *= 2;
outpuProcessed *= 2;
if (outpuProcessed < pageDataFree()) {
m_pageDataUsed = SPI_FLASH_PageSize - outpuProcessed;
// TODO: To avoid this copy, use a circular buffer
memmove(m_pageData + m_pageDataUsed, m_pageData, outpuProcessed);
}
m_compressedDataUsed += inputProcessed;
}
#else
loadPage(m_pageData);
m_pageDataUsed = 0;
m_currentPage++;
#endif
}
uint16_t SPIFlashStorage::inData(uint8_t* data, uint16_t size) {
// Don't write more than we can
NOMORE(size, pageDataFree());
memcpy(m_pageData + m_pageDataUsed, data, size);
m_pageDataUsed += size;
return size;
}
void SPIFlashStorage::writeData(uint8_t* data, uint16_t size) {
// Flush a page if needed
if (pageDataFree() == 0) flushPage();
while (size > 0) {
uint16_t written = inData(data, size);
size -= written;
// Need to write more? Flush page and continue!
if (size > 0) {
flushPage();
data += written;
}
}
}
void SPIFlashStorage::beginRead(uint32_t startAddress) {
m_startAddress = startAddress;
m_currentPage = 0;
// Nothing in memory now
m_pageDataUsed = SPI_FLASH_PageSize;
#if HAS_SPI_FLASH_COMPRESSION
m_compressedDataUsed = sizeof(m_compressedData);
#endif
}
uint16_t SPIFlashStorage::outData(uint8_t* data, uint16_t size) {
// Don't read more than we have
NOMORE(size > pageDataFree());
memcpy(data, m_pageData + m_pageDataUsed, size);
m_pageDataUsed += size;
return size;
}
void SPIFlashStorage::readData(uint8_t* data, uint16_t size) {
// Read a page if needed
if (pageDataFree() == 0) readPage();
while (size > 0) {
uint16_t read = outData(data, size);
size -= read;
// Need to write more? Flush page and continue!
if (size > 0) {
readPage();
data += read;
}
}
}
SPIFlashStorage SPIFlash;
#endif // HAS_TFT_LVGL_UI

108
Marlin/src/lcd/extui/lib/mks_ui/SPIFlashStorage.h Normal file
View file

@ -0,0 +1,108 @@
/**
* Marlin 3D Printer Firmware
* Copyright (c) 2020 MarlinFirmware [https://github.com/MarlinFirmware/Marlin]
*
* Based on Sprinter and grbl.
* Copyright (c) 2011 Camiel Gubbels / Erik van der Zalm
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
#pragma once
#include "W25Qxx.h"
#define HAS_SPI_FLASH_COMPRESSION 1
/**
* This class manages and optimizes SPI Flash data storage,
* keeping an internal buffer to write and save full SPI flash
* pages as needed.
*
* Since the data is always in the buffer, the class is also
* able to support fast on-the-fly RLE compression/decompression.
*
* In testing with the current LVGL_UI it compacts 2.9MB of icons
* (which have lots of runs) down to 370kB!!! As a result the UI
* refresh rate becomes faster and now all LVGL UI can fit into a
* tiny 2MB SPI Flash, such as the Chitu Board.
*
* == Usage ==
*
* Writing:
*
* The class keeps an internal buffer that caches data until it
* fits into a full SPI Flash page. Each time the buffer fills up
* the page is saved to SPI Flash. Sequential writes are optimal.
*
* SPIFlashStorage.beginWrite(myStartAddress);
* while (there is data to write)
* SPIFlashStorage.addData(myBuffer, bufferSize);
* SPIFlashStorage.endWrite(); // Flush remaining buffer data
*
* Reading:
*
* When reading, it loads a full page from SPI Flash at once and
* keeps it in a private SRAM buffer. Data is loaded as needed to
* fullfill requests. Sequential reads are optimal.
*
* SPIFlashStorage.beginRead(myStartAddress);
* while (there is data to read)
* SPIFlashStorage.readData(myBuffer, bufferSize);
*
* Compression:
*
* The biggest advantage of this class is the RLE compression.
* With compression activated a second buffer holds the compressed
* data, so when writing data, as this buffer becomes full it is
* flushed to SPI Flash.
*
* The same goes for reading: A compressed page is read from SPI
* flash, and the data is uncompressed as needed to provide the
* requested amount of data.
*/
class SPIFlashStorage {
public:
// Write operation
static void beginWrite(uint32_t startAddress);
static void endWrite();
static void writeData(uint8_t* data, uint16_t size);
static // Read operation
static void beginRead(uint32_t startAddress);
static void readData(uint8_t* data, uint16_t size);
static uint32_t getCurrentPage() { return m_currentPage; }
private:
static void flushPage();
static void savePage(uint8_t* buffer);
static void loadPage(uint8_t* buffer);
static void readPage();
static uint16_t inData(uint8_t* data, uint16_t size);
static uint16_t outData(uint8_t* data, uint16_t size);
static uint8_t m_pageData[SPI_FLASH_PageSize];
static uint32_t m_currentPage;
static uint16_t m_pageDataUsed;
static inline uint16_t pageDataFree() { return SPI_FLASH_PageSize - m_pageDataUsed; }
static uint32_t m_startAddress;
#if HAS_SPI_FLASH_COMPRESSION
static uint8_t m_compressedData[SPI_FLASH_PageSize];
static uint16_t m_compressedDataUsed;
static inline uint16_t compressedDataFree() { return SPI_FLASH_PageSize - m_compressedDataUsed; }
#endif
};
extern SPIFlashStorage SPIFlash;

86
Marlin/src/lcd/extui/lib/mks_ui/pic_manager.cpp
View file

@ -23,16 +23,17 @@
#if HAS_TFT_LVGL_UI
#include "../../../../MarlinCore.h"
#include "string.h"
#include "pic_manager.h"
#include "W25Qxx.h"
#include "../../../../sd/cardreader.h"
#include "draw_ready_print.h"
#include "mks_hardware_test.h"
#include "SPIFlashStorage.h"
#include "W25Qxx.h"
#include "../../../../MarlinCore.h"
#include "../../../../sd/cardreader.h"
extern uint16_t DeviceCode;
extern unsigned char bmp_public_buf[17 * 1024];
@ -205,11 +206,11 @@ static char assets[][LONG_FILENAME_LENGTH] = {
};
#if HAS_SPI_FLASH_FONT
static char fonts[][LONG_FILENAME_LENGTH] = {
"FontUNIGBK.bin",
};
static char fonts[][LONG_FILENAME_LENGTH] = { "FontUNIGBK.bin" };
#endif
static uint8_t currentFlashPage = 0;
uint32_t lv_get_pic_addr(uint8_t *Pname) {
uint8_t Pic_cnt;
uint8_t i, j;
@ -217,6 +218,8 @@ uint32_t lv_get_pic_addr(uint8_t *Pname) {
uint32_t tmp_cnt = 0;
uint32_t addr = 0;
currentFlashPage = 0;
#if ENABLED(MARLIN_DEV_MODE)
SERIAL_ECHOLNPAIR("Getting picture SPI Flash Address: ", (const char*)Pname);
#endif
@ -371,6 +374,10 @@ uint8_t public_buf[512];
return -1;
}
#if ENABLED(MARLIN_DEV_MODE)
static uint32_t totalSizes = 0, totalCompressed = 0;
#endif
#define ASSET_TYPE_ICON 0
#define ASSET_TYPE_LOGO 1
#define ASSET_TYPE_TITLE_LOGO 2
@ -398,43 +405,52 @@ uint8_t public_buf[512];
pfileSize = file.fileSize();
totalSizeLoaded += pfileSize;
if (assetType == ASSET_TYPE_LOGO) {
while (1) {
do {
pbr = file.read(public_buf, BMP_WRITE_BUF_LEN);
Pic_Logo_Write((uint8_t *)fn, public_buf, pbr); //
if (pbr < BMP_WRITE_BUF_LEN) break;
}
Pic_Logo_Write((uint8_t *)fn, public_buf, pbr);
} while (pbr >= BMP_WRITE_BUF_LEN);
}
else if (assetType == ASSET_TYPE_TITLE_LOGO) {
while (1) {
do {
pbr = file.read(public_buf, BMP_WRITE_BUF_LEN);
Pic_TitleLogo_Write((uint8_t *)fn, public_buf, pbr); //
if (pbr < BMP_WRITE_BUF_LEN) break;
}
Pic_TitleLogo_Write((uint8_t *)fn, public_buf, pbr);
} while (pbr >= BMP_WRITE_BUF_LEN);
}
else if (assetType == ASSET_TYPE_G_PREVIEW) {
while (1) {
do {
pbr = file.read(public_buf, BMP_WRITE_BUF_LEN);
default_view_Write(public_buf, pbr); //
if (pbr < BMP_WRITE_BUF_LEN) break;
}
default_view_Write(public_buf, pbr);
} while (pbr >= BMP_WRITE_BUF_LEN);
}
else if (assetType == ASSET_TYPE_ICON) {
Pic_Write_Addr = Pic_Info_Write((uint8_t *)fn, pfileSize);
while (1) {
SPIFlash.beginWrite(Pic_Write_Addr);
#if HAS_SPI_FLASH_COMPRESSION
do {
pbr = file.read(public_buf, SPI_FLASH_PageSize);
TERN_(MARLIN_DEV_MODE, totalSizes += pbr);
SPIFlash.writeData(public_buf, SPI_FLASH_PageSize);
} while (pbr >= SPI_FLASH_PageSize);
#else
do {
pbr = file.read(public_buf, BMP_WRITE_BUF_LEN);
W25QXX.SPI_FLASH_BufferWrite(public_buf, Pic_Write_Addr, pbr);
Pic_Write_Addr += pbr;
if (pbr < BMP_WRITE_BUF_LEN) break;
}
} while (pbr >= BMP_WRITE_BUF_LEN);
#endif
#if ENABLED(MARLIN_DEV_MODE)
SERIAL_ECHOLNPAIR("Space used: ", fn, " - ", (SPIFlash.getCurrentPage() + 1) * SPI_FLASH_PageSize / 1024, "KB");
totalCompressed += (SPIFlash.getCurrentPage() + 1) * SPI_FLASH_PageSize;
#endif
SPIFlash.endWrite();
}
else if (assetType == ASSET_TYPE_FONT) {
Pic_Write_Addr = UNIGBK_FLASH_ADDR;
while (1) {
do {
pbr = file.read(public_buf, BMP_WRITE_BUF_LEN);
W25QXX.SPI_FLASH_BufferWrite(public_buf, Pic_Write_Addr, pbr);
Pic_Write_Addr += pbr;
if (pbr < BMP_WRITE_BUF_LEN) break;
}
} while (pbr >= BMP_WRITE_BUF_LEN);
}
file.close();
@ -459,13 +475,13 @@ uint8_t public_buf[512];
disp_assets_update_progress("Reading files...");
dir_t d;
while (dir.readDir(&d, card.longFilename) > 0) {
// if we dont get a long name, but gets a short one, try it
// If we dont get a long name, but gets a short one, try it
if (card.longFilename[0] == 0 && d.name[0] != 0)
dosName2LongName((const char*)d.name, card.longFilename);
if (card.longFilename[0] == 0) continue;
if (card.longFilename[0] == '.') continue;
uint8_t a = arrayFindStr(assets, COUNT(assets), card.longFilename);
int8_t a = arrayFindStr(assets, COUNT(assets), card.longFilename);
if (a >= 0 && a < COUNT(assets)) {
uint8_t assetType = ASSET_TYPE_ICON;
if (strstr(assets[a], "_logo"))
@ -482,9 +498,8 @@ uint8_t public_buf[512];
#if HAS_SPI_FLASH_FONT
a = arrayFindStr(fonts, COUNT(fonts), card.longFilename);
if (a >= 0 && a < COUNT(fonts)) {
if (a >= 0 && a < COUNT(fonts))
loadAsset(dir, d, fonts[a], ASSET_TYPE_FONT);
}
#endif
}
dir.rename(&root, bakPath);
@ -496,11 +511,13 @@ uint8_t public_buf[512];
W25QXX.SPI_FLASH_BufferRead(&pic_counter, PIC_COUNTER_ADDR, 1);
SERIAL_ECHOLNPAIR("Total assets loaded: ", pic_counter);
#endif
SERIAL_ECHOLNPAIR("Total Uncompressed: ", totalSizes, ", Compressed: ", totalCompressed);
}
#if HAS_SPI_FLASH_FONT
void spi_flash_read_test() { W25QXX.SPI_FLASH_BufferRead(public_buf, UNIGBK_FLASH_ADDR, BMP_WRITE_BUF_LEN); }
#endif // HAS_SPI_FLASH_FONT
#endif
#endif // SDSUPPORT
@ -531,8 +548,15 @@ void Pic_Read(uint8_t *Pname, uint8_t *P_Rbuff) {
}
void lv_pic_test(uint8_t *P_Rbuff, uint32_t addr, uint32_t size) {
#if HAS_SPI_FLASH_COMPRESSION
if (currentFlashPage == 0)
SPIFlash.beginRead(addr);
SPIFlash.readData(P_Rbuff, size);
currentFlashPage++;
#else
W25QXX.init(SPI_QUARTER_SPEED);
W25QXX.SPI_FLASH_BufferRead((uint8_t *)P_Rbuff, addr, size);
#endif
}
#if HAS_SPI_FLASH_FONT

4
Marlin/src/lcd/extui/lib/mks_ui/pic_manager.h
View file

@ -63,7 +63,7 @@ extern "C" { /* C-declarations for C++ */
#define DEFAULT_VIEW_MAX_SIZE (200*200*2)
#define FLASH_VIEW_MAX_SIZE (200*200*2)
#define PER_PIC_MAX_SPACE_TFT35 (32*1024)
#define PER_PIC_MAX_SPACE_TFT35 (9*1024)
#define PER_PIC_MAX_SPACE_TFT32 (16*1024)
#define PER_FONT_MAX_SPACE (16*1024)
@ -88,7 +88,7 @@ extern "C" { /* C-declarations for C++ */
#define PIC_OTHER_SIZE_ADDR_TFT32 0x5EE000
// font
#define FONTINFOADDR 0x183000 // 6M -- font addr
#define FONTINFOADDR 0x150000 // 6M -- font addr
#define UNIGBK_FLASH_ADDR (FONTINFOADDR+4096) // 4*1024
#else

4
Marlin/src/pins/stm32f1/pins_CHITU3D_V5.h
View file

@ -150,10 +150,10 @@
#define SPI_FLASH_SIZE 0x200000 // 2MB
#if HAS_TFT_LVGL_UI
#define HAS_SPI_FLASH_FONT 0
#define HAS_SPI_FLASH_FONT 1
#define HAS_GCODE_PREVIEW 1
#define HAS_GCODE_DEFAULT_VIEW_IN_FLASH 0
#define HAS_LANG_SELECT_SCREEN 0
#define HAS_LANG_SELECT_SCREEN 1
#define HAS_BAK_VIEW_IN_FLASH 0
#define HAS_LOGO_IN_FLASH 0