zaubentrucker/muele-marlin
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

✨ Configurations embed and retrieve (#21321, #23303)

Browse source
This commit is contained in:
X-Ryl669 2021-12-14 07:22:06 +01:00 committed by Scott Lahteine
parent f2ca70e232
commit 0dc1a58b24
15 changed files with 436 additions and 86 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.gitignore vendored
View file

@ -22,6 +22,9 @@
# Generated files
_Version.h
bdf2u8g
marlin_config.json
mczip.h
*.gen
*.sublime-workspace
#

8
Marlin/Configuration_adv.h
View file

@ -1593,6 +1593,14 @@
#define SD_FIRMWARE_UPDATE_INACTIVE_VALUE 0xFF
#endif
/**
* Enable this option if you have more than ~3K of unused flash space.
* Marlin will embed all settings in the firmware binary as compressed data.
* Use 'M503 C' to write the settings out to the SD Card as 'mc.zip'.
* See docs/ConfigEmbedding.md for details on how to use 'mc-apply.py'.
*/
//#define CONFIGURATION_EMBEDDING
// Add an optimized binary file transfer mode, initiated with 'M28 B1'
//#define BINARY_FILE_TRANSFER

18
Marlin/src/gcode/eeprom/M500-M504.cpp
View file

@ -25,6 +25,11 @@
#include "../../core/serial.h"
#include "../../inc/MarlinConfig.h"
#if ENABLED(CONFIGURATION_EMBEDDING)
#include "../../sd/SdBaseFile.h"
#include "../../mczip.h"
#endif
/**
* M500: Store settings in EEPROM
*/
@ -50,9 +55,22 @@ void GcodeSuite::M502() {
/**
* M503: print settings currently in memory
*
* With CONFIGURATION_EMBEDDING:
* C<flag> : Save the full Marlin configuration to SD Card as "mc.zip"
*/
void GcodeSuite::M503() {
(void)settings.report(!parser.boolval('S', true));
#if ENABLED(CONFIGURATION_EMBEDDING)
if (parser.seen_test('C')) {
SdBaseFile file;
const uint16_t size = sizeof(mc_zip);
// Need to create the config size on the SD card
if (file.open("mc.zip", O_WRITE|O_CREAT) && file.write(pgm_read_ptr(mc_zip), size) != -1 && file.close())
SERIAL_ECHO_MSG("Configuration saved as 'mc.zip'");
}
#endif
}
#endif // !DISABLE_M503

18
Marlin/src/gcode/host/M115.cpp
View file

@ -24,7 +24,6 @@
#include "../../inc/MarlinConfig.h"
#include "../queue.h" // for getting the command port
#if ENABLED(M115_GEOMETRY_REPORT)
#include "../../module/motion.h"
#endif
@ -33,13 +32,25 @@
#include "../../feature/caselight.h"
#endif
//#define MINIMAL_CAP_LINES // Don't even mention the disabled capabilities
#if ENABLED(EXTENDED_CAPABILITIES_REPORT)
static void cap_line(FSTR_P const name, bool ena=false) {
#if ENABLED(MINIMAL_CAP_LINES)
#define cap_line(S,C) if (C) _cap_line(S)
static void _cap_line(FSTR_P const name) {
SERIAL_ECHOPGM("Cap:");
SERIAL_ECHOF(name);
SERIAL_ECHOLNPGM(":1");
}
#else
#define cap_line(V...) _cap_line(V)
static void _cap_line(FSTR_P const name, bool ena=false) {
SERIAL_ECHOPGM("Cap:");
SERIAL_ECHOF(name);
SERIAL_CHAR(':', '0' + ena);
SERIAL_EOL();
}
#endif
#endif
/**
@ -167,6 +178,9 @@ void GcodeSuite::M115() {
// MEATPACK Compression
cap_line(F("MEATPACK"), SERIAL_IMPL.has_feature(port, SerialFeature::MeatPack));
// CONFIG_EXPORT
cap_line(F("CONFIG_EXPORT"), ENABLED(CONFIG_EMBED_AND_SAVE_TO_SD));
// Machine Geometry
#if ENABLED(M115_GEOMETRY_REPORT)
const xyz_pos_t bmin = { 0, 0, 0 },

6
Marlin/src/inc/Conditionals_adv.h
View file

@ -1004,3 +1004,9 @@
#if EITHER(MEATPACK_ON_SERIAL_PORT_1, MEATPACK_ON_SERIAL_PORT_2)
#define HAS_MEATPACK 1
#endif
// AVR are (usually) too limited in resources to store the configuration into the binary
#if !defined(FORCE_CONFIG_EMBED) && (defined(__AVR__) || DISABLED(SDSUPPORT) || EITHER(SDCARD_READONLY, DISABLE_M503))
#undef CONFIGURATION_EMBEDDING
#define CANNOT_EMBED_CONFIGURATION defined(__AVR__)
#endif

4
Marlin/src/inc/Warnings.cpp
View file

@ -549,3 +549,7 @@
#elif !USE_SENSORLESS && ENABLED(USES_DIAG_PINS)
#warning "Driver DIAG pins must be physically removed unless SENSORLESS_HOMING is enabled. (See https://bit.ly/2ZPRlt0)"
#endif
#if CANNOT_EMBED_CONFIGURATION
#warning "Disabled CONFIGURATION_EMBEDDING because the target usually has less flash storage. Define FORCE_CONFIG_EMBED to override."
#endif

2
buildroot/bin/restore_configs
View file

@ -2,4 +2,4 @@
git checkout Marlin/Configuration*.h 2>/dev/null
git checkout Marlin/src/pins/ramps/pins_RAMPS.h 2>/dev/null
rm -f Marlin/_Bootscreen.h Marlin/_Statusscreen.h
rm -f Marlin/_Bootscreen.h Marlin/_Statusscreen.h marlin_config.json .pio/build/mc.zip

1
buildroot/share/PlatformIO/scripts/SAMD51_grandcentral_m4.py
View file

@ -6,7 +6,6 @@ import pioutil
if pioutil.is_pio_build():
from os.path import join, isfile
import shutil
from pprint import pprint
Import("env")

0
buildroot/share/PlatformIO/scripts/__init__.py Normal file
View file

87
buildroot/share/PlatformIO/scripts/common-dependencies.py
View file

@ -192,63 +192,6 @@ if pioutil.is_pio_build():
lib_ignore = env.GetProjectOption('lib_ignore') + [feat['lib_ignore']]
set_env_field('lib_ignore', lib_ignore)
#
# Find a compiler, considering the OS
#
ENV_BUILD_PATH = os.path.join(env.Dictionary('PROJECT_BUILD_DIR'), env['PIOENV'])
GCC_PATH_CACHE = os.path.join(ENV_BUILD_PATH, ".gcc_path")
def search_compiler():
try:
filepath = env.GetProjectOption('custom_gcc')
blab("Getting compiler from env")
return filepath
except:
pass
if os.path.exists(GCC_PATH_CACHE):
with open(GCC_PATH_CACHE, 'r') as f:
return f.read()
# Find the current platform compiler by searching the $PATH
# which will be in a platformio toolchain bin folder
path_regex = re.escape(env['PROJECT_PACKAGES_DIR'])
# See if the environment provides a default compiler
try:
gcc = env.GetProjectOption('custom_deps_gcc')
except:
gcc = "g++"
if env['PLATFORM'] == 'win32':
path_separator = ';'
path_regex += r'.*\\bin'
gcc += ".exe"
else:
path_separator = ':'
path_regex += r'/.+/bin'
# Search for the compiler
for pathdir in env['ENV']['PATH'].split(path_separator):
if not re.search(path_regex, pathdir, re.IGNORECASE):
continue
for filepath in os.listdir(pathdir):
if not filepath.endswith(gcc):
continue
# Use entire path to not rely on env PATH
filepath = os.path.sep.join([pathdir, filepath])
# Cache the g++ path to no search always
if os.path.exists(ENV_BUILD_PATH):
with open(GCC_PATH_CACHE, 'w+') as f:
f.write(filepath)
return filepath
filepath = env.get('CXX')
if filepath == 'CC':
filepath = gcc
blab("Couldn't find a compiler! Fallback to %s" % filepath)
return filepath
#
# Use the compiler to get a list of all enabled features
#
@ -257,25 +200,8 @@ if pioutil.is_pio_build():
return
# Process defines
build_flags = env.get('BUILD_FLAGS')
build_flags = env.ParseFlagsExtended(build_flags)
cxx = search_compiler()
cmd = ['"' + cxx + '"']
# Build flags from board.json
#if 'BOARD' in env:
# cmd += [env.BoardConfig().get("build.extra_flags")]
for s in build_flags['CPPDEFINES']:
if isinstance(s, tuple):
cmd += ['-D' + s[0] + '=' + str(s[1])]
else:
cmd += ['-D' + s]
cmd += ['-D__MARLIN_DEPS__ -w -dM -E -x c++ buildroot/share/PlatformIO/scripts/common-dependencies.h']
cmd = ' '.join(cmd)
blab(cmd, 4)
define_list = subprocess.check_output(cmd, shell=True).splitlines()
from preprocessor import run_preprocessor
define_list = run_preprocessor(env)
marlin_features = {}
for define in define_list:
feature = define[8:].strip().decode().split(' ')
@ -310,9 +236,18 @@ if pioutil.is_pio_build():
except:
pass
#
# Add a method for other PIO scripts to query enabled features
#
env.AddMethod(MarlinFeatureIsEnabled)
#
# Add dependencies for enabled Marlin features
#
apply_features_config()
force_ignore_unused_libs()
#print(env.Dump())
from signature import compute_build_signature
compute_build_signature(env)

69
buildroot/share/PlatformIO/scripts/mc-apply.py Executable file
View file

@ -0,0 +1,69 @@
#!/usr/bin/env python
#
# Create a Configuration from marlin_config.json
#
import json
import sys
import shutil
import re
opt_output = '--opt' in sys.argv
output_suffix = '.sh' if opt_output else '' if '--bare-output' in sys.argv else '.gen'
try:
with open('marlin_config.json', 'r') as infile:
conf = json.load(infile)
for key in conf:
# We don't care about the hash when restoring here
if key == '__INITIAL_HASH':
continue
if key == 'VERSION':
for k, v in sorted(conf[key].items()):
print(k + ': ' + v)
continue
# The key is the file name, so let's build it now
outfile = open('Marlin/' + key + output_suffix, 'w')
for k, v in sorted(conf[key].items()):
# Make define line now
if opt_output:
if v != '':
if '"' in v:
v = "'%s'" % v
elif ' ' in v:
v = '"%s"' % v
define = 'opt_set ' + k + ' ' + v + '\n'
else:
define = 'opt_enable ' + k + '\n'
else:
define = '#define ' + k + ' ' + v + '\n'
outfile.write(define)
outfile.close()
# Try to apply changes to the actual configuration file (in order to keep useful comments)
if output_suffix != '':
# Move the existing configuration so it doesn't interfere
shutil.move('Marlin/' + key, 'Marlin/' + key + '.orig')
infile_lines = open('Marlin/' + key + '.orig', 'r').read().split('\n')
outfile = open('Marlin/' + key, 'w')
for line in infile_lines:
sline = line.strip(" \t\n\r")
if sline[:7] == "#define":
# Extract the key here (we don't care about the value)
kv = sline[8:].strip().split(' ')
if kv[0] in conf[key]:
outfile.write('#define ' + kv[0] + ' ' + conf[key][kv[0]] + '\n')
# Remove the key from the dict, so we can still write all missing keys at the end of the file
del conf[key][kv[0]]
else:
outfile.write(line + '\n')
else:
outfile.write(line + '\n')
# Process any remaining defines here
for k, v in sorted(conf[key].items()):
define = '#define ' + k + ' ' + v + '\n'
outfile.write(define)
outfile.close()
print('Output configuration written to: ' + 'Marlin/' + key + output_suffix)
except:
print('No marlin_config.json found.')

99
buildroot/share/PlatformIO/scripts/preprocessor.py Normal file
View file

@ -0,0 +1,99 @@
#
# preprocessor.py
#
import subprocess,os,re
verbose = 0
def blab(str):
if verbose:
print(str)
################################################################################
#
# Invoke GCC to run the preprocessor and extract enabled features
#
preprocessor_cache = {}
def run_preprocessor(env, fn=None):
filename = fn or 'buildroot/share/PlatformIO/scripts/common-dependencies.h'
if filename in preprocessor_cache:
return preprocessor_cache[filename]
# Process defines
build_flags = env.get('BUILD_FLAGS')
build_flags = env.ParseFlagsExtended(build_flags)
cxx = search_compiler(env)
cmd = ['"' + cxx + '"']
# Build flags from board.json
#if 'BOARD' in env:
# cmd += [env.BoardConfig().get("build.extra_flags")]
for s in build_flags['CPPDEFINES']:
if isinstance(s, tuple):
cmd += ['-D' + s[0] + '=' + str(s[1])]
else:
cmd += ['-D' + s]
cmd += ['-D__MARLIN_DEPS__ -w -dM -E -x c++']
depcmd = cmd + [ filename ]
cmd = ' '.join(depcmd)
blab(cmd)
define_list = subprocess.check_output(cmd, shell=True).splitlines()
preprocessor_cache[filename] = define_list
return define_list
################################################################################
#
# Find a compiler, considering the OS
#
def search_compiler(env):
ENV_BUILD_PATH = os.path.join(env.Dictionary('PROJECT_BUILD_DIR'), env['PIOENV'])
GCC_PATH_CACHE = os.path.join(ENV_BUILD_PATH, ".gcc_path")
try:
filepath = env.GetProjectOption('custom_gcc')
blab("Getting compiler from env")
return filepath
except:
pass
if os.path.exists(GCC_PATH_CACHE):
blab("Getting g++ path from cache")
with open(GCC_PATH_CACHE, 'r') as f:
return f.read()
# Find the current platform compiler by searching the $PATH
# which will be in a platformio toolchain bin folder
path_regex = re.escape(env['PROJECT_PACKAGES_DIR'])
gcc = "g++"
if env['PLATFORM'] == 'win32':
path_separator = ';'
path_regex += r'.*\\bin'
gcc += ".exe"
else:
path_separator = ':'
path_regex += r'/.+/bin'
# Search for the compiler
for pathdir in env['ENV']['PATH'].split(path_separator):
if not re.search(path_regex, pathdir, re.IGNORECASE):
continue
for filepath in os.listdir(pathdir):
if not filepath.endswith(gcc):
continue
# Use entire path to not rely on env PATH
filepath = os.path.sep.join([pathdir, filepath])
# Cache the g++ path to no search always
if os.path.exists(ENV_BUILD_PATH):
blab("Caching g++ for current env")
with open(GCC_PATH_CACHE, 'w+') as f:
f.write(filepath)
return filepath
filepath = env.get('CXX')
blab("Couldn't find a compiler! Fallback to %s" % filepath)
return filepath

176
buildroot/share/PlatformIO/scripts/signature.py Normal file
View file

@ -0,0 +1,176 @@
#
# signature.py
#
import os,subprocess,re,json,hashlib
#
# The dumbest preprocessor in the world
# Extract macro name from an header file and store them in an array
# No processing is done here, so they are raw values here and it does not match what actually enabled
# in the file (since you can have #if SOMETHING_UNDEFINED / #define BOB / #endif)
# But it's useful to filter the useful macro spit out by the preprocessor from noise from the system
# headers.
#
def extract_defines(filepath):
f = open(filepath, encoding="utf8").read().split("\n")
a = []
for line in f:
sline = line.strip(" \t\n\r")
if sline[:7] == "#define":
# Extract the key here (we don't care about the value)
kv = sline[8:].strip().split(' ')
a.append(kv[0])
return a
# Compute the SHA256 hash of a file
def get_file_sha256sum(filepath):
sha256_hash = hashlib.sha256()
with open(filepath,"rb") as f:
# Read and update hash string value in blocks of 4K
for byte_block in iter(lambda: f.read(4096),b""):
sha256_hash.update(byte_block)
return sha256_hash.hexdigest()
#
# Compress a JSON file into a zip file
#
import zipfile
def compress_file(filepath, outputbase):
with zipfile.ZipFile(outputbase + '.zip', 'w', compression=zipfile.ZIP_BZIP2, compresslevel=9) as zipf:
zipf.write(filepath, compress_type=zipfile.ZIP_BZIP2, compresslevel=9)
#
# Compute the build signature. The idea is to extract all defines in the configuration headers
# to build a unique reversible signature from this build so it can be included in the binary
# We can reverse the signature to get a 1:1 equivalent configuration file
#
def compute_build_signature(env):
if 'BUILD_SIGNATURE' in env:
return
# Definitions from these files will be kept
files_to_keep = [ 'Marlin/Configuration.h', 'Marlin/Configuration_adv.h' ]
build_dir=os.path.join(env['PROJECT_BUILD_DIR'], env['PIOENV'])
# Check if we can skip processing
hashes = ''
for header in files_to_keep:
hashes += get_file_sha256sum(header)[0:10]
marlin_json = os.path.join(build_dir, 'marlin_config.json')
marlin_zip = os.path.join(build_dir, 'mc')
# Read existing config file
try:
with open(marlin_json, 'r') as infile:
conf = json.load(infile)
if conf['__INITIAL_HASH'] == hashes:
# Same configuration, skip recomputing the building signature
compress_file(marlin_json, marlin_zip)
return
except:
pass
# Get enabled config options based on preprocessor
from preprocessor import run_preprocessor
complete_cfg = run_preprocessor(env)
# Dumb #define extraction from the configuration files
real_defines = {}
all_defines = []
for header in files_to_keep:
defines = extract_defines(header)
# To filter only the define we want
all_defines = all_defines + defines
# To remember from which file it cames from
real_defines[header.split('/')[-1]] = defines
r = re.compile(r"\(+(\s*-*\s*_.*)\)+")
# First step is to collect all valid macros
defines = {}
for line in complete_cfg:
# Split the define from the value
key_val = line[8:].strip().decode().split(' ')
key, value = key_val[0], ' '.join(key_val[1:])
# Ignore values starting with two underscore, since it's low level
if len(key) > 2 and key[0:2] == "__" :
continue
# Ignore values containing a parenthesis (likely a function macro)
if '(' in key and ')' in key:
continue
# Then filter dumb values
if r.match(value):
continue
defines[key] = value if len(value) else ""
if not 'CONFIGURATION_EMBEDDING' in defines:
return
# Second step is to filter useless macro
resolved_defines = {}
for key in defines:
# Remove all boards now
if key[0:6] == "BOARD_" and key != "BOARD_INFO_NAME":
continue
# Remove all keys ending by "_NAME" as it does not make a difference to the configuration
if key[-5:] == "_NAME" and key != "CUSTOM_MACHINE_NAME":
continue
# Remove all keys ending by "_T_DECLARED" as it's a copy of not important system stuff
if key[-11:] == "_T_DECLARED":
continue
# Remove keys that are not in the #define list in the Configuration list
if not (key in all_defines) and key != "DETAILED_BUILD_VERSION" and key != "STRING_DISTRIBUTION_DATE":
continue
# Don't be that smart guy here
resolved_defines[key] = defines[key]
# Generate a build signature now
# We are making an object that's a bit more complex than a basic dictionary here
data = {}
data['__INITIAL_HASH'] = hashes
# First create a key for each header here
for header in real_defines:
data[header] = {}
# Then populate the object where each key is going to (that's a O(N^2) algorithm here...)
for key in resolved_defines:
for header in real_defines:
if key in real_defines[header]:
data[header][key] = resolved_defines[key]
# Append the source code version and date
data['VERSION'] = {}
data['VERSION']['DETAILED_BUILD_VERSION'] = resolved_defines['DETAILED_BUILD_VERSION']
data['VERSION']['STRING_DISTRIBUTION_DATE'] = resolved_defines['STRING_DISTRIBUTION_DATE']
try:
curver = subprocess.check_output(["git", "describe", "--match=NeVeRmAtCh", "--always"]).strip()
data['VERSION']['GIT_REF'] = curver.decode()
except:
pass
with open(marlin_json, 'w') as outfile:
json.dump(data, outfile, separators=(',', ':'))
# Compress the JSON file as much as we can
compress_file(marlin_json, marlin_zip)
# Generate a C source file for storing this array
with open('Marlin/src/mczip.h','wb') as result_file:
result_file.write(b'#warning "Generated file \'mc.zip\' is embedded"\n')
result_file.write(b'const unsigned char mc_zip[] PROGMEM = {\n ')
count = 0
for b in open(os.path.join(build_dir, 'mc.zip'), 'rb').read():
result_file.write(b' 0x%02X,' % b)
count += 1
if (count % 16 == 0):
result_file.write(b'\n ')
if (count % 16):
result_file.write(b'\n')
result_file.write(b'};\n')

19
docs/ConfigEmbedding.md Normal file
View file

@ -0,0 +1,19 @@
# Configuration Embedding
Starting with version 2.0.9.3, Marlin automatically extracts the configuration used to generate the firmware and stores it in the firmware binary. This is enabled by defining `CONFIGURATION_EMBEDDING` in `Configuration_adv.h`.
## How it's done
To create the embedded configuration, we do a compiler pass to process the Configuration files and extract all active options. The active options are parsed into key/value pairs, serialized to JSON format, and stored in a file called `marlin_config.json`, which also includes specific build information (like the git revision, the build date, and some version information. The JSON file is then compressed in a ZIP archive called `.pio/build/mc.zip` which is converted into a C array and stored in a C++ file called `mc.h` which is included in the build.
## Extracting configurations from a Marlin binary
To get the configuration out of a binary firmware, you'll need a non-write-protected SD card inserted into the printer while running the firmware.
Send the command `M503 C` to write the file `mc.zip` to the SD card. Copy the file to your computer, ideally in the same folder as the Marlin repository.
Run the following commands to extract and apply the configuration:
```
$ git checkout -f
$ unzip mc.zip
$ python buildroot/share/PlatformIO/scripts/mc-apply.py
```
This will attempt to update the configuration files to match the settings used for the original build. It will also dump the git reference used to build the code (which may be accessible if the firmware was built from the main repository. As a fallback it also includes the `STRING_DISTRIBUTION_DATE` which is unlikely to be modified in a fork).