From 1363b439462af4fac2e9e3a8801abde5981575cf Mon Sep 17 00:00:00 2001 From: GHGiampy <83699429+GHGiampy@users.noreply.github.com> Date: Sun, 9 Jan 2022 08:14:03 +0100 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Firmware=20Upload=20via=20Binary=20?= =?UTF-8?q?Transfer=20(#23462)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Marlin/Configuration_adv.h | 5 + Marlin/src/gcode/sd/M20.cpp | 8 +- Marlin/src/pins/pins.h | 16 +- Marlin/src/sd/cardreader.cpp | 39 +- Marlin/src/sd/cardreader.h | 11 +- .../share/scripts/MarlinBinaryProtocol.py | 434 ++++++++++++++++++ buildroot/share/scripts/upload.py | 274 +++++++++++ ini/stm32f1.ini | 7 +- 8 files changed, 770 insertions(+), 24 deletions(-) create mode 100644 buildroot/share/scripts/MarlinBinaryProtocol.py create mode 100644 buildroot/share/scripts/upload.py diff --git a/Marlin/Configuration_adv.h b/Marlin/Configuration_adv.h index 16c5587f9c..d49d2d00b9 100644 --- a/Marlin/Configuration_adv.h +++ b/Marlin/Configuration_adv.h @@ -1621,6 +1621,11 @@ // Add an optimized binary file transfer mode, initiated with 'M28 B1' //#define BINARY_FILE_TRANSFER + #if ENABLED(BINARY_FILE_TRANSFER) + // Include extra facilities (e.g., 'M20 F') supporting firmware upload via BINARY_FILE_TRANSFER + //#define CUSTOM_FIRMWARE_UPLOAD + #endif + /** * Set this option to one of the following (or the board's defaults apply): * diff --git a/Marlin/src/gcode/sd/M20.cpp b/Marlin/src/gcode/sd/M20.cpp index 5731838338..c640309be8 100644 --- a/Marlin/src/gcode/sd/M20.cpp +++ b/Marlin/src/gcode/sd/M20.cpp @@ -33,7 +33,13 @@ void GcodeSuite::M20() { if (card.flag.mounted) { SERIAL_ECHOLNPGM(STR_BEGIN_FILE_LIST); - card.ls(TERN_(LONG_FILENAME_HOST_SUPPORT, parser.boolval('L'))); + card.ls( + TERN_(CUSTOM_FIRMWARE_UPLOAD, parser.boolval('F')) + #if BOTH(CUSTOM_FIRMWARE_UPLOAD, LONG_FILENAME_HOST_SUPPORT) + , + #endif + TERN_(LONG_FILENAME_HOST_SUPPORT, parser.boolval('L')) + ); SERIAL_ECHOLNPGM(STR_END_FILE_LIST); } else diff --git a/Marlin/src/pins/pins.h b/Marlin/src/pins/pins.h index 0d24ee6696..7ca78677e9 100644 --- a/Marlin/src/pins/pins.h +++ b/Marlin/src/pins/pins.h @@ -558,21 +558,21 @@ #elif MB(CHITU3D_V9) #include "stm32f1/pins_CHITU3D_V9.h" // STM32F1 env:chitu_f103 env:chitu_f103_maple #elif MB(CREALITY_V4) - #include "stm32f1/pins_CREALITY_V4.h" // STM32F1 env:STM32F103RET6_creality env:STM32F103RET6_creality_maple + #include "stm32f1/pins_CREALITY_V4.h" // STM32F1 env:STM32F103RET6_creality env:STM32F103RET6_creality_xfer env:STM32F103RET6_creality_maple #elif MB(CREALITY_V4210) - #include "stm32f1/pins_CREALITY_V4210.h" // STM32F1 env:STM32F103RET6_creality env:STM32F103RET6_creality_maple + #include "stm32f1/pins_CREALITY_V4210.h" // STM32F1 env:STM32F103RET6_creality env:STM32F103RET6_creality_xfer env:STM32F103RET6_creality_maple #elif MB(CREALITY_V423) - #include "stm32f1/pins_CREALITY_V423.h" // STM32F1 env:STM32F103RET6_creality + #include "stm32f1/pins_CREALITY_V423.h" // STM32F1 env:STM32F103RET6_creality env:STM32F103RET6_creality_xfer #elif MB(CREALITY_V427) - #include "stm32f1/pins_CREALITY_V427.h" // STM32F1 env:STM32F103RET6_creality env:STM32F103RET6_creality_maple + #include "stm32f1/pins_CREALITY_V427.h" // STM32F1 env:STM32F103RET6_creality env:STM32F103RET6_creality_xfer env:STM32F103RET6_creality_maple #elif MB(CREALITY_V431, CREALITY_V431_A, CREALITY_V431_B, CREALITY_V431_C, CREALITY_V431_D) - #include "stm32f1/pins_CREALITY_V431.h" // STM32F1 env:STM32F103RET6_creality env:STM32F103RET6_creality_maple + #include "stm32f1/pins_CREALITY_V431.h" // STM32F1 env:STM32F103RET6_creality env:STM32F103RET6_creality_xfer env:STM32F103RET6_creality_maple #elif MB(CREALITY_V452) - #include "stm32f1/pins_CREALITY_V452.h" // STM32F1 env:STM32F103RET6_creality env:STM32F103RET6_creality_maple + #include "stm32f1/pins_CREALITY_V452.h" // STM32F1 env:STM32F103RET6_creality env:STM32F103RET6_creality_xfer env:STM32F103RET6_creality_maple #elif MB(CREALITY_V453) - #include "stm32f1/pins_CREALITY_V453.h" // STM32F1 env:STM32F103RET6_creality env:STM32F103RET6_creality_maple + #include "stm32f1/pins_CREALITY_V453.h" // STM32F1 env:STM32F103RET6_creality env:STM32F103RET6_creality_xfer env:STM32F103RET6_creality_maple #elif MB(CREALITY_V24S1) - #include "stm32f1/pins_CREALITY_V24S1.h" // STM32F1 env:STM32F103RET6_creality env:STM32F103RET6_creality_maple + #include "stm32f1/pins_CREALITY_V24S1.h" // STM32F1 env:STM32F103RET6_creality env:STM32F103RET6_creality_xfer env:STM32F103RET6_creality_maple #elif MB(TRIGORILLA_PRO) #include "stm32f1/pins_TRIGORILLA_PRO.h" // STM32F1 env:trigorilla_pro env:trigorilla_pro_maple #elif MB(FLY_MINI) diff --git a/Marlin/src/sd/cardreader.cpp b/Marlin/src/sd/cardreader.cpp index 66c08b6455..25f9d7d802 100644 --- a/Marlin/src/sd/cardreader.cpp +++ b/Marlin/src/sd/cardreader.cpp @@ -195,11 +195,15 @@ char *createFilename(char * const buffer, const dir_t &p) { } // -// Return 'true' if the item is a folder or G-code file +// Return 'true' if the item is something Marlin can read // -bool CardReader::is_dir_or_gcode(const dir_t &p) { +bool CardReader::is_visible_entity(const dir_t &p OPTARG(CUSTOM_FIRMWARE_UPLOAD, bool onlyBin/*=false*/)) { //uint8_t pn0 = p.name[0]; + #if DISABLED(CUSTOM_FIRMWARE_UPLOAD) + constexpr bool onlyBin = false; + #endif + if ( (p.attributes & DIR_ATT_HIDDEN) // Hidden by attribute // When readDir() > 0 these must be false: //|| pn0 == DIR_NAME_FREE || pn0 == DIR_NAME_DELETED // Clear or Deleted entry @@ -211,7 +215,11 @@ bool CardReader::is_dir_or_gcode(const dir_t &p) { return ( flag.filenameIsDir // All Directories are ok - || (p.name[8] == 'G' && p.name[9] != '~') // Non-backup *.G* files are accepted + || (!onlyBin && p.name[8] == 'G' + && p.name[9] != '~') // Non-backup *.G* files are accepted + || ( onlyBin && p.name[8] == 'B' + && p.name[9] == 'I' + && p.name[10] == 'N') // BIN files are accepted ); } @@ -222,7 +230,7 @@ int CardReader::countItems(SdFile dir) { dir_t p; int c = 0; while (dir.readDir(&p, longFilename) > 0) - c += is_dir_or_gcode(p); + c += is_visible_entity(p); #if ALL(SDCARD_SORT_ALPHA, SDSORT_USES_RAM, SDSORT_CACHE_NAMES) nrFiles = c; @@ -237,7 +245,7 @@ int CardReader::countItems(SdFile dir) { void CardReader::selectByIndex(SdFile dir, const uint8_t index) { dir_t p; for (uint8_t cnt = 0; dir.readDir(&p, longFilename) > 0;) { - if (is_dir_or_gcode(p)) { + if (is_visible_entity(p)) { if (cnt == index) { createFilename(filename, p); return; // 0 based index @@ -253,7 +261,7 @@ void CardReader::selectByIndex(SdFile dir, const uint8_t index) { void CardReader::selectByName(SdFile dir, const char * const match) { dir_t p; for (uint8_t cnt = 0; dir.readDir(&p, longFilename) > 0; cnt++) { - if (is_dir_or_gcode(p)) { + if (is_visible_entity(p)) { createFilename(filename, p); if (strcasecmp(match, filename) == 0) return; } @@ -272,6 +280,7 @@ void CardReader::selectByName(SdFile dir, const char * const match) { */ void CardReader::printListing( SdFile parent, const char * const prepend + OPTARG(CUSTOM_FIRMWARE_UPLOAD, bool onlyBin/*=false*/) OPTARG(LONG_FILENAME_HOST_SUPPORT, const bool includeLongNames/*=false*/) OPTARG(LONG_FILENAME_HOST_SUPPORT, const char * const prependLong/*=nullptr*/) ) { @@ -297,12 +306,12 @@ void CardReader::printListing( char pathLong[lenPrependLong + strlen(longFilename) + 1]; if (prependLong) { strcpy(pathLong, prependLong); pathLong[lenPrependLong - 1] = '/'; } strcpy(pathLong + lenPrependLong, longFilename); - printListing(child, path, true, pathLong); + printListing(child, path OPTARG(CUSTOM_FIRMWARE_UPLOAD, onlyBin), true, pathLong); } else - printListing(child, path); + printListing(child, path OPTARG(CUSTOM_FIRMWARE_UPLOAD, onlyBin)); #else - printListing(child, path); + printListing(child, path OPTARG(CUSTOM_FIRMWARE_UPLOAD, onlyBin)); #endif } else { @@ -310,7 +319,7 @@ void CardReader::printListing( return; } } - else if (is_dir_or_gcode(p)) { + else if (is_visible_entity(p OPTARG(CUSTOM_FIRMWARE_UPLOAD, onlyBin))) { if (prepend) { SERIAL_ECHO(prepend); SERIAL_CHAR('/'); } SERIAL_ECHO(createFilename(filename, p)); SERIAL_CHAR(' '); @@ -330,10 +339,16 @@ void CardReader::printListing( // // List all files on the SD card // -void CardReader::ls(TERN_(LONG_FILENAME_HOST_SUPPORT, bool includeLongNames/*=false*/)) { +void CardReader::ls( + TERN_(CUSTOM_FIRMWARE_UPLOAD, const bool onlyBin/*=false*/) + #if BOTH(CUSTOM_FIRMWARE_UPLOAD, LONG_FILENAME_HOST_SUPPORT) + , + #endif + TERN_(LONG_FILENAME_HOST_SUPPORT, const bool includeLongNames/*=false*/) +) { if (flag.mounted) { root.rewind(); - printListing(root, nullptr OPTARG(LONG_FILENAME_HOST_SUPPORT, includeLongNames)); + printListing(root, nullptr OPTARG(CUSTOM_FIRMWARE_UPLOAD, onlyBin) OPTARG(LONG_FILENAME_HOST_SUPPORT, includeLongNames)); } } diff --git a/Marlin/src/sd/cardreader.h b/Marlin/src/sd/cardreader.h index 8761f57de5..2b3dcd00fb 100644 --- a/Marlin/src/sd/cardreader.h +++ b/Marlin/src/sd/cardreader.h @@ -204,7 +204,13 @@ public: FORCE_INLINE static void getfilename_sorted(const uint16_t nr) { selectFileByIndex(nr); } #endif - static void ls(TERN_(LONG_FILENAME_HOST_SUPPORT, bool includeLongNames=false)); + static void ls( + TERN_(CUSTOM_FIRMWARE_UPLOAD, const bool onlyBin=false) + #if BOTH(CUSTOM_FIRMWARE_UPLOAD, LONG_FILENAME_HOST_SUPPORT) + , + #endif + TERN_(LONG_FILENAME_HOST_SUPPORT, const bool includeLongNames=false) + ); #if ENABLED(POWER_LOSS_RECOVERY) static bool jobRecoverFileExists(); @@ -331,12 +337,13 @@ private: // // Directory items // - static bool is_dir_or_gcode(const dir_t &p); + static bool is_visible_entity(const dir_t &p OPTARG(CUSTOM_FIRMWARE_UPLOAD, const bool onlyBin=false)); static int countItems(SdFile dir); static void selectByIndex(SdFile dir, const uint8_t index); static void selectByName(SdFile dir, const char * const match); static void printListing( SdFile parent, const char * const prepend + OPTARG(CUSTOM_FIRMWARE_UPLOAD, const bool onlyBin=false) OPTARG(LONG_FILENAME_HOST_SUPPORT, const bool includeLongNames=false) OPTARG(LONG_FILENAME_HOST_SUPPORT, const char * const prependLong=nullptr) ); diff --git a/buildroot/share/scripts/MarlinBinaryProtocol.py b/buildroot/share/scripts/MarlinBinaryProtocol.py new file mode 100644 index 0000000000..4887ad9919 --- /dev/null +++ b/buildroot/share/scripts/MarlinBinaryProtocol.py @@ -0,0 +1,434 @@ +# +# MarlinBinaryProtocol.py +# Supporting Firmware upload via USB/Serial, saving to the attached media. +# +import serial +import math +import time +from collections import deque +import threading +import sys +import datetime +import random +try: + import heatshrink + heatshrink_exists = True +except ImportError: + heatshrink_exists = False + + +def millis(): + return time.perf_counter() * 1000 + +class TimeOut(object): + def __init__(self, milliseconds): + self.duration = milliseconds + self.reset() + + def reset(self): + self.endtime = millis() + self.duration + + def timedout(self): + return millis() > self.endtime + +class ReadTimeout(Exception): + pass +class FatalError(Exception): + pass +class SycronisationError(Exception): + pass +class PayloadOverflow(Exception): + pass +class ConnectionLost(Exception): + pass + +class Protocol(object): + device = None + baud = None + max_block_size = 0 + port = None + block_size = 0 + + packet_transit = None + packet_status = None + packet_ping = None + + errors = 0 + packet_buffer = None + simulate_errors = 0 + sync = 0 + connected = False + syncronised = False + worker_thread = None + + response_timeout = 1000 + + applications = [] + responses = deque() + + def __init__(self, device, baud, bsize, simerr, timeout): + print("pySerial Version:", serial.VERSION) + self.port = serial.Serial(device, baudrate = baud, write_timeout = 0, timeout = 1) + self.device = device + self.baud = baud + self.block_size = int(bsize) + self.simulate_errors = max(min(simerr, 1.0), 0.0); + self.connected = True + self.response_timeout = timeout + + self.register(['ok', 'rs', 'ss', 'fe'], self.process_input) + + self.worker_thread = threading.Thread(target=Protocol.receive_worker, args=(self,)) + self.worker_thread.start() + + def receive_worker(self): + while self.port.in_waiting: + self.port.reset_input_buffer() + + def dispatch(data): + for tokens, callback in self.applications: + for token in tokens: + if token == data[:len(token)]: + callback((token, data[len(token):])) + return + + def reconnect(): + print("Reconnecting..") + self.port.close() + for x in range(10): + try: + if self.connected: + self.port = serial.Serial(self.device, baudrate = self.baud, write_timeout = 0, timeout = 1) + return + else: + print("Connection closed") + return + except: + time.sleep(1) + raise ConnectionLost() + + while self.connected: + try: + data = self.port.readline().decode('utf8').rstrip() + if len(data): + #print(data) + dispatch(data) + except OSError: + reconnect() + except UnicodeDecodeError: + # dodgy client output or datastream corruption + self.port.reset_input_buffer() + + def shutdown(self): + self.connected = False + self.worker_thread.join() + self.port.close() + + def process_input(self, data): + #print(data) + self.responses.append(data) + + def register(self, tokens, callback): + self.applications.append((tokens, callback)) + + def send(self, protocol, packet_type, data = bytearray()): + self.packet_transit = self.build_packet(protocol, packet_type, data) + self.packet_status = 0 + self.transmit_attempt = 0 + + timeout = TimeOut(self.response_timeout * 20) + while self.packet_status == 0: + try: + if timeout.timedout(): + raise ConnectionLost() + self.transmit_packet(self.packet_transit) + self.await_response() + except ReadTimeout: + self.errors += 1 + #print("Packetloss detected..") + self.packet_transit = None + + def await_response(self): + timeout = TimeOut(self.response_timeout) + while not len(self.responses): + time.sleep(0.00001) + if timeout.timedout(): + raise ReadTimeout() + + while len(self.responses): + token, data = self.responses.popleft() + switch = {'ok' : self.response_ok, 'rs': self.response_resend, 'ss' : self.response_stream_sync, 'fe' : self.response_fatal_error} + switch[token](data) + + def send_ascii(self, data, send_and_forget = False): + self.packet_transit = bytearray(data, "utf8") + b'\n' + self.packet_status = 0 + self.transmit_attempt = 0 + + timeout = TimeOut(self.response_timeout * 20) + while self.packet_status == 0: + try: + if timeout.timedout(): + return + self.port.write(self.packet_transit) + if send_and_forget: + self.packet_status = 1 + else: + self.await_response_ascii() + except ReadTimeout: + self.errors += 1 + #print("Packetloss detected..") + except serial.serialutil.SerialException: + return + self.packet_transit = None + + def await_response_ascii(self): + timeout = TimeOut(self.response_timeout) + while not len(self.responses): + time.sleep(0.00001) + if timeout.timedout(): + raise ReadTimeout() + token, data = self.responses.popleft() + self.packet_status = 1 + + def corrupt_array(self, data): + rid = random.randint(0, len(data) - 1) + data[rid] ^= 0xAA + return data + + def transmit_packet(self, packet): + packet = bytearray(packet) + if(self.simulate_errors > 0 and random.random() > (1.0 - self.simulate_errors)): + if random.random() > 0.9: + #random data drop + start = random.randint(0, len(packet)) + end = start + random.randint(1, 10) + packet = packet[:start] + packet[end:] + #print("Dropping {0} bytes".format(end - start)) + else: + #random corruption + packet = self.corrupt_array(packet) + #print("Single byte corruption") + self.port.write(packet) + self.transmit_attempt += 1 + + def build_packet(self, protocol, packet_type, data = bytearray()): + PACKET_TOKEN = 0xB5AD + + if len(data) > self.max_block_size: + raise PayloadOverflow() + + packet_buffer = bytearray() + + packet_buffer += self.pack_int8(self.sync) # 8bit sync id + packet_buffer += self.pack_int4_2(protocol, packet_type) # 4 bit protocol id, 4 bit packet type + packet_buffer += self.pack_int16(len(data)) # 16bit packet length + packet_buffer += self.pack_int16(self.build_checksum(packet_buffer)) # 16bit header checksum + + if len(data): + packet_buffer += data + packet_buffer += self.pack_int16(self.build_checksum(packet_buffer)) + + packet_buffer = self.pack_int16(PACKET_TOKEN) + packet_buffer # 16bit start token, not included in checksum + return packet_buffer + + # checksum 16 fletchers + def checksum(self, cs, value): + cs_low = (((cs & 0xFF) + value) % 255); + return ((((cs >> 8) + cs_low) % 255) << 8) | cs_low; + + def build_checksum(self, buffer): + cs = 0 + for b in buffer: + cs = self.checksum(cs, b) + return cs + + def pack_int32(self, value): + return value.to_bytes(4, byteorder='little') + + def pack_int16(self, value): + return value.to_bytes(2, byteorder='little') + + def pack_int8(self, value): + return value.to_bytes(1, byteorder='little') + + def pack_int4_2(self, vh, vl): + value = ((vh & 0xF) << 4) | (vl & 0xF) + return value.to_bytes(1, byteorder='little') + + def connect(self): + print("Connecting: Switching Marlin to Binary Protocol...") + self.send_ascii("M28B1") + self.send(0, 1) + + def disconnect(self): + self.send(0, 2) + self.syncronised = False + + def response_ok(self, data): + try: + packet_id = int(data); + except ValueError: + return + if packet_id != self.sync: + raise SycronisationError() + self.sync = (self.sync + 1) % 256 + self.packet_status = 1 + + def response_resend(self, data): + packet_id = int(data); + self.errors += 1 + if not self.syncronised: + print("Retrying syncronisation") + elif packet_id != self.sync: + raise SycronisationError() + + def response_stream_sync(self, data): + sync, max_block_size, protocol_version = data.split(',') + self.sync = int(sync) + self.max_block_size = int(max_block_size) + self.block_size = self.max_block_size if self.max_block_size < self.block_size else self.block_size + self.protocol_version = protocol_version + self.packet_status = 1 + self.syncronised = True + print("Connection synced [{0}], binary protocol version {1}, {2} byte payload buffer".format(self.sync, self.protocol_version, self.max_block_size)) + + def response_fatal_error(self, data): + raise FatalError() + + +class FileTransferProtocol(object): + protocol_id = 1 + + class Packet(object): + QUERY = 0 + OPEN = 1 + CLOSE = 2 + WRITE = 3 + ABORT = 4 + + responses = deque() + def __init__(self, protocol, timeout = None): + protocol.register(['PFT:success', 'PFT:version:', 'PFT:fail', 'PFT:busy', 'PFT:ioerror', 'PTF:invalid'], self.process_input) + self.protocol = protocol + self.response_timeout = timeout or protocol.response_timeout + + def process_input(self, data): + #print(data) + self.responses.append(data) + + def await_response(self, timeout = None): + timeout = TimeOut(timeout or self.response_timeout) + while not len(self.responses): + time.sleep(0.0001) + if timeout.timedout(): + raise ReadTimeout() + + return self.responses.popleft() + + def connect(self): + self.protocol.send(FileTransferProtocol.protocol_id, FileTransferProtocol.Packet.QUERY); + + token, data = self.await_response() + if token != 'PFT:version:': + return False + + self.version, _, compression = data.split(':') + if compression != 'none': + algorithm, window, lookahead = compression.split(',') + self.compression = {'algorithm': algorithm, 'window': int(window), 'lookahead': int(lookahead)} + else: + self.compression = {'algorithm': 'none'} + + print("File Transfer version: {0}, compression: {1}".format(self.version, self.compression['algorithm'])) + + def open(self, filename, compression, dummy): + payload = b'\1' if dummy else b'\0' # dummy transfer + payload += b'\1' if compression else b'\0' # payload compression + payload += bytearray(filename, 'utf8') + b'\0'# target filename + null terminator + + timeout = TimeOut(5000) + token = None + self.protocol.send(FileTransferProtocol.protocol_id, FileTransferProtocol.Packet.OPEN, payload); + while token != 'PFT:success' and not timeout.timedout(): + try: + token, data = self.await_response(1000) + if token == 'PFT:success': + print(filename,"opened") + return + elif token == 'PFT:busy': + print("Broken transfer detected, purging") + self.abort() + time.sleep(0.1) + self.protocol.send(FileTransferProtocol.protocol_id, FileTransferProtocol.Packet.OPEN, payload); + timeout.reset() + elif token == 'PFT:fail': + raise Exception("Can not open file on client") + except ReadTimeout: + pass + raise ReadTimeout() + + def write(self, data): + self.protocol.send(FileTransferProtocol.protocol_id, FileTransferProtocol.Packet.WRITE, data); + + def close(self): + self.protocol.send(FileTransferProtocol.protocol_id, FileTransferProtocol.Packet.CLOSE); + token, data = self.await_response(1000) + if token == 'PFT:success': + print("File closed") + return + elif token == 'PFT:ioerror': + print("Client storage device IO error") + elif token == 'PFT:invalid': + print("No open file") + + def abort(self): + self.protocol.send(FileTransferProtocol.protocol_id, FileTransferProtocol.Packet.ABORT); + token, data = self.await_response() + if token == 'PFT:success': + print("Transfer Aborted") + + def copy(self, filename, dest_filename, compression, dummy): + self.connect() + + compression_support = heatshrink_exists and self.compression['algorithm'] == 'heatshrink' and compression + if compression and (not heatshrink_exists or not self.compression['algorithm'] == 'heatshrink'): + print("Compression not supported by client") + #compression_support = False + + data = open(filename, "rb").read() + filesize = len(data) + + self.open(dest_filename, compression_support, dummy) + + block_size = self.protocol.block_size + if compression_support: + data = heatshrink.encode(data, window_sz2=self.compression['window'], lookahead_sz2=self.compression['lookahead']) + + cratio = filesize / len(data) + + blocks = math.floor((len(data) + block_size - 1) / block_size) + kibs = 0 + dump_pctg = 0 + start_time = millis() + for i in range(blocks): + start = block_size * i + end = start + block_size + self.write(data[start:end]) + kibs = (( (i+1) * block_size) / 1024) / (millis() + 1 - start_time) * 1000 + if (i / blocks) >= dump_pctg: + print("\r{0:2.2f}% {1:4.2f}KiB/s {2} Errors: {3}".format((i / blocks) * 100, kibs, "[{0:4.2f}KiB/s]".format(kibs * cratio) if compression_support else "", self.protocol.errors), end='') + dump_pctg += 0.1 + print("\r{0:2.2f}% {1:4.2f}KiB/s {2} Errors: {3}".format(100, kibs, "[{0:4.2f}KiB/s]".format(kibs * cratio) if compression_support else "", self.protocol.errors)) # no one likes transfers finishing at 99.8% + + self.close() + print("Transfer complete") + + +class EchoProtocol(object): + def __init__(self, protocol): + protocol.register(['echo:'], self.process_input) + self.protocol = protocol + + def process_input(self, data): + print(data) diff --git a/buildroot/share/scripts/upload.py b/buildroot/share/scripts/upload.py new file mode 100644 index 0000000000..4a152067a4 --- /dev/null +++ b/buildroot/share/scripts/upload.py @@ -0,0 +1,274 @@ +import argparse +import sys +import os +import time +import random +import serial + +Import("env") + +# Needed (only) for compression, but there are problems with pip install heatshrink +#try: +# import heatshrink +#except ImportError: +# # Install heatshrink +# print("Installing 'heatshrink' python module...") +# env.Execute(env.subst("$PYTHONEXE -m pip install heatshrink")) +# +# Not tested: If it's safe to install python libraries in PIO python try: +# env.Execute(env.subst("$PYTHONEXE -m pip install https://github.com/p3p/pyheatshrink/releases/download/0.3.3/pyheatshrink-pip.zip")) + +import MarlinBinaryProtocol + +# Internal debug flag +Debug = False + +#-----------------# +# Upload Callback # +#-----------------# +def Upload(source, target, env): + + #------------------# + # Marlin functions # + #------------------# + def _GetMarlinEnv(marlinEnv, feature): + if not marlinEnv: return None + return marlinEnv[feature] if feature in marlinEnv else None + + #----------------# + # Port functions # + #----------------# + def _GetUploadPort(env): + if Debug: print('Autodetecting upload port...') + env.AutodetectUploadPort(env) + port = env.subst('$UPLOAD_PORT') + if not port: + raise Exception('Error detecting the upload port.') + if Debug: print('OK') + return port + + #-------------------------# + # Simple serial functions # + #-------------------------# + def _Send(data): + if Debug: print(f'>> {data}') + strdata = bytearray(data, 'utf8') + b'\n' + port.write(strdata) + time.sleep(0.010) + + def _Recv(): + clean_responses = [] + responses = port.readlines() + for Resp in responses: + # Test: suppress invaid chars (coming from debug info) + try: + clean_response = Resp.decode('utf8').rstrip().lstrip() + clean_responses.append(clean_response) + except: + pass + if Debug: print(f'<< {clean_response}') + return clean_responses + + #------------------# + # SDCard functions # + #------------------# + def _CheckSDCard(): + if Debug: print('Checking SD card...') + _Send('M21') + Responses = _Recv() + if len(Responses) < 1 or not any('SD card ok' in r for r in Responses): + raise Exception('Error accessing SD card') + if Debug: print('SD Card OK') + return True + + #----------------# + # File functions # + #----------------# + def _GetFirmwareFiles(): + if Debug: print('Get firmware files...') + _Send('M20 F') + Responses = _Recv() + if len(Responses) < 3 or not any('file list' in r for r in Responses): + raise Exception('Error getting firmware files') + if Debug: print('OK') + return Responses + + def _FilterFirmwareFiles(FirmwareList): + Firmwares = [] + for FWFile in FirmwareList: + if not '/' in FWFile and '.BIN' in FWFile: + idx = FWFile.index('.BIN') + Firmwares.append(FWFile[:idx+4]) + return Firmwares + + def _RemoveFirmwareFile(FirmwareFile): + _Send(f'M30 /{FirmwareFile}') + Responses = _Recv() + Removed = len(Responses) >= 1 and any('File deleted' in r for r in Responses) + if not Removed: + raise Exception(f"Firmware file '{FirmwareFile}' not removed") + return Removed + + + #---------------------# + # Callback Entrypoint # + #---------------------# + port = None + protocol = None + filetransfer = None + + # Get Marlin evironment vars + MarlinEnv = env['MARLIN_FEATURES'] + marlin_pioenv = _GetMarlinEnv(MarlinEnv, 'PIOENV') + marlin_motherboard = _GetMarlinEnv(MarlinEnv, 'MOTHERBOARD') + marlin_board_info_name = _GetMarlinEnv(MarlinEnv, 'BOARD_INFO_NAME') + marlin_board_custom_build_flags = _GetMarlinEnv(MarlinEnv, 'BOARD_CUSTOM_BUILD_FLAGS') + marlin_firmware_bin = _GetMarlinEnv(MarlinEnv, 'FIRMWARE_BIN') + marlin_custom_firmware_upload = _GetMarlinEnv(MarlinEnv, 'CUSTOM_FIRMWARE_UPLOAD') is not None + marlin_short_build_version = _GetMarlinEnv(MarlinEnv, 'SHORT_BUILD_VERSION') + marlin_string_config_h_author = _GetMarlinEnv(MarlinEnv, 'STRING_CONFIG_H_AUTHOR') + + # Get firmware upload params + upload_firmware_source_name = str(source[0]) # Source firmware filename + upload_speed = env['UPLOAD_SPEED'] if 'UPLOAD_SPEED' in env else 115200 + # baud rate of serial connection + upload_port = _GetUploadPort(env) # Serial port to use + + # Set local upload params + upload_firmware_target_name = os.path.basename(upload_firmware_source_name) # WARNING! Need rework on "binary_stream" to allow filename > 8.3 + # Target firmware filename + upload_timeout = 1000 # Communication timout, lossy/slow connections need higher values + upload_blocksize = 512 # Transfer block size. 512 = Autodetect + upload_compression = True # Enable compression + upload_error_ratio = 0 # Simulated corruption ratio + upload_test = False # Benchmark the serial link without storing the file + upload_reset = True # Trigger a soft reset for firmware update after the upload + + # Set local upload params based on board type to change script behavior + # "upload_delete_old_bins": delete all *.bin files in the root of SD Card + upload_delete_old_bins = marlin_motherboard in ['BOARD_CREALITY_V4', 'BOARD_CREALITY_V4210', 'BOARD_CREALITY_V423', 'BOARD_CREALITY_V427', + 'BOARD_CREALITY_V431', 'BOARD_CREALITY_V452', 'BOARD_CREALITY_V453', 'BOARD_CREALITY_V24S1'] + try: + + # Start upload job + print(f"Uploading firmware '{os.path.basename(upload_firmware_target_name)}' to '{marlin_motherboard}' via '{upload_port}'") + + # Dump some debug info + if Debug: + print('Upload using:') + print('---- Marlin --------------------') + print(f' PIOENV : {marlin_pioenv}') + print(f' SHORT_BUILD_VERSION : {marlin_short_build_version}') + print(f' STRING_CONFIG_H_AUTHOR : {marlin_string_config_h_author}') + print(f' MOTHERBOARD : {marlin_motherboard}') + print(f' BOARD_INFO_NAME : {marlin_board_info_name}') + print(f' CUSTOM_BUILD_FLAGS : {marlin_board_custom_build_flags}') + print(f' FIRMWARE_BIN : {marlin_firmware_bin}') + print(f' CUSTOM_FIRMWARE_UPLOAD : {marlin_custom_firmware_upload}') + print('---- Upload parameters ---------') + print(f' Source : {upload_firmware_source_name}') + print(f' Target : {upload_firmware_target_name}') + print(f' Port : {upload_port} @ {upload_speed} baudrate') + print(f' Timeout : {upload_timeout}') + print(f' Block size : {upload_blocksize}') + print(f' Compression : {upload_compression}') + print(f' Error ratio : {upload_error_ratio}') + print(f' Test : {upload_test}') + print(f' Reset : {upload_reset}') + print('--------------------------------') + + # Custom implementations based on board parameters + + # Delete all *.bin files on the root of SD Card (if flagged) + if upload_delete_old_bins: + # CUSTOM_FIRMWARE_UPLOAD is needed for this feature + if not marlin_custom_firmware_upload: + raise Exception(f"CUSTOM_FIRMWARE_UPLOAD must be enabled in 'Configuration_adv.h' for '{marlin_motherboard}'") + + # Generate a new 8.3 random filename + # This board remember the last firmware filename and doesn't allow to flash from that filename + upload_firmware_target_name = f"fw-{''.join(random.choices('ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789', k=5))}.BIN" + print(f"Board {marlin_motherboard}: Overriding firmware filename to '{upload_firmware_target_name}'") + + # Init serial port + port = serial.Serial(upload_port, baudrate = upload_speed, write_timeout = 0, timeout = 0.1) + port.reset_input_buffer() + + # Check SD card status + _CheckSDCard() + + # Get firmware files + FirmwareFiles = _GetFirmwareFiles() + if Debug: + for FirmwareFile in FirmwareFiles: + print(f'Found: {FirmwareFile}') + + # Get all 1st level firmware files (to remove) + OldFirmwareFiles = _FilterFirmwareFiles(FirmwareFiles[1:len(FirmwareFiles)-2]) # Skip header and footers of list + if len(OldFirmwareFiles) == 0: + print('No old firmware files to delete') + else: + print(f"Remove {len(OldFirmwareFiles)} old firmware file{'s' if len(OldFirmwareFiles) != 1 else ''}:") + for OldFirmwareFile in OldFirmwareFiles: + print(f" -Removing- '{OldFirmwareFile}'...") + print(' OK' if _RemoveFirmwareFile(OldFirmwareFile) else ' Error!') + + # Close serial + port.close() + + # Cleanup completed + if Debug: print('Cleanup completed') + + # WARNING! The serial port must be closed here because the serial transfer that follow needs it! + + # Upload firmware file + if Debug: print(f"Copy '{upload_firmware_source_name}' --> '{upload_firmware_target_name}'") + protocol = MarlinBinaryProtocol.Protocol(upload_port, upload_speed, upload_blocksize, float(upload_error_ratio), int(upload_timeout)) + #echologger = MarlinBinaryProtocol.EchoProtocol(protocol) + protocol.connect() + filetransfer = MarlinBinaryProtocol.FileTransferProtocol(protocol) + filetransfer.copy(upload_firmware_source_name, upload_firmware_target_name, upload_compression, upload_test) + protocol.disconnect() + + # Notify upload completed + protocol.send_ascii('M117 Firmware uploaded') + + # Remount SD card + print('Wait for SD card release...') + time.sleep(1) + print('Remount SD card') + protocol.send_ascii('M21') + + # Trigger firmware update + if upload_reset: + print('Trigger firmware update...') + protocol.send_ascii('M997', True) + + protocol: protocol.shutdown() + print('Firmware update completed') + + except KeyboardInterrupt: + if port: port.close() + if filetransfer: filetransfer.abort() + if protocol: protocol.shutdown() + raise + + except serial.SerialException as se: + if port: port.close() + print(f'Serial excepion: {se}') + raise Exception(se) + + except MarlinBinaryProtocol.FatalError: + if port: port.close() + if protocol: protocol.shutdown() + print('Too many retries, Abort') + raise + + except: + if port: port.close() + if protocol: protocol.shutdown() + print('Firmware not updated') + raise + +# Attach custom upload callback +env.Replace(UPLOADCMD=Upload) diff --git a/ini/stm32f1.ini b/ini/stm32f1.ini index 939f51ffbf..a0957dbaec 100644 --- a/ini/stm32f1.ini +++ b/ini/stm32f1.ini @@ -100,7 +100,6 @@ build_flags = ${common_STM32F103RC_variant.build_flags} -DTIMER_SERVO=TIM5 -DDEFAULT_SPI=3 build_unflags = ${common_STM32F103RC_variant.build_unflags} -DUSBCON -DUSBD_USE_CDC -monitor_speed = 115200 debug_tool = stlink # @@ -124,6 +123,12 @@ monitor_speed = 115200 debug_tool = jlink upload_protocol = jlink +[env:STM32F103RET6_creality_xfer] +extends = env:STM32F103RET6_creality +extra_scripts = ${env:STM32F103RET6_creality.extra_scripts} + pre:buildroot/share/scripts/upload.py +upload_protocol = custom + # # BigTree SKR Mini E3 V2.0 & DIP / SKR CR6 (STM32F103RET6 ARM Cortex-M3) #