diff --git a/.gitattributes b/.gitattributes index 38bba729fc191..d226cebf5d81d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -27,4 +27,5 @@ # These should also not be modified by git. tests/basics/string_cr_conversion.py -text tests/basics/string_crlf_conversion.py -text +tests/micropython/test_normalize_newlines.py.exp -text # CIRCUITPY-CHANGE: remove non-CircuitPython tests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4db04e1d0eddb..c39faf99f00d2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: check-yaml - id: end-of-file-fixer @@ -55,7 +55,7 @@ repos: language: python - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.9.4 + rev: v0.15.7 hooks: # Run the linter. - id: ruff @@ -63,6 +63,6 @@ repos: # Run the formatter. - id: ruff-format - repo: https://github.com/tox-dev/pyproject-fmt - rev: "v2.5.0" + rev: "v2.21.0" hooks: - id: pyproject-fmt diff --git a/LICENSE_MicroPython b/LICENSE_MicroPython index 469ae1927396d..1fad9b4134e01 100644 --- a/LICENSE_MicroPython +++ b/LICENSE_MicroPython @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2013-2024 Damien P. George +Copyright (c) 2013-2025 Damien P. George Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Makefile b/Makefile index c2aebc61f1c0b..175a1b6a3025f 100644 --- a/Makefile +++ b/Makefile @@ -375,5 +375,6 @@ coverage-fresh: make -j -C ports/unix VARIANT=coverage .PHONY: run-tests +# If TESTS="abc.py def.py" is specified as an arg, run only those tests. Otherwise, run all tests. run-tests: - cd tests; MICROPY_MICROPYTHON=../ports/unix/build-coverage/micropython ./run-tests.py + cd tests; MICROPY_MICROPYTHON=../ports/unix/build-coverage/micropython ./run-tests.py $(TESTS) diff --git a/docs/library/collections.rst b/docs/library/collections.rst index b006a97dcaf22..c3ec763d20e1b 100644 --- a/docs/library/collections.rst +++ b/docs/library/collections.rst @@ -109,3 +109,19 @@ Classes a 2 w 5 b 3 + + .. method:: OrderedDict.popitem() + + Remove and return a (key, value) pair from the dictionary. + Pairs are returned in LIFO order. + + .. admonition:: Difference to CPython + :class: attention + + ``OrderedDict.popitem()`` does not support the ``last=False`` argument and + will always remove and return the last item if present. + + A workaround for this is to use ``pop()`` to remove the first item:: + + first_key = next(iter(d)) + d.pop(first_key) diff --git a/docs/library/platform.rst b/docs/library/platform.rst index c091477d84cb1..c19ef0f5df524 100644 --- a/docs/library/platform.rst +++ b/docs/library/platform.rst @@ -36,3 +36,11 @@ Functions Returns a tuple of strings *(lib, version)*, where *lib* is the name of the libc that MicroPython is linked to, and *version* the corresponding version of this libc. + +.. function:: processor() + + Returns a string with a detailed name of the processor, if one is available. + If no name for the processor is known, it will return an empty string + instead. + + This is currently available only on RISC-V targets (both 32 and 64 bits). diff --git a/docs/library/re.rst b/docs/library/re.rst index 19b15d2d2c299..b8aeefd90cfa4 100644 --- a/docs/library/re.rst +++ b/docs/library/re.rst @@ -154,8 +154,8 @@ Regex objects Compiled regular expression. Instances of this class are created using `re.compile()`. -.. method:: regex.match(string) - regex.search(string) +.. method:: regex.match(string, [pos, [endpos]]) + regex.search(string, [pos, [endpos]]) regex.sub(replace, string, count=0, flags=0, /) Similar to the module-level functions :meth:`match`, :meth:`search` @@ -163,6 +163,16 @@ Compiled regular expression. Instances of this class are created using Using methods is (much) more efficient if the same regex is applied to multiple strings. + The optional second parameter *pos* gives an index in the string where the + search is to start; it defaults to ``0``. This is not completely equivalent + to slicing the string; the ``'^'`` pattern character matches at the real + beginning of the string and at positions just after a newline, but not + necessarily at the index where the search is to start. + + The optional parameter *endpos* limits how far the string will be searched; + it will be as if the string is *endpos* characters long, so only the + characters from *pos* to ``endpos - 1`` will be searched for a match. + .. method:: regex.split(string, max_split=-1, /) Split a *string* using regex. If *max_split* is given, it specifies diff --git a/docs/library/sys.rst b/docs/library/sys.rst index 8def36a2b07a2..c9c687df35936 100644 --- a/docs/library/sys.rst +++ b/docs/library/sys.rst @@ -12,17 +12,9 @@ Functions .. function:: exit(retval=0, /) Terminate current program with a given exit code. Underlyingly, this - function raise as `SystemExit` exception. If an argument is given, its + function raises a `SystemExit` exception. If an argument is given, its value given as an argument to `SystemExit`. -.. function:: print_exception(exc, file=sys.stdout, /) - - This function is deprecated and will be removed starting in - CircuitPython 10.x, `traceback.print_exception()` should be used instead. - - Print exception with a traceback to a file-like object *file* (or - `sys.stdout` by default). - .. admonition:: Difference to CPython :class: attention @@ -52,6 +44,8 @@ Constants * *version* - tuple (major, minor, micro), e.g. (1, 7, 0) * *_machine* - string describing the underlying machine * *_mpy* - supported mpy file-format version (optional attribute) + * *_build* - string that can help identify the configuration that + MicroPython was built with This object is the recommended way to distinguish CircuitPython from other Python implementations (note that it still may not exist in the very @@ -116,15 +110,15 @@ Constants .. data:: stderr - Standard error ``stream``. + Standard error `stream`. .. data:: stdin - Standard input ``stream``. + Standard input `stream`. .. data:: stdout - Standard output ``stream``. + Standard output `stream`. .. data:: version diff --git a/docs/reference/glossary.rst b/docs/reference/glossary.rst index 9e9330de4ccfe..391dc307d2343 100644 --- a/docs/reference/glossary.rst +++ b/docs/reference/glossary.rst @@ -186,6 +186,13 @@ Glossary Most MicroPython boards make a REPL available over a UART, and this is typically accessible on a host PC via USB. + small integer + MicroPython optimises the internal representation of integers such that + "small" values do not take up space on the heap, and calculations with + them do not require heap allocation. On most 32-bit ports, this + corresponds to values in the interval ``-2**30 <= x < 2**30``, but this + should be considered an implementation detail and not relied upon. + stream Also known as a "file-like object". A Python object which provides sequential read-write access to the underlying data. A stream object diff --git a/extmod/extmod.mk b/extmod/extmod.mk index daa6f267342ae..5b2772e4e2a99 100644 --- a/extmod/extmod.mk +++ b/extmod/extmod.mk @@ -162,7 +162,7 @@ endif ifeq ($(MICROPY_VFS_LFS2),1) CFLAGS_EXTMOD += -DMICROPY_VFS_LFS2=1 -CFLAGS_THIRDPARTY += -DLFS2_NO_MALLOC -DLFS2_NO_DEBUG -DLFS2_NO_WARN -DLFS2_NO_ERROR -DLFS2_NO_ASSERT +CFLAGS_THIRDPARTY += -DLFS2_NO_MALLOC -DLFS2_NO_DEBUG -DLFS2_NO_WARN -DLFS2_NO_ERROR -DLFS2_NO_ASSERT -DLFS2_DEFINES=extmod/littlefs-include/lfs2_defines.h SRC_THIRDPARTY_C += $(addprefix $(LITTLEFS_DIR)/,\ lfs2.c \ lfs2_util.c \ @@ -461,7 +461,7 @@ ESP_HOSTED_SRC_C = $(addprefix $(ESP_HOSTED_DIR)/,\ ) ifeq ($(MICROPY_PY_BLUETOOTH),1) -ESP_HOSTED_SRC_C += $(ESP_HOSTED_DIR)/esp_hosted_bthci.c +ESP_HOSTED_SRC_C += $(ESP_HOSTED_DIR)/esp_hosted_bthci_uart.c endif # Include the protobuf-c support functions diff --git a/extmod/lwip-include/arch/cc.h b/extmod/lwip-include/arch/cc.h deleted file mode 100644 index 400dc6ec75de1..0000000000000 --- a/extmod/lwip-include/arch/cc.h +++ /dev/null @@ -1,41 +0,0 @@ -#ifndef MICROPY_INCLUDED_EXTMOD_LWIP_INCLUDE_ARCH_CC_H -#define MICROPY_INCLUDED_EXTMOD_LWIP_INCLUDE_ARCH_CC_H - -#include - -// Generate lwip's internal types from stdint - -typedef uint8_t u8_t; -typedef int8_t s8_t; -typedef uint16_t u16_t; -typedef int16_t s16_t; -typedef uint32_t u32_t; -typedef int32_t s32_t; - -typedef u32_t mem_ptr_t; - -#define U16_F "hu" -#define S16_F "hd" -#define X16_F "hx" -#define U32_F "u" -#define S32_F "d" -#define X32_F "x" - -#define X8_F "02x" -#define SZT_F "u" - -#define BYTE_ORDER LITTLE_ENDIAN - -#define LWIP_CHKSUM_ALGORITHM 2 - -#include -#define LWIP_PLATFORM_DIAG(x) -#define LWIP_PLATFORM_ASSERT(x) { assert(1); } - -//#define PACK_STRUCT_FIELD(x) x __attribute__((packed)) -#define PACK_STRUCT_FIELD(x) x -#define PACK_STRUCT_STRUCT __attribute__((packed)) -#define PACK_STRUCT_BEGIN -#define PACK_STRUCT_END - -#endif // MICROPY_INCLUDED_EXTMOD_LWIP_INCLUDE_ARCH_CC_H diff --git a/extmod/lwip-include/arch/perf.h b/extmod/lwip-include/arch/perf.h deleted file mode 100644 index d310fc339f162..0000000000000 --- a/extmod/lwip-include/arch/perf.h +++ /dev/null @@ -1,7 +0,0 @@ -#ifndef MICROPY_INCLUDED_EXTMOD_LWIP_INCLUDE_ARCH_PERF_H -#define MICROPY_INCLUDED_EXTMOD_LWIP_INCLUDE_ARCH_PERF_H - -#define PERF_START /* null definition */ -#define PERF_STOP(x) /* null definition */ - -#endif // MICROPY_INCLUDED_EXTMOD_LWIP_INCLUDE_ARCH_PERF_H diff --git a/extmod/lwip-include/lwipopts.h b/extmod/lwip-include/lwipopts.h deleted file mode 100644 index 584decfe85f83..0000000000000 --- a/extmod/lwip-include/lwipopts.h +++ /dev/null @@ -1,36 +0,0 @@ -#ifndef MICROPY_INCLUDED_EXTMOD_LWIP_INCLUDE_LWIPOPTS_H -#define MICROPY_INCLUDED_EXTMOD_LWIP_INCLUDE_LWIPOPTS_H - -#include -#include -#include - -// We're running without an OS for this port. We don't provide any services except light protection. -#define NO_SYS 1 - -#define SYS_LIGHTWEIGHT_PROT 1 -#include -typedef uint32_t sys_prot_t; - -#define TCP_LISTEN_BACKLOG 1 - -// We'll put these into a proper ifdef once somebody implements an ethernet driver -#define LWIP_ARP 0 -#define LWIP_ETHERNET 0 - -#define LWIP_DNS 1 - -#define LWIP_NETCONN 0 -#define LWIP_SOCKET 0 - -// CIRCUITPY-CHANGE: #if instead of #ifdef -#if MICROPY_PY_LWIP_SLIP -#define LWIP_HAVE_SLIPIF 1 -#endif - -// For now, we can simply define this as a macro for the timer code. But this function isn't -// universal and other ports will need to do something else. It may be necessary to move -// things like this into a port-provided header file. -#define sys_now mp_hal_ticks_ms - -#endif // MICROPY_INCLUDED_EXTMOD_LWIP_INCLUDE_LWIPOPTS_H diff --git a/extmod/lwip-include/lwipopts_common.h b/extmod/lwip-include/lwipopts_common.h deleted file mode 100644 index 3e4230909499e..0000000000000 --- a/extmod/lwip-include/lwipopts_common.h +++ /dev/null @@ -1,111 +0,0 @@ -/* - * This file is part of the MicroPython project, http://micropython.org/ - * - * The MIT License (MIT) - * - * Copyright (c) 2025 Damien P. George - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MICROPY_INCLUDED_LWIPOPTS_COMMON_H -#define MICROPY_INCLUDED_LWIPOPTS_COMMON_H - -#include "py/mpconfig.h" - -// This sys-arch protection is not needed. -// Ports either protect lwIP code with flags, or run it at PendSV priority. -#define SYS_ARCH_DECL_PROTECT(lev) do { } while (0) -#define SYS_ARCH_PROTECT(lev) do { } while (0) -#define SYS_ARCH_UNPROTECT(lev) do { } while (0) - -#define NO_SYS 1 -#define SYS_LIGHTWEIGHT_PROT 1 -#define MEM_ALIGNMENT 4 - -#define LWIP_CHKSUM_ALGORITHM 3 -#define LWIP_CHECKSUM_CTRL_PER_NETIF 1 - -#define LWIP_ARP 1 -#define LWIP_ETHERNET 1 -#define LWIP_RAW 1 -#define LWIP_NETCONN 0 -#define LWIP_SOCKET 0 -#define LWIP_STATS 0 -#define LWIP_NETIF_HOSTNAME 1 - -#define LWIP_DHCP 1 -#define LWIP_DHCP_CHECK_LINK_UP 1 -#define LWIP_DHCP_DOES_ACD_CHECK 0 // to speed DHCP up -#define LWIP_DNS 1 -#define LWIP_DNS_SUPPORT_MDNS_QUERIES 1 -#define LWIP_MDNS_RESPONDER 1 -#define LWIP_IGMP 1 - -#if MICROPY_PY_LWIP_PPP -#define PPP_SUPPORT 1 -#define PAP_SUPPORT 1 -#define CHAP_SUPPORT 1 -#endif - -#define LWIP_NUM_NETIF_CLIENT_DATA LWIP_MDNS_RESPONDER -#define MEMP_NUM_UDP_PCB (4 + LWIP_MDNS_RESPONDER) - -// The mDNS responder requires 5 timers per IP version plus 2 others. Not having enough silently breaks it. -#define MEMP_NUM_SYS_TIMEOUT (LWIP_NUM_SYS_TIMEOUT_INTERNAL + (LWIP_MDNS_RESPONDER * (2 + (5 * (LWIP_IPV4 + LWIP_IPV6))))) - -#define SO_REUSE 1 -#define TCP_LISTEN_BACKLOG 1 - -// TCP memory settings. -// Default lwIP settings takes 15800 bytes; TCP d/l: 380k/s local, 7.2k/s remote; TCP u/l is very slow. -#ifndef MEM_SIZE - -#if 0 -// lwIP takes 19159 bytes; TCP d/l and u/l are around 320k/s on local network. -#define MEM_SIZE (5000) -#define TCP_WND (4 * TCP_MSS) -#define TCP_SND_BUF (4 * TCP_MSS) -#endif - -#if 1 -// lwIP takes 26700 bytes; TCP dl/ul are around 750/600 k/s on local network. -#define MEM_SIZE (8000) -#define TCP_MSS (800) -#define TCP_WND (8 * TCP_MSS) -#define TCP_SND_BUF (8 * TCP_MSS) -#define MEMP_NUM_TCP_SEG (32) -#endif - -#if 0 -// lwIP takes 45600 bytes; TCP dl/ul are around 1200/1000 k/s on local network. -#define MEM_SIZE (16000) -#define TCP_MSS (1460) -#define TCP_WND (8 * TCP_MSS) -#define TCP_SND_BUF (8 * TCP_MSS) -#define MEMP_NUM_TCP_SEG (32) -#endif - -#endif // MEM_SIZE - -// Needed for PPP. -#define sys_jiffies sys_now - -typedef uint32_t sys_prot_t; - -#endif // MICROPY_INCLUDED_LWIPOPTS_COMMON_H diff --git a/extmod/modjson.c b/extmod/modjson.c index 67da36f0ccf71..77f4a336aa821 100644 --- a/extmod/modjson.c +++ b/extmod/modjson.c @@ -227,7 +227,8 @@ static mp_obj_t _mod_json_load(mp_obj_t stream_obj, bool return_first_json) { for (;;) { cont: if (S_END(s)) { - break; + // Input finished abruptly in the middle of a composite entity. + goto fail; } mp_obj_t next = MP_OBJ_NULL; bool enter = false; diff --git a/extmod/modplatform.c b/extmod/modplatform.c index c6d4d31b97ec1..e1f5476c3b3aa 100644 --- a/extmod/modplatform.c +++ b/extmod/modplatform.c @@ -61,11 +61,49 @@ static mp_obj_t platform_libc_ver(size_t n_args, const mp_obj_t *pos_args, mp_ma } static MP_DEFINE_CONST_FUN_OBJ_KW(platform_libc_ver_obj, 0, platform_libc_ver); +#ifdef __riscv +static mp_obj_t platform_processor(void) { + #if (__riscv_xlen <= 64) && !defined(__linux__) + uintptr_t misa_csr = 0; + + // Load the MISA CSR directly. + __asm volatile ( + "csrr %0, misa \n" + : "+r" (misa_csr) + : + : + ); + + char processor_buffer[31] = { // "RV32"/"RV64" + up to 26 chars. + #if (__riscv_xlen < 64) + "RV32" + #else + "RV64" + #endif + }; + mp_uint_t offset = 4; + for (mp_uint_t bit = 0; bit < 26; bit++) { + if (misa_csr & (1U << bit)) { + processor_buffer[offset++] = 'A' + bit; + } + } + + return mp_obj_new_str(processor_buffer, offset); + #else + return MP_OBJ_NEW_QSTR(MP_QSTR_); + #endif +} +static MP_DEFINE_CONST_FUN_OBJ_0(platform_processor_obj, platform_processor); +#endif + static const mp_rom_map_elem_t modplatform_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_platform) }, { MP_ROM_QSTR(MP_QSTR_platform), MP_ROM_PTR(&platform_platform_obj) }, { MP_ROM_QSTR(MP_QSTR_python_compiler), MP_ROM_PTR(&platform_python_compiler_obj) }, { MP_ROM_QSTR(MP_QSTR_libc_ver), MP_ROM_PTR(&platform_libc_ver_obj) }, + #ifdef __riscv + { MP_ROM_QSTR(MP_QSTR_processor), MP_ROM_PTR(&platform_processor_obj) }, + #endif }; static MP_DEFINE_CONST_DICT(modplatform_globals, modplatform_globals_table); diff --git a/extmod/modre.c b/extmod/modre.c index ba2927eb4a8a6..36fff1d1dc668 100644 --- a/extmod/modre.c +++ b/extmod/modre.c @@ -196,10 +196,11 @@ static void re_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t // Note: this function can't be named re_exec because it may clash with system headers, eg on FreeBSD static mp_obj_t re_exec_helper(bool is_anchored, uint n_args, const mp_obj_t *args) { - (void)n_args; mp_obj_re_t *self; + bool was_compiled = false; if (mp_obj_is_type(args[0], (mp_obj_type_t *)&re_type)) { self = MP_OBJ_TO_PTR(args[0]); + was_compiled = true; } else { self = MP_OBJ_TO_PTR(mod_re_compile(1, args)); } @@ -207,37 +208,28 @@ static mp_obj_t re_exec_helper(bool is_anchored, uint n_args, const mp_obj_t *ar size_t len; subj.begin_line = subj.begin = mp_obj_str_get_data(args[1], &len); subj.end = subj.begin + len; - // CIRCUITPY-CHANGE - #if MICROPY_PY_RE_MATCH_SPAN_START_END && !(defined(MICROPY_ENABLE_DYNRUNTIME) && MICROPY_ENABLE_DYNRUNTIME) - - if (n_args > 2) { - const mp_obj_type_t *self_type = mp_obj_get_type(args[1]); - mp_int_t str_len = MP_OBJ_SMALL_INT_VALUE(mp_obj_len(args[1])); - const byte *begin = (const byte *)subj.begin; - int pos = mp_obj_get_int(args[2]); - if (pos >= str_len) { - return mp_const_none; - } - if (pos < 0) { - pos = 0; + if (was_compiled && n_args > 2) { + // Arg #2 is starting-pos + mp_int_t startpos = mp_obj_get_int(args[2]); + if (startpos > (mp_int_t)len) { + startpos = len; + } else if (startpos < 0) { + startpos = 0; } - const byte *pos_ptr = str_index_to_ptr(self_type, begin, len, MP_OBJ_NEW_SMALL_INT(pos), true); - - const byte *endpos_ptr = (const byte *)subj.end; + subj.begin += startpos; if (n_args > 3) { - int endpos = mp_obj_get_int(args[3]); - if (endpos <= pos) { - return mp_const_none; + // Arg #3 is ending-pos + mp_int_t endpos = mp_obj_get_int(args[3]); + if (endpos > (mp_int_t)len) { + endpos = len; + } else if (endpos < startpos) { + endpos = startpos; } - // Will cap to length - endpos_ptr = str_index_to_ptr(self_type, begin, len, args[3], true); + subj.end = subj.begin_line + endpos; } - - subj.begin = (const char *)pos_ptr; - subj.end = (const char *)endpos_ptr; } - #endif + int caps_num = (self->re.sub + 1) * 2; mp_obj_match_t *match = m_new_obj_var(mp_obj_match_t, caps, char *, caps_num); // cast is a workaround for a bug in msvc: it treats const char** as a const pointer instead of a pointer to pointer to const char @@ -458,6 +450,9 @@ static mp_obj_t mod_re_compile(size_t n_args, const mp_obj_t *args) { const char *re_str = mp_obj_str_get_str(args[0]); int size = re1_5_sizecode(re_str); if (size == -1) { + #if MICROPY_ERROR_REPORTING >= MICROPY_ERROR_REPORTING_NORMAL + mp_raise_ValueError(MP_ERROR_TEXT("regex too complex")); + #endif goto error; } mp_obj_re_t *o = mp_obj_malloc_var(mp_obj_re_t, re.insts, char, size, (mp_obj_type_t *)&re_type); @@ -470,6 +465,7 @@ static mp_obj_t mod_re_compile(size_t n_args, const mp_obj_t *args) { int error = re1_5_compilecode(&o->re, re_str); if (error != 0) { error: + // CIRCUITPY-CHANGE: capitalized mp_raise_ValueError(MP_ERROR_TEXT("Error in regex")); } #if MICROPY_PY_RE_DEBUG diff --git a/extmod/modtime.c b/extmod/modtime.c index deb4bb4c9ace7..ee898828a4ab1 100644 --- a/extmod/modtime.c +++ b/extmod/modtime.c @@ -53,26 +53,26 @@ // - weekday is 0-6 for Mon-Sun // - yearday is 1-366 static mp_obj_t time_localtime(size_t n_args, const mp_obj_t *args) { + timeutils_struct_time_t tm; if (n_args == 0 || args[0] == mp_const_none) { // Get current date and time. - return mp_time_localtime_get(); + mp_time_localtime_get(&tm); } else { // Convert given seconds to tuple. - mp_int_t seconds = mp_obj_get_int(args[0]); - timeutils_struct_time_t tm; + mp_timestamp_t seconds = timeutils_obj_get_timestamp(args[0]); timeutils_seconds_since_epoch_to_struct_time(seconds, &tm); - mp_obj_t tuple[8] = { - tuple[0] = mp_obj_new_int(tm.tm_year), - tuple[1] = mp_obj_new_int(tm.tm_mon), - tuple[2] = mp_obj_new_int(tm.tm_mday), - tuple[3] = mp_obj_new_int(tm.tm_hour), - tuple[4] = mp_obj_new_int(tm.tm_min), - tuple[5] = mp_obj_new_int(tm.tm_sec), - tuple[6] = mp_obj_new_int(tm.tm_wday), - tuple[7] = mp_obj_new_int(tm.tm_yday), - }; - return mp_obj_new_tuple(8, tuple); } + mp_obj_t tuple[8] = { + mp_obj_new_int(tm.tm_year), + mp_obj_new_int(tm.tm_mon), + mp_obj_new_int(tm.tm_mday), + mp_obj_new_int(tm.tm_hour), + mp_obj_new_int(tm.tm_min), + mp_obj_new_int(tm.tm_sec), + mp_obj_new_int(tm.tm_wday), + mp_obj_new_int(tm.tm_yday), + }; + return mp_obj_new_tuple(8, tuple); } MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_time_localtime_obj, 0, 1, time_localtime); @@ -90,7 +90,7 @@ static mp_obj_t time_mktime(mp_obj_t tuple) { mp_raise_TypeError(MP_ERROR_TEXT("mktime needs a tuple of length 8 or 9")); } - return mp_obj_new_int_from_uint(timeutils_mktime(mp_obj_get_int(elem[0]), + return timeutils_obj_from_timestamp(timeutils_mktime(mp_obj_get_int(elem[0]), mp_obj_get_int(elem[1]), mp_obj_get_int(elem[2]), mp_obj_get_int(elem[3]), mp_obj_get_int(elem[4]), mp_obj_get_int(elem[5]))); } diff --git a/extmod/vfs.c b/extmod/vfs.c index 0cf93c3966c90..c4156990bcc38 100644 --- a/extmod/vfs.c +++ b/extmod/vfs.c @@ -101,7 +101,7 @@ mp_vfs_mount_t *mp_vfs_lookup_path(const char *path, const char **path_out) { return MP_STATE_VM(vfs_cur); } -// Version of mp_vfs_lookup_path that takes and returns uPy string objects. +// Version of mp_vfs_lookup_path that takes and returns MicroPython string objects. static mp_vfs_mount_t *lookup_path(mp_obj_t path_in, mp_obj_t *path_out) { const char *path = mp_obj_str_get_str(path_in); const char *p_out; diff --git a/extmod/vfs_blockdev.c b/extmod/vfs_blockdev.c index 0cd3cc6f0c631..74d1262364ef4 100644 --- a/extmod/vfs_blockdev.c +++ b/extmod/vfs_blockdev.c @@ -108,7 +108,7 @@ static int mp_vfs_blockdev_call_rw(mp_obj_t *args, size_t block_num, size_t bloc // and negative integer on errors. Check for positive integer // results as some callers (i.e. littlefs) will produce corrupt // results from these. - int i = MP_OBJ_SMALL_INT_VALUE(ret); + int i = mp_obj_get_int(ret); return i > 0 ? (-MP_EINVAL) : i; } } diff --git a/extmod/vfs_fat.c b/extmod/vfs_fat.c index 6267452cd32c7..95ed79eb18bd5 100644 --- a/extmod/vfs_fat.c +++ b/extmod/vfs_fat.c @@ -58,7 +58,7 @@ // CIRCUITPY-CHANGE // Factoring this common call saves about 90 bytes. -static NORETURN void mp_raise_OSError_fresult(FRESULT res) { +static MP_NORETURN void mp_raise_OSError_fresult(FRESULT res) { mp_raise_OSError(fresult_to_errno_table[res]); } diff --git a/extmod/vfs_lfsx.c b/extmod/vfs_lfsx.c index 4b10ca3aa597c..bbdd21cfb9176 100644 --- a/extmod/vfs_lfsx.c +++ b/extmod/vfs_lfsx.c @@ -104,6 +104,12 @@ static void MP_VFS_LFSx(init_config)(MP_OBJ_VFS_LFSx * self, mp_obj_t bdev, size config->read_buffer = m_new(uint8_t, config->cache_size); config->prog_buffer = m_new(uint8_t, config->cache_size); config->lookahead_buffer = m_new(uint8_t, config->lookahead_size); + #ifdef LFS2_MULTIVERSION + // This can be set to override the on-disk lfs version. + // eg. for compat with lfs2 < v2.6 add the following to make: + // CFLAGS += '-DLFS2_MULTIVERSION=0x00020000' + config->disk_version = LFS2_MULTIVERSION; + #endif #endif } @@ -300,7 +306,7 @@ static mp_obj_t MP_VFS_LFSx(chdir)(mp_obj_t self_in, mp_obj_t path_in) { struct LFSx_API (info) info; int ret = LFSx_API(stat)(&self->lfs, path, &info); if (ret < 0 || info.type != LFSx_MACRO(_TYPE_DIR)) { - mp_raise_OSError(-MP_ENOENT); + mp_raise_OSError(MP_ENOENT); } } @@ -378,7 +384,7 @@ static mp_obj_t MP_VFS_LFSx(stat)(mp_obj_t self_in, mp_obj_t path_in) { mp_raise_OSError(-ret); } - mp_uint_t mtime = 0; + mp_timestamp_t mtime = 0; #if LFS_BUILD_VERSION == 2 uint8_t mtime_buf[8]; lfs2_ssize_t sz = lfs2_getattr(&self->lfs, path, LFS_ATTR_MTIME, &mtime_buf, sizeof(mtime_buf)); @@ -400,9 +406,9 @@ static mp_obj_t MP_VFS_LFSx(stat)(mp_obj_t self_in, mp_obj_t path_in) { t->items[4] = MP_OBJ_NEW_SMALL_INT(0); // st_uid t->items[5] = MP_OBJ_NEW_SMALL_INT(0); // st_gid t->items[6] = mp_obj_new_int_from_uint(info.size); // st_size - t->items[7] = mp_obj_new_int_from_uint(mtime); // st_atime - t->items[8] = mp_obj_new_int_from_uint(mtime); // st_mtime - t->items[9] = mp_obj_new_int_from_uint(mtime); // st_ctime + t->items[7] = timeutils_obj_from_timestamp(mtime); // st_atime + t->items[8] = timeutils_obj_from_timestamp(mtime); // st_mtime + t->items[9] = timeutils_obj_from_timestamp(mtime); // st_ctime return MP_OBJ_FROM_PTR(t); } diff --git a/extmod/vfs_lfsx_file.c b/extmod/vfs_lfsx_file.c index ab5cce50088b9..56daa53e06869 100644 --- a/extmod/vfs_lfsx_file.c +++ b/extmod/vfs_lfsx_file.c @@ -90,9 +90,9 @@ mp_obj_t MP_VFS_LFSx(file_open)(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mod } #if LFS_BUILD_VERSION == 1 - MP_OBJ_VFS_LFSx_FILE *o = mp_obj_malloc_var_with_finaliser(MP_OBJ_VFS_LFSx_FILE, uint8_t, self->lfs.cfg->prog_size, type); + MP_OBJ_VFS_LFSx_FILE *o = mp_obj_malloc_var_with_finaliser(MP_OBJ_VFS_LFSx_FILE, file_buffer, uint8_t, self->lfs.cfg->prog_size, type); #else - MP_OBJ_VFS_LFSx_FILE *o = mp_obj_malloc_var_with_finaliser(MP_OBJ_VFS_LFSx_FILE, uint8_t, self->lfs.cfg->cache_size, type); + MP_OBJ_VFS_LFSx_FILE *o = mp_obj_malloc_var_with_finaliser(MP_OBJ_VFS_LFSx_FILE, file_buffer, uint8_t, self->lfs.cfg->cache_size, type); #endif o->vfs = self; #if !MICROPY_GC_CONSERVATIVE_CLEAR diff --git a/extmod/vfs_posix.c b/extmod/vfs_posix.c index bd9a6d84062cf..27f833e802f69 100644 --- a/extmod/vfs_posix.c +++ b/extmod/vfs_posix.c @@ -137,7 +137,7 @@ static mp_obj_t vfs_posix_make_new(const mp_obj_type_t *type, size_t n_args, siz vstr_add_char(&vfs->root, '/'); } vfs->root_len = vfs->root.len; - vfs->readonly = false; + vfs->readonly = !MICROPY_VFS_POSIX_WRITABLE; return MP_OBJ_FROM_PTR(vfs); } @@ -160,10 +160,21 @@ static mp_obj_t vfs_posix_umount(mp_obj_t self_in) { } static MP_DEFINE_CONST_FUN_OBJ_1(vfs_posix_umount_obj, vfs_posix_umount); +static inline bool vfs_posix_is_readonly(mp_obj_vfs_posix_t *self) { + return !MICROPY_VFS_POSIX_WRITABLE || self->readonly; +} + +static void vfs_posix_require_writable(mp_obj_t self_in) { + mp_obj_vfs_posix_t *self = MP_OBJ_TO_PTR(self_in); + if (vfs_posix_is_readonly(self)) { + mp_raise_OSError(MP_EROFS); + } +} + static mp_obj_t vfs_posix_open(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mode_in) { mp_obj_vfs_posix_t *self = MP_OBJ_TO_PTR(self_in); const char *mode = mp_obj_str_get_str(mode_in); - if (self->readonly + if (vfs_posix_is_readonly(self) && (strchr(mode, 'w') != NULL || strchr(mode, 'a') != NULL || strchr(mode, '+') != NULL)) { mp_raise_OSError(MP_EROFS); } @@ -303,6 +314,7 @@ typedef struct _mp_obj_listdir_t { } mp_obj_listdir_t; static mp_obj_t vfs_posix_mkdir(mp_obj_t self_in, mp_obj_t path_in) { + vfs_posix_require_writable(self_in); mp_obj_vfs_posix_t *self = MP_OBJ_TO_PTR(self_in); const char *path = vfs_posix_get_path_str(self, path_in); MP_THREAD_GIL_EXIT(); @@ -320,11 +332,13 @@ static mp_obj_t vfs_posix_mkdir(mp_obj_t self_in, mp_obj_t path_in) { static MP_DEFINE_CONST_FUN_OBJ_2(vfs_posix_mkdir_obj, vfs_posix_mkdir); static mp_obj_t vfs_posix_remove(mp_obj_t self_in, mp_obj_t path_in) { + vfs_posix_require_writable(self_in); return vfs_posix_fun1_helper(self_in, path_in, unlink); } static MP_DEFINE_CONST_FUN_OBJ_2(vfs_posix_remove_obj, vfs_posix_remove); static mp_obj_t vfs_posix_rename(mp_obj_t self_in, mp_obj_t old_path_in, mp_obj_t new_path_in) { + vfs_posix_require_writable(self_in); mp_obj_vfs_posix_t *self = MP_OBJ_TO_PTR(self_in); const char *old_path = vfs_posix_get_path_str(self, old_path_in); const char *new_path = vfs_posix_get_path_str(self, new_path_in); @@ -339,6 +353,7 @@ static mp_obj_t vfs_posix_rename(mp_obj_t self_in, mp_obj_t old_path_in, mp_obj_ static MP_DEFINE_CONST_FUN_OBJ_3(vfs_posix_rename_obj, vfs_posix_rename); static mp_obj_t vfs_posix_rmdir(mp_obj_t self_in, mp_obj_t path_in) { + vfs_posix_require_writable(self_in); return vfs_posix_fun1_helper(self_in, path_in, rmdir); } static MP_DEFINE_CONST_FUN_OBJ_2(vfs_posix_rmdir_obj, vfs_posix_rmdir); diff --git a/extmod/vfs_reader.c b/extmod/vfs_reader.c index de5c4e03d3c13..6c36770295be6 100644 --- a/extmod/vfs_reader.c +++ b/extmod/vfs_reader.c @@ -83,7 +83,7 @@ void mp_reader_new_file(mp_reader_t *reader, qstr filename) { }; mp_obj_t file = mp_vfs_open(MP_ARRAY_SIZE(args), &args[0], (mp_map_t *)&mp_const_empty_map); - const mp_stream_p_t *stream_p = mp_get_stream(file); + const mp_stream_p_t *stream_p = mp_get_stream_raise(file, MP_STREAM_OP_READ); int errcode = 0; #if MICROPY_VFS_ROM diff --git a/frozen/Adafruit_CircuitPython_asyncio b/frozen/Adafruit_CircuitPython_asyncio index e69ac03dccfd8..1f9fee3c5d99d 160000 --- a/frozen/Adafruit_CircuitPython_asyncio +++ b/frozen/Adafruit_CircuitPython_asyncio @@ -1 +1 @@ -Subproject commit e69ac03dccfd87ccaf3655dc751331ff922f525f +Subproject commit 1f9fee3c5d99d60b069c4a366678b391c198d955 diff --git a/lib/berkeley-db-1.xx b/lib/berkeley-db-1.xx index 85373b548f1fb..0f3bb6947c2f5 160000 --- a/lib/berkeley-db-1.xx +++ b/lib/berkeley-db-1.xx @@ -1 +1 @@ -Subproject commit 85373b548f1fb0119a463582570b44189dfb09ae +Subproject commit 0f3bb6947c2f57233916dccd7bb425d7bf86e5a6 diff --git a/lib/libm_dbl/libm.h b/lib/libm_dbl/libm.h index cbae6916625e2..fa49ad1cddda5 100644 --- a/lib/libm_dbl/libm.h +++ b/lib/libm_dbl/libm.h @@ -89,7 +89,9 @@ do { \ (d) = __u.f; \ } while (0) +#if !defined(DBL_EPSILON) #define DBL_EPSILON 2.22044604925031308085e-16 +#endif int __rem_pio2(double, double*); int __rem_pio2_large(double*, double*, int, int, int); diff --git a/lib/libm_dbl/rint.c b/lib/libm_dbl/rint.c index fbba390e7d723..b85dec8f2465e 100644 --- a/lib/libm_dbl/rint.c +++ b/lib/libm_dbl/rint.c @@ -2,7 +2,7 @@ #include #include -#if FLT_EVAL_METHOD==0 || FLT_EVAL_METHOD==1 +#if FLT_EVAL_METHOD==0 || FLT_EVAL_METHOD==1 || FLT_EVAL_METHOD==16 #define EPS DBL_EPSILON #elif FLT_EVAL_METHOD==2 #define EPS LDBL_EPSILON diff --git a/lib/littlefs/lfs2.c b/lib/littlefs/lfs2.c index 613213669c8f7..8d18b7d110521 100644 --- a/lib/littlefs/lfs2.c +++ b/lib/littlefs/lfs2.c @@ -93,6 +93,7 @@ static int lfs2_bd_read(lfs2_t *lfs2, // bypass cache? diff = lfs2_aligndown(diff, lfs2->cfg->read_size); int err = lfs2->cfg->read(lfs2->cfg, block, off, data, diff); + LFS2_ASSERT(err <= 0); if (err) { return err; } @@ -282,6 +283,21 @@ static int lfs2_bd_erase(lfs2_t *lfs2, lfs2_block_t block) { /// Small type-level utilities /// + +// some operations on paths +static inline lfs2_size_t lfs2_path_namelen(const char *path) { + return strcspn(path, "/"); +} + +static inline bool lfs2_path_islast(const char *path) { + lfs2_size_t namelen = lfs2_path_namelen(path); + return path[namelen + strspn(path + namelen, "/")] == '\0'; +} + +static inline bool lfs2_path_isdir(const char *path) { + return path[lfs2_path_namelen(path)] != '\0'; +} + // operations on block pairs static inline void lfs2_pair_swap(lfs2_block_t pair[2]) { lfs2_block_t t = pair[0]; @@ -389,18 +405,15 @@ struct lfs2_diskoff { // operations on global state static inline void lfs2_gstate_xor(lfs2_gstate_t *a, const lfs2_gstate_t *b) { - for (int i = 0; i < 3; i++) { - ((uint32_t*)a)[i] ^= ((const uint32_t*)b)[i]; - } + a->tag ^= b->tag; + a->pair[0] ^= b->pair[0]; + a->pair[1] ^= b->pair[1]; } static inline bool lfs2_gstate_iszero(const lfs2_gstate_t *a) { - for (int i = 0; i < 3; i++) { - if (((uint32_t*)a)[i] != 0) { - return false; - } - } - return true; + return a->tag == 0 + && a->pair[0] == 0 + && a->pair[1] == 0; } #ifndef LFS2_READONLY @@ -550,9 +563,9 @@ static int lfs2_dir_compact(lfs2_t *lfs2, lfs2_mdir_t *source, uint16_t begin, uint16_t end); static lfs2_ssize_t lfs2_file_flushedwrite(lfs2_t *lfs2, lfs2_file_t *file, const void *buffer, lfs2_size_t size); -static lfs2_ssize_t lfs2_file_rawwrite(lfs2_t *lfs2, lfs2_file_t *file, +static lfs2_ssize_t lfs2_file_write_(lfs2_t *lfs2, lfs2_file_t *file, const void *buffer, lfs2_size_t size); -static int lfs2_file_rawsync(lfs2_t *lfs2, lfs2_file_t *file); +static int lfs2_file_sync_(lfs2_t *lfs2, lfs2_file_t *file); static int lfs2_file_outline(lfs2_t *lfs2, lfs2_file_t *file); static int lfs2_file_flush(lfs2_t *lfs2, lfs2_file_t *file); @@ -574,65 +587,72 @@ static int lfs21_traverse(lfs2_t *lfs2, int (*cb)(void*, lfs2_block_t), void *data); #endif -static int lfs2_dir_rawrewind(lfs2_t *lfs2, lfs2_dir_t *dir); +static int lfs2_dir_rewind_(lfs2_t *lfs2, lfs2_dir_t *dir); static lfs2_ssize_t lfs2_file_flushedread(lfs2_t *lfs2, lfs2_file_t *file, void *buffer, lfs2_size_t size); -static lfs2_ssize_t lfs2_file_rawread(lfs2_t *lfs2, lfs2_file_t *file, +static lfs2_ssize_t lfs2_file_read_(lfs2_t *lfs2, lfs2_file_t *file, void *buffer, lfs2_size_t size); -static int lfs2_file_rawclose(lfs2_t *lfs2, lfs2_file_t *file); -static lfs2_soff_t lfs2_file_rawsize(lfs2_t *lfs2, lfs2_file_t *file); +static int lfs2_file_close_(lfs2_t *lfs2, lfs2_file_t *file); +static lfs2_soff_t lfs2_file_size_(lfs2_t *lfs2, lfs2_file_t *file); -static lfs2_ssize_t lfs2_fs_rawsize(lfs2_t *lfs2); -static int lfs2_fs_rawtraverse(lfs2_t *lfs2, +static lfs2_ssize_t lfs2_fs_size_(lfs2_t *lfs2); +static int lfs2_fs_traverse_(lfs2_t *lfs2, int (*cb)(void *data, lfs2_block_t block), void *data, bool includeorphans); static int lfs2_deinit(lfs2_t *lfs2); -static int lfs2_rawunmount(lfs2_t *lfs2); +static int lfs2_unmount_(lfs2_t *lfs2); /// Block allocator /// + +// allocations should call this when all allocated blocks are committed to +// the filesystem +// +// after a checkpoint, the block allocator may realloc any untracked blocks +static void lfs2_alloc_ckpoint(lfs2_t *lfs2) { + lfs2->lookahead.ckpoint = lfs2->block_count; +} + +// drop the lookahead buffer, this is done during mounting and failed +// traversals in order to avoid invalid lookahead state +static void lfs2_alloc_drop(lfs2_t *lfs2) { + lfs2->lookahead.size = 0; + lfs2->lookahead.next = 0; + lfs2_alloc_ckpoint(lfs2); +} + #ifndef LFS2_READONLY static int lfs2_alloc_lookahead(void *p, lfs2_block_t block) { lfs2_t *lfs2 = (lfs2_t*)p; - lfs2_block_t off = ((block - lfs2->free.off) + lfs2_block_t off = ((block - lfs2->lookahead.start) + lfs2->block_count) % lfs2->block_count; - if (off < lfs2->free.size) { - lfs2->free.buffer[off / 32] |= 1U << (off % 32); + if (off < lfs2->lookahead.size) { + lfs2->lookahead.buffer[off / 8] |= 1U << (off % 8); } return 0; } #endif -// indicate allocated blocks have been committed into the filesystem, this -// is to prevent blocks from being garbage collected in the middle of a -// commit operation -static void lfs2_alloc_ack(lfs2_t *lfs2) { - lfs2->free.ack = lfs2->block_count; -} - -// drop the lookahead buffer, this is done during mounting and failed -// traversals in order to avoid invalid lookahead state -static void lfs2_alloc_drop(lfs2_t *lfs2) { - lfs2->free.size = 0; - lfs2->free.i = 0; - lfs2_alloc_ack(lfs2); -} - #ifndef LFS2_READONLY -static int lfs2_fs_rawgc(lfs2_t *lfs2) { - // Move free offset at the first unused block (lfs2->free.i) - // lfs2->free.i is equal lfs2->free.size when all blocks are used - lfs2->free.off = (lfs2->free.off + lfs2->free.i) % lfs2->block_count; - lfs2->free.size = lfs2_min(8*lfs2->cfg->lookahead_size, lfs2->free.ack); - lfs2->free.i = 0; +static int lfs2_alloc_scan(lfs2_t *lfs2) { + // move lookahead buffer to the first unused block + // + // note we limit the lookahead buffer to at most the amount of blocks + // checkpointed, this prevents the math in lfs2_alloc from underflowing + lfs2->lookahead.start = (lfs2->lookahead.start + lfs2->lookahead.next) + % lfs2->block_count; + lfs2->lookahead.next = 0; + lfs2->lookahead.size = lfs2_min( + 8*lfs2->cfg->lookahead_size, + lfs2->lookahead.ckpoint); // find mask of free blocks from tree - memset(lfs2->free.buffer, 0, lfs2->cfg->lookahead_size); - int err = lfs2_fs_rawtraverse(lfs2, lfs2_alloc_lookahead, lfs2, true); + memset(lfs2->lookahead.buffer, 0, lfs2->cfg->lookahead_size); + int err = lfs2_fs_traverse_(lfs2, lfs2_alloc_lookahead, lfs2, true); if (err) { lfs2_alloc_drop(lfs2); return err; @@ -645,36 +665,49 @@ static int lfs2_fs_rawgc(lfs2_t *lfs2) { #ifndef LFS2_READONLY static int lfs2_alloc(lfs2_t *lfs2, lfs2_block_t *block) { while (true) { - while (lfs2->free.i != lfs2->free.size) { - lfs2_block_t off = lfs2->free.i; - lfs2->free.i += 1; - lfs2->free.ack -= 1; - - if (!(lfs2->free.buffer[off / 32] & (1U << (off % 32)))) { + // scan our lookahead buffer for free blocks + while (lfs2->lookahead.next < lfs2->lookahead.size) { + if (!(lfs2->lookahead.buffer[lfs2->lookahead.next / 8] + & (1U << (lfs2->lookahead.next % 8)))) { // found a free block - *block = (lfs2->free.off + off) % lfs2->block_count; - - // eagerly find next off so an alloc ack can - // discredit old lookahead blocks - while (lfs2->free.i != lfs2->free.size && - (lfs2->free.buffer[lfs2->free.i / 32] - & (1U << (lfs2->free.i % 32)))) { - lfs2->free.i += 1; - lfs2->free.ack -= 1; + *block = (lfs2->lookahead.start + lfs2->lookahead.next) + % lfs2->block_count; + + // eagerly find next free block to maximize how many blocks + // lfs2_alloc_ckpoint makes available for scanning + while (true) { + lfs2->lookahead.next += 1; + lfs2->lookahead.ckpoint -= 1; + + if (lfs2->lookahead.next >= lfs2->lookahead.size + || !(lfs2->lookahead.buffer[lfs2->lookahead.next / 8] + & (1U << (lfs2->lookahead.next % 8)))) { + return 0; + } } - - return 0; } + + lfs2->lookahead.next += 1; + lfs2->lookahead.ckpoint -= 1; } - // check if we have looked at all blocks since last ack - if (lfs2->free.ack == 0) { - LFS2_ERROR("No more free space %"PRIu32, - lfs2->free.i + lfs2->free.off); + // In order to keep our block allocator from spinning forever when our + // filesystem is full, we mark points where there are no in-flight + // allocations with a checkpoint before starting a set of allocations. + // + // If we've looked at all blocks since the last checkpoint, we report + // the filesystem as out of storage. + // + if (lfs2->lookahead.ckpoint <= 0) { + LFS2_ERROR("No more free space 0x%"PRIx32, + (lfs2->lookahead.start + lfs2->lookahead.next) + % lfs2->block_count); return LFS2_ERR_NOSPC; } - int err = lfs2_fs_rawgc(lfs2); + // No blocks in our lookahead buffer, we need to scan the filesystem for + // unused blocks in the next lookahead window. + int err = lfs2_alloc_scan(lfs2); if(err) { return err; } @@ -690,11 +723,14 @@ static lfs2_stag_t lfs2_dir_getslice(lfs2_t *lfs2, const lfs2_mdir_t *dir, lfs2_tag_t ntag = dir->etag; lfs2_stag_t gdiff = 0; + // synthetic moves if (lfs2_gstate_hasmovehere(&lfs2->gdisk, dir->pair) && - lfs2_tag_id(gmask) != 0 && - lfs2_tag_id(lfs2->gdisk.tag) <= lfs2_tag_id(gtag)) { - // synthetic moves - gdiff -= LFS2_MKTAG(0, 1, 0); + lfs2_tag_id(gmask) != 0) { + if (lfs2_tag_id(lfs2->gdisk.tag) == lfs2_tag_id(gtag)) { + return LFS2_ERR_NOENT; + } else if (lfs2_tag_id(lfs2->gdisk.tag) < lfs2_tag_id(gtag)) { + gdiff -= LFS2_MKTAG(0, 1, 0); + } } // iterate over dir block backwards (for faster lookups) @@ -704,6 +740,7 @@ static lfs2_stag_t lfs2_dir_getslice(lfs2_t *lfs2, const lfs2_mdir_t *dir, int err = lfs2_bd_read(lfs2, NULL, &lfs2->rcache, sizeof(ntag), dir->pair[0], off, &ntag, sizeof(ntag)); + LFS2_ASSERT(err <= 0); if (err) { return err; } @@ -732,6 +769,7 @@ static lfs2_stag_t lfs2_dir_getslice(lfs2_t *lfs2, const lfs2_mdir_t *dir, err = lfs2_bd_read(lfs2, NULL, &lfs2->rcache, diff, dir->pair[0], off+sizeof(tag)+goff, gbuffer, diff); + LFS2_ASSERT(err <= 0); if (err) { return err; } @@ -793,9 +831,6 @@ static int lfs2_dir_getread(lfs2_t *lfs2, const lfs2_mdir_t *dir, size -= diff; continue; } - - // rcache takes priority - diff = lfs2_min(diff, rcache->off-off); } // load to cache, first condition can no longer fail @@ -1247,6 +1282,7 @@ static lfs2_stag_t lfs2_dir_fetchmatch(lfs2_t *lfs2, if (err == LFS2_ERR_CORRUPT) { break; } + return err; } lfs2_fcrc_fromle32(&fcrc); @@ -1438,32 +1474,46 @@ static int lfs2_dir_find_match(void *data, return LFS2_CMP_EQ; } +// lfs2_dir_find tries to set path and id even if file is not found +// +// returns: +// - 0 if file is found +// - LFS2_ERR_NOENT if file or parent is not found +// - LFS2_ERR_NOTDIR if parent is not a dir static lfs2_stag_t lfs2_dir_find(lfs2_t *lfs2, lfs2_mdir_t *dir, const char **path, uint16_t *id) { // we reduce path to a single name if we can find it const char *name = *path; - if (id) { - *id = 0x3ff; - } // default to root dir lfs2_stag_t tag = LFS2_MKTAG(LFS2_TYPE_DIR, 0x3ff, 0); dir->tail[0] = lfs2->root[0]; dir->tail[1] = lfs2->root[1]; + // empty paths are not allowed + if (*name == '\0') { + return LFS2_ERR_INVAL; + } + while (true) { nextname: - // skip slashes - name += strspn(name, "/"); + // skip slashes if we're a directory + if (lfs2_tag_type3(tag) == LFS2_TYPE_DIR) { + name += strspn(name, "/"); + } lfs2_size_t namelen = strcspn(name, "/"); - // skip '.' and root '..' - if ((namelen == 1 && memcmp(name, ".", 1) == 0) || - (namelen == 2 && memcmp(name, "..", 2) == 0)) { + // skip '.' + if (namelen == 1 && memcmp(name, ".", 1) == 0) { name += namelen; goto nextname; } + // error on unmatched '..', trying to go above root? + if (namelen == 2 && memcmp(name, "..", 2) == 0) { + return LFS2_ERR_INVAL; + } + // skip if matched by '..' in name const char *suffix = name + namelen; lfs2_size_t sufflen; @@ -1475,7 +1525,9 @@ static lfs2_stag_t lfs2_dir_find(lfs2_t *lfs2, lfs2_mdir_t *dir, break; } - if (sufflen == 2 && memcmp(suffix, "..", 2) == 0) { + if (sufflen == 1 && memcmp(suffix, ".", 1) == 0) { + // noop + } else if (sufflen == 2 && memcmp(suffix, "..", 2) == 0) { depth -= 1; if (depth == 0) { name = suffix + sufflen; @@ -1489,14 +1541,14 @@ static lfs2_stag_t lfs2_dir_find(lfs2_t *lfs2, lfs2_mdir_t *dir, } // found path - if (name[0] == '\0') { + if (*name == '\0') { return tag; } // update what we've found so far *path = name; - // only continue if we hit a directory + // only continue if we're a directory if (lfs2_tag_type3(tag) != LFS2_TYPE_DIR) { return LFS2_ERR_NOTDIR; } @@ -1516,8 +1568,7 @@ static lfs2_stag_t lfs2_dir_find(lfs2_t *lfs2, lfs2_mdir_t *dir, tag = lfs2_dir_fetchmatch(lfs2, dir, dir->tail, LFS2_MKTAG(0x780, 0, 0), LFS2_MKTAG(LFS2_TYPE_NAME, 0, namelen), - // are we last name? - (strchr(name, '/') == NULL) ? id : NULL, + id, lfs2_dir_find_match, &(struct lfs2_dir_find_match){ lfs2, name, namelen}); if (tag < 0) { @@ -2105,13 +2156,14 @@ static int lfs2_dir_splittingcompact(lfs2_t *lfs2, lfs2_mdir_t *dir, // And we cap at half a block to avoid degenerate cases with // nearly-full metadata blocks. // + lfs2_size_t metadata_max = (lfs2->cfg->metadata_max) + ? lfs2->cfg->metadata_max + : lfs2->cfg->block_size; if (end - split < 0xff && size <= lfs2_min( - lfs2->cfg->block_size - 40, + metadata_max - 40, lfs2_alignup( - (lfs2->cfg->metadata_max - ? lfs2->cfg->metadata_max - : lfs2->cfg->block_size)/2, + metadata_max/2, lfs2->cfg->prog_size))) { break; } @@ -2146,14 +2198,16 @@ static int lfs2_dir_splittingcompact(lfs2_t *lfs2, lfs2_mdir_t *dir, && lfs2_pair_cmp(dir->pair, (const lfs2_block_t[2]){0, 1}) == 0) { // oh no! we're writing too much to the superblock, // should we expand? - lfs2_ssize_t size = lfs2_fs_rawsize(lfs2); + lfs2_ssize_t size = lfs2_fs_size_(lfs2); if (size < 0) { return size; } - // do we have extra space? littlefs can't reclaim this space - // by itself, so expand cautiously - if ((lfs2_size_t)size < lfs2->block_count/2) { + // littlefs cannot reclaim expanded superblocks, so expand cautiously + // + // if our filesystem is more than ~88% full, don't expand, this is + // somewhat arbitrary + if (lfs2->block_count - size > lfs2->block_count/8) { LFS2_DEBUG("Expanding superblock at rev %"PRIu32, dir->rev); int err = lfs2_dir_split(lfs2, dir, attrs, attrcount, source, begin, end); @@ -2166,7 +2220,8 @@ static int lfs2_dir_splittingcompact(lfs2_t *lfs2, lfs2_mdir_t *dir, // we can do, we'll error later if we've become frozen LFS2_WARN("Unable to expand superblock"); } else { - end = begin; + // duplicate the superblock entry into the new superblock + end = 1; } } } @@ -2213,7 +2268,7 @@ static int lfs2_dir_relocatingcommit(lfs2_t *lfs2, lfs2_mdir_t *dir, } } - if (dir->erased) { + if (dir->erased && dir->count < 0xff) { // try to commit struct lfs2_commit commit = { .block = dir->pair[0], @@ -2312,7 +2367,8 @@ fixmlist:; if (d->m.pair != pair) { for (int i = 0; i < attrcount; i++) { if (lfs2_tag_type3(attrs[i].tag) == LFS2_TYPE_DELETE && - d->id == lfs2_tag_id(attrs[i].tag)) { + d->id == lfs2_tag_id(attrs[i].tag) && + d->type != LFS2_TYPE_DIR) { d->m.pair[0] = LFS2_BLOCK_NULL; d->m.pair[1] = LFS2_BLOCK_NULL; } else if (lfs2_tag_type3(attrs[i].tag) == LFS2_TYPE_DELETE && @@ -2333,7 +2389,9 @@ fixmlist:; while (d->id >= d->m.count && d->m.split) { // we split and id is on tail now - d->id -= d->m.count; + if (lfs2_pair_cmp(d->m.tail, lfs2->root) != 0) { + d->id -= d->m.count; + } int err = lfs2_dir_fetch(lfs2, &d->m, d->m.tail); if (err) { return err; @@ -2499,7 +2557,7 @@ static int lfs2_dir_orphaningcommit(lfs2_t *lfs2, lfs2_mdir_t *dir, if (err != LFS2_ERR_NOENT) { if (lfs2_gstate_hasorphans(&lfs2->gstate)) { // next step, clean up orphans - err = lfs2_fs_preporphans(lfs2, -hasparent); + err = lfs2_fs_preporphans(lfs2, -(int8_t)hasparent); if (err) { return err; } @@ -2564,7 +2622,7 @@ static int lfs2_dir_commit(lfs2_t *lfs2, lfs2_mdir_t *dir, /// Top level directory operations /// #ifndef LFS2_READONLY -static int lfs2_rawmkdir(lfs2_t *lfs2, const char *path) { +static int lfs2_mkdir_(lfs2_t *lfs2, const char *path) { // deorphan if we haven't yet, needed at most once after poweron int err = lfs2_fs_forceconsistency(lfs2); if (err) { @@ -2575,18 +2633,18 @@ static int lfs2_rawmkdir(lfs2_t *lfs2, const char *path) { cwd.next = lfs2->mlist; uint16_t id; err = lfs2_dir_find(lfs2, &cwd.m, &path, &id); - if (!(err == LFS2_ERR_NOENT && id != 0x3ff)) { + if (!(err == LFS2_ERR_NOENT && lfs2_path_islast(path))) { return (err < 0) ? err : LFS2_ERR_EXIST; } // check that name fits - lfs2_size_t nlen = strlen(path); + lfs2_size_t nlen = lfs2_path_namelen(path); if (nlen > lfs2->name_max) { return LFS2_ERR_NAMETOOLONG; } // build up new directory - lfs2_alloc_ack(lfs2); + lfs2_alloc_ckpoint(lfs2); lfs2_mdir_t dir; err = lfs2_dir_alloc(lfs2, &dir); if (err) { @@ -2660,7 +2718,7 @@ static int lfs2_rawmkdir(lfs2_t *lfs2, const char *path) { } #endif -static int lfs2_dir_rawopen(lfs2_t *lfs2, lfs2_dir_t *dir, const char *path) { +static int lfs2_dir_open_(lfs2_t *lfs2, lfs2_dir_t *dir, const char *path) { lfs2_stag_t tag = lfs2_dir_find(lfs2, &dir->m, &path, NULL); if (tag < 0) { return tag; @@ -2704,14 +2762,14 @@ static int lfs2_dir_rawopen(lfs2_t *lfs2, lfs2_dir_t *dir, const char *path) { return 0; } -static int lfs2_dir_rawclose(lfs2_t *lfs2, lfs2_dir_t *dir) { +static int lfs2_dir_close_(lfs2_t *lfs2, lfs2_dir_t *dir) { // remove from list of mdirs lfs2_mlist_remove(lfs2, (struct lfs2_mlist *)dir); return 0; } -static int lfs2_dir_rawread(lfs2_t *lfs2, lfs2_dir_t *dir, struct lfs2_info *info) { +static int lfs2_dir_read_(lfs2_t *lfs2, lfs2_dir_t *dir, struct lfs2_info *info) { memset(info, 0, sizeof(*info)); // special offset for '.' and '..' @@ -2756,9 +2814,9 @@ static int lfs2_dir_rawread(lfs2_t *lfs2, lfs2_dir_t *dir, struct lfs2_info *inf return true; } -static int lfs2_dir_rawseek(lfs2_t *lfs2, lfs2_dir_t *dir, lfs2_off_t off) { +static int lfs2_dir_seek_(lfs2_t *lfs2, lfs2_dir_t *dir, lfs2_off_t off) { // simply walk from head dir - int err = lfs2_dir_rawrewind(lfs2, dir); + int err = lfs2_dir_rewind_(lfs2, dir); if (err) { return err; } @@ -2793,12 +2851,12 @@ static int lfs2_dir_rawseek(lfs2_t *lfs2, lfs2_dir_t *dir, lfs2_off_t off) { return 0; } -static lfs2_soff_t lfs2_dir_rawtell(lfs2_t *lfs2, lfs2_dir_t *dir) { +static lfs2_soff_t lfs2_dir_tell_(lfs2_t *lfs2, lfs2_dir_t *dir) { (void)lfs2; return dir->pos; } -static int lfs2_dir_rawrewind(lfs2_t *lfs2, lfs2_dir_t *dir) { +static int lfs2_dir_rewind_(lfs2_t *lfs2, lfs2_dir_t *dir) { // reload the head dir int err = lfs2_dir_fetch(lfs2, &dir->m, dir->head); if (err) { @@ -3004,7 +3062,7 @@ static int lfs2_ctz_traverse(lfs2_t *lfs2, /// Top level file operations /// -static int lfs2_file_rawopencfg(lfs2_t *lfs2, lfs2_file_t *file, +static int lfs2_file_opencfg_(lfs2_t *lfs2, lfs2_file_t *file, const char *path, int flags, const struct lfs2_file_config *cfg) { #ifndef LFS2_READONLY @@ -3029,7 +3087,7 @@ static int lfs2_file_rawopencfg(lfs2_t *lfs2, lfs2_file_t *file, // allocate entry for file if it doesn't exist lfs2_stag_t tag = lfs2_dir_find(lfs2, &file->m, &path, &file->id); - if (tag < 0 && !(tag == LFS2_ERR_NOENT && file->id != 0x3ff)) { + if (tag < 0 && !(tag == LFS2_ERR_NOENT && lfs2_path_islast(path))) { err = tag; goto cleanup; } @@ -3049,8 +3107,14 @@ static int lfs2_file_rawopencfg(lfs2_t *lfs2, lfs2_file_t *file, goto cleanup; } + // don't allow trailing slashes + if (lfs2_path_isdir(path)) { + err = LFS2_ERR_NOTDIR; + goto cleanup; + } + // check that name fits - lfs2_size_t nlen = strlen(path); + lfs2_size_t nlen = lfs2_path_namelen(path); if (nlen > lfs2->name_max) { err = LFS2_ERR_NAMETOOLONG; goto cleanup; @@ -3166,22 +3230,22 @@ static int lfs2_file_rawopencfg(lfs2_t *lfs2, lfs2_file_t *file, #ifndef LFS2_READONLY file->flags |= LFS2_F_ERRED; #endif - lfs2_file_rawclose(lfs2, file); + lfs2_file_close_(lfs2, file); return err; } #ifndef LFS2_NO_MALLOC -static int lfs2_file_rawopen(lfs2_t *lfs2, lfs2_file_t *file, +static int lfs2_file_open_(lfs2_t *lfs2, lfs2_file_t *file, const char *path, int flags) { static const struct lfs2_file_config defaults = {0}; - int err = lfs2_file_rawopencfg(lfs2, file, path, flags, &defaults); + int err = lfs2_file_opencfg_(lfs2, file, path, flags, &defaults); return err; } #endif -static int lfs2_file_rawclose(lfs2_t *lfs2, lfs2_file_t *file) { +static int lfs2_file_close_(lfs2_t *lfs2, lfs2_file_t *file) { #ifndef LFS2_READONLY - int err = lfs2_file_rawsync(lfs2, file); + int err = lfs2_file_sync_(lfs2, file); #else int err = 0; #endif @@ -3272,7 +3336,7 @@ static int lfs2_file_relocate(lfs2_t *lfs2, lfs2_file_t *file) { #ifndef LFS2_READONLY static int lfs2_file_outline(lfs2_t *lfs2, lfs2_file_t *file) { file->off = file->pos; - lfs2_alloc_ack(lfs2); + lfs2_alloc_ckpoint(lfs2); int err = lfs2_file_relocate(lfs2, file); if (err) { return err; @@ -3364,7 +3428,7 @@ static int lfs2_file_flush(lfs2_t *lfs2, lfs2_file_t *file) { } #ifndef LFS2_READONLY -static int lfs2_file_rawsync(lfs2_t *lfs2, lfs2_file_t *file) { +static int lfs2_file_sync_(lfs2_t *lfs2, lfs2_file_t *file) { if (file->flags & LFS2_F_ERRED) { // it's not safe to do anything if our file errored return 0; @@ -3379,6 +3443,15 @@ static int lfs2_file_rawsync(lfs2_t *lfs2, lfs2_file_t *file) { if ((file->flags & LFS2_F_DIRTY) && !lfs2_pair_isnull(file->m.pair)) { + // before we commit metadata, we need sync the disk to make sure + // data writes don't complete after metadata writes + if (!(file->flags & LFS2_F_INLINE)) { + err = lfs2_bd_sync(lfs2, &lfs2->pcache, &lfs2->rcache, false); + if (err) { + return err; + } + } + // update dir entry uint16_t type; const void *buffer; @@ -3477,7 +3550,7 @@ static lfs2_ssize_t lfs2_file_flushedread(lfs2_t *lfs2, lfs2_file_t *file, return size; } -static lfs2_ssize_t lfs2_file_rawread(lfs2_t *lfs2, lfs2_file_t *file, +static lfs2_ssize_t lfs2_file_read_(lfs2_t *lfs2, lfs2_file_t *file, void *buffer, lfs2_size_t size) { LFS2_ASSERT((file->flags & LFS2_O_RDONLY) == LFS2_O_RDONLY); @@ -3502,11 +3575,7 @@ static lfs2_ssize_t lfs2_file_flushedwrite(lfs2_t *lfs2, lfs2_file_t *file, lfs2_size_t nsize = size; if ((file->flags & LFS2_F_INLINE) && - lfs2_max(file->pos+nsize, file->ctz.size) > - lfs2_min(0x3fe, lfs2_min( - lfs2->cfg->cache_size, - (lfs2->cfg->metadata_max ? - lfs2->cfg->metadata_max : lfs2->cfg->block_size) / 8))) { + lfs2_max(file->pos+nsize, file->ctz.size) > lfs2->inline_max) { // inline file doesn't fit anymore int err = lfs2_file_outline(lfs2, file); if (err) { @@ -3535,7 +3604,7 @@ static lfs2_ssize_t lfs2_file_flushedwrite(lfs2_t *lfs2, lfs2_file_t *file, } // extend file with new blocks - lfs2_alloc_ack(lfs2); + lfs2_alloc_ckpoint(lfs2); int err = lfs2_ctz_extend(lfs2, &file->cache, &lfs2->rcache, file->block, file->pos, &file->block, &file->off); @@ -3578,13 +3647,13 @@ static lfs2_ssize_t lfs2_file_flushedwrite(lfs2_t *lfs2, lfs2_file_t *file, data += diff; nsize -= diff; - lfs2_alloc_ack(lfs2); + lfs2_alloc_ckpoint(lfs2); } return size; } -static lfs2_ssize_t lfs2_file_rawwrite(lfs2_t *lfs2, lfs2_file_t *file, +static lfs2_ssize_t lfs2_file_write_(lfs2_t *lfs2, lfs2_file_t *file, const void *buffer, lfs2_size_t size) { LFS2_ASSERT((file->flags & LFS2_O_WRONLY) == LFS2_O_WRONLY); @@ -3628,25 +3697,19 @@ static lfs2_ssize_t lfs2_file_rawwrite(lfs2_t *lfs2, lfs2_file_t *file, } #endif -static lfs2_soff_t lfs2_file_rawseek(lfs2_t *lfs2, lfs2_file_t *file, +static lfs2_soff_t lfs2_file_seek_(lfs2_t *lfs2, lfs2_file_t *file, lfs2_soff_t off, int whence) { // find new pos + // + // fortunately for us, littlefs is limited to 31-bit file sizes, so we + // don't have to worry too much about integer overflow lfs2_off_t npos = file->pos; if (whence == LFS2_SEEK_SET) { npos = off; } else if (whence == LFS2_SEEK_CUR) { - if ((lfs2_soff_t)file->pos + off < 0) { - return LFS2_ERR_INVAL; - } else { - npos = file->pos + off; - } + npos = file->pos + (lfs2_off_t)off; } else if (whence == LFS2_SEEK_END) { - lfs2_soff_t res = lfs2_file_rawsize(lfs2, file) + off; - if (res < 0) { - return LFS2_ERR_INVAL; - } else { - npos = res; - } + npos = (lfs2_off_t)lfs2_file_size_(lfs2, file) + (lfs2_off_t)off; } if (npos > lfs2->file_max) { @@ -3661,13 +3724,8 @@ static lfs2_soff_t lfs2_file_rawseek(lfs2_t *lfs2, lfs2_file_t *file, // if we're only reading and our new offset is still in the file's cache // we can avoid flushing and needing to reread the data - if ( -#ifndef LFS2_READONLY - !(file->flags & LFS2_F_WRITING) -#else - true -#endif - ) { + if ((file->flags & LFS2_F_READING) + && file->off != lfs2->cfg->block_size) { int oindex = lfs2_ctz_index(lfs2, &(lfs2_off_t){file->pos}); lfs2_off_t noff = npos; int nindex = lfs2_ctz_index(lfs2, &noff); @@ -3692,7 +3750,7 @@ static lfs2_soff_t lfs2_file_rawseek(lfs2_t *lfs2, lfs2_file_t *file, } #ifndef LFS2_READONLY -static int lfs2_file_rawtruncate(lfs2_t *lfs2, lfs2_file_t *file, lfs2_off_t size) { +static int lfs2_file_truncate_(lfs2_t *lfs2, lfs2_file_t *file, lfs2_off_t size) { LFS2_ASSERT((file->flags & LFS2_O_WRONLY) == LFS2_O_WRONLY); if (size > LFS2_FILE_MAX) { @@ -3700,15 +3758,12 @@ static int lfs2_file_rawtruncate(lfs2_t *lfs2, lfs2_file_t *file, lfs2_off_t siz } lfs2_off_t pos = file->pos; - lfs2_off_t oldsize = lfs2_file_rawsize(lfs2, file); + lfs2_off_t oldsize = lfs2_file_size_(lfs2, file); if (size < oldsize) { // revert to inline file? - if (size <= lfs2_min(0x3fe, lfs2_min( - lfs2->cfg->cache_size, - (lfs2->cfg->metadata_max ? - lfs2->cfg->metadata_max : lfs2->cfg->block_size) / 8))) { + if (size <= lfs2->inline_max) { // flush+seek to head - lfs2_soff_t res = lfs2_file_rawseek(lfs2, file, 0, LFS2_SEEK_SET); + lfs2_soff_t res = lfs2_file_seek_(lfs2, file, 0, LFS2_SEEK_SET); if (res < 0) { return (int)res; } @@ -3753,14 +3808,14 @@ static int lfs2_file_rawtruncate(lfs2_t *lfs2, lfs2_file_t *file, lfs2_off_t siz } } else if (size > oldsize) { // flush+seek if not already at end - lfs2_soff_t res = lfs2_file_rawseek(lfs2, file, 0, LFS2_SEEK_END); + lfs2_soff_t res = lfs2_file_seek_(lfs2, file, 0, LFS2_SEEK_END); if (res < 0) { return (int)res; } // fill with zeros while (file->pos < size) { - res = lfs2_file_rawwrite(lfs2, file, &(uint8_t){0}, 1); + res = lfs2_file_write_(lfs2, file, &(uint8_t){0}, 1); if (res < 0) { return (int)res; } @@ -3768,7 +3823,7 @@ static int lfs2_file_rawtruncate(lfs2_t *lfs2, lfs2_file_t *file, lfs2_off_t siz } // restore pos - lfs2_soff_t res = lfs2_file_rawseek(lfs2, file, pos, LFS2_SEEK_SET); + lfs2_soff_t res = lfs2_file_seek_(lfs2, file, pos, LFS2_SEEK_SET); if (res < 0) { return (int)res; } @@ -3777,13 +3832,13 @@ static int lfs2_file_rawtruncate(lfs2_t *lfs2, lfs2_file_t *file, lfs2_off_t siz } #endif -static lfs2_soff_t lfs2_file_rawtell(lfs2_t *lfs2, lfs2_file_t *file) { +static lfs2_soff_t lfs2_file_tell_(lfs2_t *lfs2, lfs2_file_t *file) { (void)lfs2; return file->pos; } -static int lfs2_file_rawrewind(lfs2_t *lfs2, lfs2_file_t *file) { - lfs2_soff_t res = lfs2_file_rawseek(lfs2, file, 0, LFS2_SEEK_SET); +static int lfs2_file_rewind_(lfs2_t *lfs2, lfs2_file_t *file) { + lfs2_soff_t res = lfs2_file_seek_(lfs2, file, 0, LFS2_SEEK_SET); if (res < 0) { return (int)res; } @@ -3791,7 +3846,7 @@ static int lfs2_file_rawrewind(lfs2_t *lfs2, lfs2_file_t *file) { return 0; } -static lfs2_soff_t lfs2_file_rawsize(lfs2_t *lfs2, lfs2_file_t *file) { +static lfs2_soff_t lfs2_file_size_(lfs2_t *lfs2, lfs2_file_t *file) { (void)lfs2; #ifndef LFS2_READONLY @@ -3805,18 +3860,24 @@ static lfs2_soff_t lfs2_file_rawsize(lfs2_t *lfs2, lfs2_file_t *file) { /// General fs operations /// -static int lfs2_rawstat(lfs2_t *lfs2, const char *path, struct lfs2_info *info) { +static int lfs2_stat_(lfs2_t *lfs2, const char *path, struct lfs2_info *info) { lfs2_mdir_t cwd; lfs2_stag_t tag = lfs2_dir_find(lfs2, &cwd, &path, NULL); if (tag < 0) { return (int)tag; } + // only allow trailing slashes on dirs + if (strchr(path, '/') != NULL + && lfs2_tag_type3(tag) != LFS2_TYPE_DIR) { + return LFS2_ERR_NOTDIR; + } + return lfs2_dir_getinfo(lfs2, &cwd, lfs2_tag_id(tag), info); } #ifndef LFS2_READONLY -static int lfs2_rawremove(lfs2_t *lfs2, const char *path) { +static int lfs2_remove_(lfs2_t *lfs2, const char *path) { // deorphan if we haven't yet, needed at most once after poweron int err = lfs2_fs_forceconsistency(lfs2); if (err) { @@ -3872,7 +3933,9 @@ static int lfs2_rawremove(lfs2_t *lfs2, const char *path) { } lfs2->mlist = dir.next; - if (lfs2_tag_type3(tag) == LFS2_TYPE_DIR) { + if (lfs2_gstate_hasorphans(&lfs2->gstate)) { + LFS2_ASSERT(lfs2_tag_type3(tag) == LFS2_TYPE_DIR); + // fix orphan err = lfs2_fs_preporphans(lfs2, -1); if (err) { @@ -3895,7 +3958,7 @@ static int lfs2_rawremove(lfs2_t *lfs2, const char *path) { #endif #ifndef LFS2_READONLY -static int lfs2_rawrename(lfs2_t *lfs2, const char *oldpath, const char *newpath) { +static int lfs2_rename_(lfs2_t *lfs2, const char *oldpath, const char *newpath) { // deorphan if we haven't yet, needed at most once after poweron int err = lfs2_fs_forceconsistency(lfs2); if (err) { @@ -3914,7 +3977,7 @@ static int lfs2_rawrename(lfs2_t *lfs2, const char *oldpath, const char *newpath uint16_t newid; lfs2_stag_t prevtag = lfs2_dir_find(lfs2, &newcwd, &newpath, &newid); if ((prevtag < 0 || lfs2_tag_id(prevtag) == 0x3ff) && - !(prevtag == LFS2_ERR_NOENT && newid != 0x3ff)) { + !(prevtag == LFS2_ERR_NOENT && lfs2_path_islast(newpath))) { return (prevtag < 0) ? (int)prevtag : LFS2_ERR_INVAL; } @@ -3925,8 +3988,14 @@ static int lfs2_rawrename(lfs2_t *lfs2, const char *oldpath, const char *newpath struct lfs2_mlist prevdir; prevdir.next = lfs2->mlist; if (prevtag == LFS2_ERR_NOENT) { + // if we're a file, don't allow trailing slashes + if (lfs2_path_isdir(newpath) + && lfs2_tag_type3(oldtag) != LFS2_TYPE_DIR) { + return LFS2_ERR_NOTDIR; + } + // check that name fits - lfs2_size_t nlen = strlen(newpath); + lfs2_size_t nlen = lfs2_path_namelen(newpath); if (nlen > lfs2->name_max) { return LFS2_ERR_NAMETOOLONG; } @@ -3938,7 +4007,9 @@ static int lfs2_rawrename(lfs2_t *lfs2, const char *oldpath, const char *newpath newoldid += 1; } } else if (lfs2_tag_type3(prevtag) != lfs2_tag_type3(oldtag)) { - return LFS2_ERR_ISDIR; + return (lfs2_tag_type3(prevtag) == LFS2_TYPE_DIR) + ? LFS2_ERR_ISDIR + : LFS2_ERR_NOTDIR; } else if (samepair && newid == newoldid) { // we're renaming to ourselves?? return 0; @@ -3984,7 +4055,8 @@ static int lfs2_rawrename(lfs2_t *lfs2, const char *oldpath, const char *newpath {LFS2_MKTAG_IF(prevtag != LFS2_ERR_NOENT, LFS2_TYPE_DELETE, newid, 0), NULL}, {LFS2_MKTAG(LFS2_TYPE_CREATE, newid, 0), NULL}, - {LFS2_MKTAG(lfs2_tag_type3(oldtag), newid, strlen(newpath)), newpath}, + {LFS2_MKTAG(lfs2_tag_type3(oldtag), + newid, lfs2_path_namelen(newpath)), newpath}, {LFS2_MKTAG(LFS2_FROM_MOVE, newid, lfs2_tag_id(oldtag)), &oldcwd}, {LFS2_MKTAG_IF(samepair, LFS2_TYPE_DELETE, newoldid, 0), NULL})); @@ -4007,8 +4079,10 @@ static int lfs2_rawrename(lfs2_t *lfs2, const char *oldpath, const char *newpath } lfs2->mlist = prevdir.next; - if (prevtag != LFS2_ERR_NOENT - && lfs2_tag_type3(prevtag) == LFS2_TYPE_DIR) { + if (lfs2_gstate_hasorphans(&lfs2->gstate)) { + LFS2_ASSERT(prevtag != LFS2_ERR_NOENT + && lfs2_tag_type3(prevtag) == LFS2_TYPE_DIR); + // fix orphan err = lfs2_fs_preporphans(lfs2, -1); if (err) { @@ -4030,7 +4104,7 @@ static int lfs2_rawrename(lfs2_t *lfs2, const char *oldpath, const char *newpath } #endif -static lfs2_ssize_t lfs2_rawgetattr(lfs2_t *lfs2, const char *path, +static lfs2_ssize_t lfs2_getattr_(lfs2_t *lfs2, const char *path, uint8_t type, void *buffer, lfs2_size_t size) { lfs2_mdir_t cwd; lfs2_stag_t tag = lfs2_dir_find(lfs2, &cwd, &path, NULL); @@ -4088,7 +4162,7 @@ static int lfs2_commitattr(lfs2_t *lfs2, const char *path, #endif #ifndef LFS2_READONLY -static int lfs2_rawsetattr(lfs2_t *lfs2, const char *path, +static int lfs2_setattr_(lfs2_t *lfs2, const char *path, uint8_t type, const void *buffer, lfs2_size_t size) { if (size > lfs2->attr_max) { return LFS2_ERR_NOSPC; @@ -4099,13 +4173,28 @@ static int lfs2_rawsetattr(lfs2_t *lfs2, const char *path, #endif #ifndef LFS2_READONLY -static int lfs2_rawremoveattr(lfs2_t *lfs2, const char *path, uint8_t type) { +static int lfs2_removeattr_(lfs2_t *lfs2, const char *path, uint8_t type) { return lfs2_commitattr(lfs2, path, type, NULL, 0x3ff); } #endif /// Filesystem operations /// + +// compile time checks, see lfs2.h for why these limits exist +#if LFS2_NAME_MAX > 1022 +#error "Invalid LFS2_NAME_MAX, must be <= 1022" +#endif + +#if LFS2_FILE_MAX > 2147483647 +#error "Invalid LFS2_FILE_MAX, must be <= 2147483647" +#endif + +#if LFS2_ATTR_MAX > 1022 +#error "Invalid LFS2_ATTR_MAX, must be <= 1022" +#endif + +// common filesystem initialization static int lfs2_init(lfs2_t *lfs2, const struct lfs2_config *cfg) { lfs2->cfg = cfg; lfs2->block_count = cfg->block_count; // May be 0 @@ -4126,6 +4215,14 @@ static int lfs2_init(lfs2_t *lfs2, const struct lfs2_config *cfg) { // which littlefs currently does not support LFS2_ASSERT((bool)0x80000000); + // check that the required io functions are provided + LFS2_ASSERT(lfs2->cfg->read != NULL); +#ifndef LFS2_READONLY + LFS2_ASSERT(lfs2->cfg->prog != NULL); + LFS2_ASSERT(lfs2->cfg->erase != NULL); + LFS2_ASSERT(lfs2->cfg->sync != NULL); +#endif + // validate that the lfs2-cfg sizes were initiated properly before // performing any arithmetic logics with them LFS2_ASSERT(lfs2->cfg->read_size != 0); @@ -4153,6 +4250,23 @@ static int lfs2_init(lfs2_t *lfs2, const struct lfs2_config *cfg) { // wear-leveling. LFS2_ASSERT(lfs2->cfg->block_cycles != 0); + // check that compact_thresh makes sense + // + // metadata can't be compacted below block_size/2, and metadata can't + // exceed a block_size + LFS2_ASSERT(lfs2->cfg->compact_thresh == 0 + || lfs2->cfg->compact_thresh >= lfs2->cfg->block_size/2); + LFS2_ASSERT(lfs2->cfg->compact_thresh == (lfs2_size_t)-1 + || lfs2->cfg->compact_thresh <= lfs2->cfg->block_size); + + // check that metadata_max is a multiple of read_size and prog_size, + // and a factor of the block_size + LFS2_ASSERT(!lfs2->cfg->metadata_max + || lfs2->cfg->metadata_max % lfs2->cfg->read_size == 0); + LFS2_ASSERT(!lfs2->cfg->metadata_max + || lfs2->cfg->metadata_max % lfs2->cfg->prog_size == 0); + LFS2_ASSERT(!lfs2->cfg->metadata_max + || lfs2->cfg->block_size % lfs2->cfg->metadata_max == 0); // setup read cache if (lfs2->cfg->read_buffer) { @@ -4180,15 +4294,14 @@ static int lfs2_init(lfs2_t *lfs2, const struct lfs2_config *cfg) { lfs2_cache_zero(lfs2, &lfs2->rcache); lfs2_cache_zero(lfs2, &lfs2->pcache); - // setup lookahead, must be multiple of 64-bits, 32-bit aligned + // setup lookahead buffer, note mount finishes initializing this after + // we establish a decent pseudo-random seed LFS2_ASSERT(lfs2->cfg->lookahead_size > 0); - LFS2_ASSERT(lfs2->cfg->lookahead_size % 8 == 0 && - (uintptr_t)lfs2->cfg->lookahead_buffer % 4 == 0); if (lfs2->cfg->lookahead_buffer) { - lfs2->free.buffer = lfs2->cfg->lookahead_buffer; + lfs2->lookahead.buffer = lfs2->cfg->lookahead_buffer; } else { - lfs2->free.buffer = lfs2_malloc(lfs2->cfg->lookahead_size); - if (!lfs2->free.buffer) { + lfs2->lookahead.buffer = lfs2_malloc(lfs2->cfg->lookahead_size); + if (!lfs2->lookahead.buffer) { err = LFS2_ERR_NOMEM; goto cleanup; } @@ -4215,6 +4328,27 @@ static int lfs2_init(lfs2_t *lfs2, const struct lfs2_config *cfg) { LFS2_ASSERT(lfs2->cfg->metadata_max <= lfs2->cfg->block_size); + LFS2_ASSERT(lfs2->cfg->inline_max == (lfs2_size_t)-1 + || lfs2->cfg->inline_max <= lfs2->cfg->cache_size); + LFS2_ASSERT(lfs2->cfg->inline_max == (lfs2_size_t)-1 + || lfs2->cfg->inline_max <= lfs2->attr_max); + LFS2_ASSERT(lfs2->cfg->inline_max == (lfs2_size_t)-1 + || lfs2->cfg->inline_max <= ((lfs2->cfg->metadata_max) + ? lfs2->cfg->metadata_max + : lfs2->cfg->block_size)/8); + lfs2->inline_max = lfs2->cfg->inline_max; + if (lfs2->inline_max == (lfs2_size_t)-1) { + lfs2->inline_max = 0; + } else if (lfs2->inline_max == 0) { + lfs2->inline_max = lfs2_min( + lfs2->cfg->cache_size, + lfs2_min( + lfs2->attr_max, + ((lfs2->cfg->metadata_max) + ? lfs2->cfg->metadata_max + : lfs2->cfg->block_size)/8)); + } + // setup default state lfs2->root[0] = LFS2_BLOCK_NULL; lfs2->root[1] = LFS2_BLOCK_NULL; @@ -4245,7 +4379,7 @@ static int lfs2_deinit(lfs2_t *lfs2) { } if (!lfs2->cfg->lookahead_buffer) { - lfs2_free(lfs2->free.buffer); + lfs2_free(lfs2->lookahead.buffer); } return 0; @@ -4254,7 +4388,7 @@ static int lfs2_deinit(lfs2_t *lfs2) { #ifndef LFS2_READONLY -static int lfs2_rawformat(lfs2_t *lfs2, const struct lfs2_config *cfg) { +static int lfs2_format_(lfs2_t *lfs2, const struct lfs2_config *cfg) { int err = 0; { err = lfs2_init(lfs2, cfg); @@ -4265,12 +4399,12 @@ static int lfs2_rawformat(lfs2_t *lfs2, const struct lfs2_config *cfg) { LFS2_ASSERT(cfg->block_count != 0); // create free lookahead - memset(lfs2->free.buffer, 0, lfs2->cfg->lookahead_size); - lfs2->free.off = 0; - lfs2->free.size = lfs2_min(8*lfs2->cfg->lookahead_size, + memset(lfs2->lookahead.buffer, 0, lfs2->cfg->lookahead_size); + lfs2->lookahead.start = 0; + lfs2->lookahead.size = lfs2_min(8*lfs2->cfg->lookahead_size, lfs2->block_count); - lfs2->free.i = 0; - lfs2_alloc_ack(lfs2); + lfs2->lookahead.next = 0; + lfs2_alloc_ckpoint(lfs2); // create root dir lfs2_mdir_t root; @@ -4321,7 +4455,31 @@ static int lfs2_rawformat(lfs2_t *lfs2, const struct lfs2_config *cfg) { } #endif -static int lfs2_rawmount(lfs2_t *lfs2, const struct lfs2_config *cfg) { +struct lfs2_tortoise_t { + lfs2_block_t pair[2]; + lfs2_size_t i; + lfs2_size_t period; +}; + +static int lfs2_tortoise_detectcycles( + const lfs2_mdir_t *dir, struct lfs2_tortoise_t *tortoise) { + // detect cycles with Brent's algorithm + if (lfs2_pair_issync(dir->tail, tortoise->pair)) { + LFS2_WARN("Cycle detected in tail list"); + return LFS2_ERR_CORRUPT; + } + if (tortoise->i == tortoise->period) { + tortoise->pair[0] = dir->tail[0]; + tortoise->pair[1] = dir->tail[1]; + tortoise->i = 0; + tortoise->period *= 2; + } + tortoise->i += 1; + + return LFS2_ERR_OK; +} + +static int lfs2_mount_(lfs2_t *lfs2, const struct lfs2_config *cfg) { int err = lfs2_init(lfs2, cfg); if (err) { return err; @@ -4329,23 +4487,16 @@ static int lfs2_rawmount(lfs2_t *lfs2, const struct lfs2_config *cfg) { // scan directory blocks for superblock and any global updates lfs2_mdir_t dir = {.tail = {0, 1}}; - lfs2_block_t tortoise[2] = {LFS2_BLOCK_NULL, LFS2_BLOCK_NULL}; - lfs2_size_t tortoise_i = 1; - lfs2_size_t tortoise_period = 1; + struct lfs2_tortoise_t tortoise = { + .pair = {LFS2_BLOCK_NULL, LFS2_BLOCK_NULL}, + .i = 1, + .period = 1, + }; while (!lfs2_pair_isnull(dir.tail)) { - // detect cycles with Brent's algorithm - if (lfs2_pair_issync(dir.tail, tortoise)) { - LFS2_WARN("Cycle detected in tail list"); - err = LFS2_ERR_CORRUPT; + err = lfs2_tortoise_detectcycles(&dir, &tortoise); + if (err < 0) { goto cleanup; } - if (tortoise_i == tortoise_period) { - tortoise[0] = dir.tail[0]; - tortoise[1] = dir.tail[1]; - tortoise_i = 0; - tortoise_period *= 2; - } - tortoise_i += 1; // fetch next block in tail list lfs2_stag_t tag = lfs2_dir_fetchmatch(lfs2, &dir, dir.tail, @@ -4394,6 +4545,7 @@ static int lfs2_rawmount(lfs2_t *lfs2, const struct lfs2_config *cfg) { // found older minor version? set an in-device only bit in the // gstate so we know we need to rewrite the superblock before // the first write + bool needssuperblock = false; if (minor_version < lfs2_fs_disk_version_minor(lfs2)) { LFS2_DEBUG("Found older minor version " "v%"PRIu16".%"PRIu16" < v%"PRIu16".%"PRIu16, @@ -4401,10 +4553,11 @@ static int lfs2_rawmount(lfs2_t *lfs2, const struct lfs2_config *cfg) { minor_version, lfs2_fs_disk_version_major(lfs2), lfs2_fs_disk_version_minor(lfs2)); - // note this bit is reserved on disk, so fetching more gstate - // will not interfere here - lfs2_fs_prepsuperblock(lfs2, true); + needssuperblock = true; } + // note this bit is reserved on disk, so fetching more gstate + // will not interfere here + lfs2_fs_prepsuperblock(lfs2, needssuperblock); // check superblock configuration if (superblock.name_max) { @@ -4438,6 +4591,9 @@ static int lfs2_rawmount(lfs2_t *lfs2, const struct lfs2_config *cfg) { } lfs2->attr_max = superblock.attr_max; + + // we also need to update inline_max in case attr_max changed + lfs2->inline_max = lfs2_min(lfs2->inline_max, lfs2->attr_max); } // this is where we get the block_count from disk if block_count=0 @@ -4478,23 +4634,23 @@ static int lfs2_rawmount(lfs2_t *lfs2, const struct lfs2_config *cfg) { // setup free lookahead, to distribute allocations uniformly across // boots, we start the allocator at a random location - lfs2->free.off = lfs2->seed % lfs2->block_count; + lfs2->lookahead.start = lfs2->seed % lfs2->block_count; lfs2_alloc_drop(lfs2); return 0; cleanup: - lfs2_rawunmount(lfs2); + lfs2_unmount_(lfs2); return err; } -static int lfs2_rawunmount(lfs2_t *lfs2) { +static int lfs2_unmount_(lfs2_t *lfs2) { return lfs2_deinit(lfs2); } /// Filesystem filesystem operations /// -static int lfs2_fs_rawstat(lfs2_t *lfs2, struct lfs2_fsinfo *fsinfo) { +static int lfs2_fs_stat_(lfs2_t *lfs2, struct lfs2_fsinfo *fsinfo) { // if the superblock is up-to-date, we must be on the most recent // minor version of littlefs if (!lfs2_gstate_needssuperblock(&lfs2->gstate)) { @@ -4534,7 +4690,7 @@ static int lfs2_fs_rawstat(lfs2_t *lfs2, struct lfs2_fsinfo *fsinfo) { return 0; } -int lfs2_fs_rawtraverse(lfs2_t *lfs2, +int lfs2_fs_traverse_(lfs2_t *lfs2, int (*cb)(void *data, lfs2_block_t block), void *data, bool includeorphans) { // iterate over metadata pairs @@ -4553,22 +4709,17 @@ int lfs2_fs_rawtraverse(lfs2_t *lfs2, } #endif - lfs2_block_t tortoise[2] = {LFS2_BLOCK_NULL, LFS2_BLOCK_NULL}; - lfs2_size_t tortoise_i = 1; - lfs2_size_t tortoise_period = 1; + struct lfs2_tortoise_t tortoise = { + .pair = {LFS2_BLOCK_NULL, LFS2_BLOCK_NULL}, + .i = 1, + .period = 1, + }; + int err = LFS2_ERR_OK; while (!lfs2_pair_isnull(dir.tail)) { - // detect cycles with Brent's algorithm - if (lfs2_pair_issync(dir.tail, tortoise)) { - LFS2_WARN("Cycle detected in tail list"); + err = lfs2_tortoise_detectcycles(&dir, &tortoise); + if (err < 0) { return LFS2_ERR_CORRUPT; } - if (tortoise_i == tortoise_period) { - tortoise[0] = dir.tail[0]; - tortoise[1] = dir.tail[1]; - tortoise_i = 0; - tortoise_period *= 2; - } - tortoise_i += 1; for (int i = 0; i < 2; i++) { int err = cb(data, dir.tail[i]); @@ -4647,22 +4798,17 @@ static int lfs2_fs_pred(lfs2_t *lfs2, // iterate over all directory directory entries pdir->tail[0] = 0; pdir->tail[1] = 1; - lfs2_block_t tortoise[2] = {LFS2_BLOCK_NULL, LFS2_BLOCK_NULL}; - lfs2_size_t tortoise_i = 1; - lfs2_size_t tortoise_period = 1; + struct lfs2_tortoise_t tortoise = { + .pair = {LFS2_BLOCK_NULL, LFS2_BLOCK_NULL}, + .i = 1, + .period = 1, + }; + int err = LFS2_ERR_OK; while (!lfs2_pair_isnull(pdir->tail)) { - // detect cycles with Brent's algorithm - if (lfs2_pair_issync(pdir->tail, tortoise)) { - LFS2_WARN("Cycle detected in tail list"); + err = lfs2_tortoise_detectcycles(pdir, &tortoise); + if (err < 0) { return LFS2_ERR_CORRUPT; } - if (tortoise_i == tortoise_period) { - tortoise[0] = pdir->tail[0]; - tortoise[1] = pdir->tail[1]; - tortoise_i = 0; - tortoise_period *= 2; - } - tortoise_i += 1; if (lfs2_pair_cmp(pdir->tail, pair) == 0) { return 0; @@ -4712,22 +4858,17 @@ static lfs2_stag_t lfs2_fs_parent(lfs2_t *lfs2, const lfs2_block_t pair[2], // use fetchmatch with callback to find pairs parent->tail[0] = 0; parent->tail[1] = 1; - lfs2_block_t tortoise[2] = {LFS2_BLOCK_NULL, LFS2_BLOCK_NULL}; - lfs2_size_t tortoise_i = 1; - lfs2_size_t tortoise_period = 1; + struct lfs2_tortoise_t tortoise = { + .pair = {LFS2_BLOCK_NULL, LFS2_BLOCK_NULL}, + .i = 1, + .period = 1, + }; + int err = LFS2_ERR_OK; while (!lfs2_pair_isnull(parent->tail)) { - // detect cycles with Brent's algorithm - if (lfs2_pair_issync(parent->tail, tortoise)) { - LFS2_WARN("Cycle detected in tail list"); - return LFS2_ERR_CORRUPT; - } - if (tortoise_i == tortoise_period) { - tortoise[0] = parent->tail[0]; - tortoise[1] = parent->tail[1]; - tortoise_i = 0; - tortoise_period *= 2; + err = lfs2_tortoise_detectcycles(parent, &tortoise); + if (err < 0) { + return err; } - tortoise_i += 1; lfs2_stag_t tag = lfs2_dir_fetchmatch(lfs2, parent, parent->tail, LFS2_MKTAG(0x7ff, 0, 0x3ff), @@ -4999,7 +5140,7 @@ static int lfs2_fs_forceconsistency(lfs2_t *lfs2) { #endif #ifndef LFS2_READONLY -static int lfs2_fs_rawmkconsistent(lfs2_t *lfs2) { +static int lfs2_fs_mkconsistent_(lfs2_t *lfs2) { // lfs2_fs_forceconsistency does most of the work here int err = lfs2_fs_forceconsistency(lfs2); if (err) { @@ -5035,9 +5176,9 @@ static int lfs2_fs_size_count(void *p, lfs2_block_t block) { return 0; } -static lfs2_ssize_t lfs2_fs_rawsize(lfs2_t *lfs2) { +static lfs2_ssize_t lfs2_fs_size_(lfs2_t *lfs2) { lfs2_size_t size = 0; - int err = lfs2_fs_rawtraverse(lfs2, lfs2_fs_size_count, &size, false); + int err = lfs2_fs_traverse_(lfs2, lfs2_fs_size_count, &size, false); if (err) { return err; } @@ -5045,41 +5186,118 @@ static lfs2_ssize_t lfs2_fs_rawsize(lfs2_t *lfs2) { return size; } +// explicit garbage collection #ifndef LFS2_READONLY -static int lfs2_fs_rawgrow(lfs2_t *lfs2, lfs2_size_t block_count) { - // shrinking is not supported - LFS2_ASSERT(block_count >= lfs2->block_count); +static int lfs2_fs_gc_(lfs2_t *lfs2) { + // force consistency, even if we're not necessarily going to write, + // because this function is supposed to take care of janitorial work + // isn't it? + int err = lfs2_fs_forceconsistency(lfs2); + if (err) { + return err; + } - if (block_count > lfs2->block_count) { - lfs2->block_count = block_count; + // try to compact metadata pairs, note we can't really accomplish + // anything if compact_thresh doesn't at least leave a prog_size + // available + if (lfs2->cfg->compact_thresh + < lfs2->cfg->block_size - lfs2->cfg->prog_size) { + // iterate over all mdirs + lfs2_mdir_t mdir = {.tail = {0, 1}}; + while (!lfs2_pair_isnull(mdir.tail)) { + err = lfs2_dir_fetch(lfs2, &mdir, mdir.tail); + if (err) { + return err; + } - // fetch the root - lfs2_mdir_t root; - int err = lfs2_dir_fetch(lfs2, &root, lfs2->root); + // not erased? exceeds our compaction threshold? + if (!mdir.erased || ((lfs2->cfg->compact_thresh == 0) + ? mdir.off > lfs2->cfg->block_size - lfs2->cfg->block_size/8 + : mdir.off > lfs2->cfg->compact_thresh)) { + // the easiest way to trigger a compaction is to mark + // the mdir as unerased and add an empty commit + mdir.erased = false; + err = lfs2_dir_commit(lfs2, &mdir, NULL, 0); + if (err) { + return err; + } + } + } + } + + // try to populate the lookahead buffer, unless it's already full + if (lfs2->lookahead.size < lfs2_min( + 8 * lfs2->cfg->lookahead_size, + lfs2->block_count)) { + err = lfs2_alloc_scan(lfs2); if (err) { return err; } + } - // update the superblock - lfs2_superblock_t superblock; - lfs2_stag_t tag = lfs2_dir_get(lfs2, &root, LFS2_MKTAG(0x7ff, 0x3ff, 0), - LFS2_MKTAG(LFS2_TYPE_INLINESTRUCT, 0, sizeof(superblock)), - &superblock); - if (tag < 0) { - return tag; - } - lfs2_superblock_fromle32(&superblock); + return 0; +} +#endif - superblock.block_count = lfs2->block_count; +#ifndef LFS2_READONLY +#ifdef LFS2_SHRINKNONRELOCATING +static int lfs2_shrink_checkblock(void *data, lfs2_block_t block) { + lfs2_size_t threshold = *((lfs2_size_t*)data); + if (block >= threshold) { + return LFS2_ERR_NOTEMPTY; + } + return 0; +} +#endif - lfs2_superblock_tole32(&superblock); - err = lfs2_dir_commit(lfs2, &root, LFS2_MKATTRS( - {tag, &superblock})); +static int lfs2_fs_grow_(lfs2_t *lfs2, lfs2_size_t block_count) { + int err; + + if (block_count == lfs2->block_count) { + return 0; + } + + +#ifndef LFS2_SHRINKNONRELOCATING + // shrinking is not supported + LFS2_ASSERT(block_count >= lfs2->block_count); +#endif +#ifdef LFS2_SHRINKNONRELOCATING + if (block_count < lfs2->block_count) { + err = lfs2_fs_traverse_(lfs2, lfs2_shrink_checkblock, &block_count, true); if (err) { return err; } } +#endif + + lfs2->block_count = block_count; + // fetch the root + lfs2_mdir_t root; + err = lfs2_dir_fetch(lfs2, &root, lfs2->root); + if (err) { + return err; + } + + // update the superblock + lfs2_superblock_t superblock; + lfs2_stag_t tag = lfs2_dir_get(lfs2, &root, LFS2_MKTAG(0x7ff, 0x3ff, 0), + LFS2_MKTAG(LFS2_TYPE_INLINESTRUCT, 0, sizeof(superblock)), + &superblock); + if (tag < 0) { + return tag; + } + lfs2_superblock_fromle32(&superblock); + + superblock.block_count = lfs2->block_count; + + lfs2_superblock_tole32(&superblock); + err = lfs2_dir_commit(lfs2, &root, LFS2_MKATTRS( + {tag, &superblock})); + if (err) { + return err; + } return 0; } #endif @@ -5451,10 +5669,10 @@ static int lfs21_mount(lfs2_t *lfs2, struct lfs21 *lfs21, lfs2->lfs21->root[1] = LFS2_BLOCK_NULL; // setup free lookahead - lfs2->free.off = 0; - lfs2->free.size = 0; - lfs2->free.i = 0; - lfs2_alloc_ack(lfs2); + lfs2->lookahead.start = 0; + lfs2->lookahead.size = 0; + lfs2->lookahead.next = 0; + lfs2_alloc_ckpoint(lfs2); // load superblock lfs21_dir_t dir; @@ -5505,7 +5723,7 @@ static int lfs21_unmount(lfs2_t *lfs2) { } /// v1 migration /// -static int lfs2_rawmigrate(lfs2_t *lfs2, const struct lfs2_config *cfg) { +static int lfs2_migrate_(lfs2_t *lfs2, const struct lfs2_config *cfg) { struct lfs21 lfs21; // Indeterminate filesystem size not allowed for migration. @@ -5759,7 +5977,7 @@ int lfs2_format(lfs2_t *lfs2, const struct lfs2_config *cfg) { ".read=%p, .prog=%p, .erase=%p, .sync=%p, " ".read_size=%"PRIu32", .prog_size=%"PRIu32", " ".block_size=%"PRIu32", .block_count=%"PRIu32", " - ".block_cycles=%"PRIu32", .cache_size=%"PRIu32", " + ".block_cycles=%"PRId32", .cache_size=%"PRIu32", " ".lookahead_size=%"PRIu32", .read_buffer=%p, " ".prog_buffer=%p, .lookahead_buffer=%p, " ".name_max=%"PRIu32", .file_max=%"PRIu32", " @@ -5772,7 +5990,7 @@ int lfs2_format(lfs2_t *lfs2, const struct lfs2_config *cfg) { cfg->read_buffer, cfg->prog_buffer, cfg->lookahead_buffer, cfg->name_max, cfg->file_max, cfg->attr_max); - err = lfs2_rawformat(lfs2, cfg); + err = lfs2_format_(lfs2, cfg); LFS2_TRACE("lfs2_format -> %d", err); LFS2_UNLOCK(cfg); @@ -5789,7 +6007,7 @@ int lfs2_mount(lfs2_t *lfs2, const struct lfs2_config *cfg) { ".read=%p, .prog=%p, .erase=%p, .sync=%p, " ".read_size=%"PRIu32", .prog_size=%"PRIu32", " ".block_size=%"PRIu32", .block_count=%"PRIu32", " - ".block_cycles=%"PRIu32", .cache_size=%"PRIu32", " + ".block_cycles=%"PRId32", .cache_size=%"PRIu32", " ".lookahead_size=%"PRIu32", .read_buffer=%p, " ".prog_buffer=%p, .lookahead_buffer=%p, " ".name_max=%"PRIu32", .file_max=%"PRIu32", " @@ -5802,7 +6020,7 @@ int lfs2_mount(lfs2_t *lfs2, const struct lfs2_config *cfg) { cfg->read_buffer, cfg->prog_buffer, cfg->lookahead_buffer, cfg->name_max, cfg->file_max, cfg->attr_max); - err = lfs2_rawmount(lfs2, cfg); + err = lfs2_mount_(lfs2, cfg); LFS2_TRACE("lfs2_mount -> %d", err); LFS2_UNLOCK(cfg); @@ -5816,7 +6034,7 @@ int lfs2_unmount(lfs2_t *lfs2) { } LFS2_TRACE("lfs2_unmount(%p)", (void*)lfs2); - err = lfs2_rawunmount(lfs2); + err = lfs2_unmount_(lfs2); LFS2_TRACE("lfs2_unmount -> %d", err); LFS2_UNLOCK(lfs2->cfg); @@ -5831,7 +6049,7 @@ int lfs2_remove(lfs2_t *lfs2, const char *path) { } LFS2_TRACE("lfs2_remove(%p, \"%s\")", (void*)lfs2, path); - err = lfs2_rawremove(lfs2, path); + err = lfs2_remove_(lfs2, path); LFS2_TRACE("lfs2_remove -> %d", err); LFS2_UNLOCK(lfs2->cfg); @@ -5847,7 +6065,7 @@ int lfs2_rename(lfs2_t *lfs2, const char *oldpath, const char *newpath) { } LFS2_TRACE("lfs2_rename(%p, \"%s\", \"%s\")", (void*)lfs2, oldpath, newpath); - err = lfs2_rawrename(lfs2, oldpath, newpath); + err = lfs2_rename_(lfs2, oldpath, newpath); LFS2_TRACE("lfs2_rename -> %d", err); LFS2_UNLOCK(lfs2->cfg); @@ -5862,7 +6080,7 @@ int lfs2_stat(lfs2_t *lfs2, const char *path, struct lfs2_info *info) { } LFS2_TRACE("lfs2_stat(%p, \"%s\", %p)", (void*)lfs2, path, (void*)info); - err = lfs2_rawstat(lfs2, path, info); + err = lfs2_stat_(lfs2, path, info); LFS2_TRACE("lfs2_stat -> %d", err); LFS2_UNLOCK(lfs2->cfg); @@ -5878,7 +6096,7 @@ lfs2_ssize_t lfs2_getattr(lfs2_t *lfs2, const char *path, LFS2_TRACE("lfs2_getattr(%p, \"%s\", %"PRIu8", %p, %"PRIu32")", (void*)lfs2, path, type, buffer, size); - lfs2_ssize_t res = lfs2_rawgetattr(lfs2, path, type, buffer, size); + lfs2_ssize_t res = lfs2_getattr_(lfs2, path, type, buffer, size); LFS2_TRACE("lfs2_getattr -> %"PRId32, res); LFS2_UNLOCK(lfs2->cfg); @@ -5895,7 +6113,7 @@ int lfs2_setattr(lfs2_t *lfs2, const char *path, LFS2_TRACE("lfs2_setattr(%p, \"%s\", %"PRIu8", %p, %"PRIu32")", (void*)lfs2, path, type, buffer, size); - err = lfs2_rawsetattr(lfs2, path, type, buffer, size); + err = lfs2_setattr_(lfs2, path, type, buffer, size); LFS2_TRACE("lfs2_setattr -> %d", err); LFS2_UNLOCK(lfs2->cfg); @@ -5911,7 +6129,7 @@ int lfs2_removeattr(lfs2_t *lfs2, const char *path, uint8_t type) { } LFS2_TRACE("lfs2_removeattr(%p, \"%s\", %"PRIu8")", (void*)lfs2, path, type); - err = lfs2_rawremoveattr(lfs2, path, type); + err = lfs2_removeattr_(lfs2, path, type); LFS2_TRACE("lfs2_removeattr -> %d", err); LFS2_UNLOCK(lfs2->cfg); @@ -5926,10 +6144,10 @@ int lfs2_file_open(lfs2_t *lfs2, lfs2_file_t *file, const char *path, int flags) return err; } LFS2_TRACE("lfs2_file_open(%p, %p, \"%s\", %x)", - (void*)lfs2, (void*)file, path, flags); + (void*)lfs2, (void*)file, path, (unsigned)flags); LFS2_ASSERT(!lfs2_mlist_isopen(lfs2->mlist, (struct lfs2_mlist*)file)); - err = lfs2_file_rawopen(lfs2, file, path, flags); + err = lfs2_file_open_(lfs2, file, path, flags); LFS2_TRACE("lfs2_file_open -> %d", err); LFS2_UNLOCK(lfs2->cfg); @@ -5946,11 +6164,11 @@ int lfs2_file_opencfg(lfs2_t *lfs2, lfs2_file_t *file, } LFS2_TRACE("lfs2_file_opencfg(%p, %p, \"%s\", %x, %p {" ".buffer=%p, .attrs=%p, .attr_count=%"PRIu32"})", - (void*)lfs2, (void*)file, path, flags, + (void*)lfs2, (void*)file, path, (unsigned)flags, (void*)cfg, cfg->buffer, (void*)cfg->attrs, cfg->attr_count); LFS2_ASSERT(!lfs2_mlist_isopen(lfs2->mlist, (struct lfs2_mlist*)file)); - err = lfs2_file_rawopencfg(lfs2, file, path, flags, cfg); + err = lfs2_file_opencfg_(lfs2, file, path, flags, cfg); LFS2_TRACE("lfs2_file_opencfg -> %d", err); LFS2_UNLOCK(lfs2->cfg); @@ -5965,7 +6183,7 @@ int lfs2_file_close(lfs2_t *lfs2, lfs2_file_t *file) { LFS2_TRACE("lfs2_file_close(%p, %p)", (void*)lfs2, (void*)file); LFS2_ASSERT(lfs2_mlist_isopen(lfs2->mlist, (struct lfs2_mlist*)file)); - err = lfs2_file_rawclose(lfs2, file); + err = lfs2_file_close_(lfs2, file); LFS2_TRACE("lfs2_file_close -> %d", err); LFS2_UNLOCK(lfs2->cfg); @@ -5981,7 +6199,7 @@ int lfs2_file_sync(lfs2_t *lfs2, lfs2_file_t *file) { LFS2_TRACE("lfs2_file_sync(%p, %p)", (void*)lfs2, (void*)file); LFS2_ASSERT(lfs2_mlist_isopen(lfs2->mlist, (struct lfs2_mlist*)file)); - err = lfs2_file_rawsync(lfs2, file); + err = lfs2_file_sync_(lfs2, file); LFS2_TRACE("lfs2_file_sync -> %d", err); LFS2_UNLOCK(lfs2->cfg); @@ -5999,7 +6217,7 @@ lfs2_ssize_t lfs2_file_read(lfs2_t *lfs2, lfs2_file_t *file, (void*)lfs2, (void*)file, buffer, size); LFS2_ASSERT(lfs2_mlist_isopen(lfs2->mlist, (struct lfs2_mlist*)file)); - lfs2_ssize_t res = lfs2_file_rawread(lfs2, file, buffer, size); + lfs2_ssize_t res = lfs2_file_read_(lfs2, file, buffer, size); LFS2_TRACE("lfs2_file_read -> %"PRId32, res); LFS2_UNLOCK(lfs2->cfg); @@ -6017,7 +6235,7 @@ lfs2_ssize_t lfs2_file_write(lfs2_t *lfs2, lfs2_file_t *file, (void*)lfs2, (void*)file, buffer, size); LFS2_ASSERT(lfs2_mlist_isopen(lfs2->mlist, (struct lfs2_mlist*)file)); - lfs2_ssize_t res = lfs2_file_rawwrite(lfs2, file, buffer, size); + lfs2_ssize_t res = lfs2_file_write_(lfs2, file, buffer, size); LFS2_TRACE("lfs2_file_write -> %"PRId32, res); LFS2_UNLOCK(lfs2->cfg); @@ -6035,7 +6253,7 @@ lfs2_soff_t lfs2_file_seek(lfs2_t *lfs2, lfs2_file_t *file, (void*)lfs2, (void*)file, off, whence); LFS2_ASSERT(lfs2_mlist_isopen(lfs2->mlist, (struct lfs2_mlist*)file)); - lfs2_soff_t res = lfs2_file_rawseek(lfs2, file, off, whence); + lfs2_soff_t res = lfs2_file_seek_(lfs2, file, off, whence); LFS2_TRACE("lfs2_file_seek -> %"PRId32, res); LFS2_UNLOCK(lfs2->cfg); @@ -6052,7 +6270,7 @@ int lfs2_file_truncate(lfs2_t *lfs2, lfs2_file_t *file, lfs2_off_t size) { (void*)lfs2, (void*)file, size); LFS2_ASSERT(lfs2_mlist_isopen(lfs2->mlist, (struct lfs2_mlist*)file)); - err = lfs2_file_rawtruncate(lfs2, file, size); + err = lfs2_file_truncate_(lfs2, file, size); LFS2_TRACE("lfs2_file_truncate -> %d", err); LFS2_UNLOCK(lfs2->cfg); @@ -6068,7 +6286,7 @@ lfs2_soff_t lfs2_file_tell(lfs2_t *lfs2, lfs2_file_t *file) { LFS2_TRACE("lfs2_file_tell(%p, %p)", (void*)lfs2, (void*)file); LFS2_ASSERT(lfs2_mlist_isopen(lfs2->mlist, (struct lfs2_mlist*)file)); - lfs2_soff_t res = lfs2_file_rawtell(lfs2, file); + lfs2_soff_t res = lfs2_file_tell_(lfs2, file); LFS2_TRACE("lfs2_file_tell -> %"PRId32, res); LFS2_UNLOCK(lfs2->cfg); @@ -6082,7 +6300,7 @@ int lfs2_file_rewind(lfs2_t *lfs2, lfs2_file_t *file) { } LFS2_TRACE("lfs2_file_rewind(%p, %p)", (void*)lfs2, (void*)file); - err = lfs2_file_rawrewind(lfs2, file); + err = lfs2_file_rewind_(lfs2, file); LFS2_TRACE("lfs2_file_rewind -> %d", err); LFS2_UNLOCK(lfs2->cfg); @@ -6097,9 +6315,9 @@ lfs2_soff_t lfs2_file_size(lfs2_t *lfs2, lfs2_file_t *file) { LFS2_TRACE("lfs2_file_size(%p, %p)", (void*)lfs2, (void*)file); LFS2_ASSERT(lfs2_mlist_isopen(lfs2->mlist, (struct lfs2_mlist*)file)); - lfs2_soff_t res = lfs2_file_rawsize(lfs2, file); + lfs2_soff_t res = lfs2_file_size_(lfs2, file); - LFS2_TRACE("lfs2_file_size -> %"PRId32, res); + LFS2_TRACE("lfs2_file_size -> %"PRIu32, res); LFS2_UNLOCK(lfs2->cfg); return res; } @@ -6112,7 +6330,7 @@ int lfs2_mkdir(lfs2_t *lfs2, const char *path) { } LFS2_TRACE("lfs2_mkdir(%p, \"%s\")", (void*)lfs2, path); - err = lfs2_rawmkdir(lfs2, path); + err = lfs2_mkdir_(lfs2, path); LFS2_TRACE("lfs2_mkdir -> %d", err); LFS2_UNLOCK(lfs2->cfg); @@ -6128,7 +6346,7 @@ int lfs2_dir_open(lfs2_t *lfs2, lfs2_dir_t *dir, const char *path) { LFS2_TRACE("lfs2_dir_open(%p, %p, \"%s\")", (void*)lfs2, (void*)dir, path); LFS2_ASSERT(!lfs2_mlist_isopen(lfs2->mlist, (struct lfs2_mlist*)dir)); - err = lfs2_dir_rawopen(lfs2, dir, path); + err = lfs2_dir_open_(lfs2, dir, path); LFS2_TRACE("lfs2_dir_open -> %d", err); LFS2_UNLOCK(lfs2->cfg); @@ -6142,7 +6360,7 @@ int lfs2_dir_close(lfs2_t *lfs2, lfs2_dir_t *dir) { } LFS2_TRACE("lfs2_dir_close(%p, %p)", (void*)lfs2, (void*)dir); - err = lfs2_dir_rawclose(lfs2, dir); + err = lfs2_dir_close_(lfs2, dir); LFS2_TRACE("lfs2_dir_close -> %d", err); LFS2_UNLOCK(lfs2->cfg); @@ -6157,7 +6375,7 @@ int lfs2_dir_read(lfs2_t *lfs2, lfs2_dir_t *dir, struct lfs2_info *info) { LFS2_TRACE("lfs2_dir_read(%p, %p, %p)", (void*)lfs2, (void*)dir, (void*)info); - err = lfs2_dir_rawread(lfs2, dir, info); + err = lfs2_dir_read_(lfs2, dir, info); LFS2_TRACE("lfs2_dir_read -> %d", err); LFS2_UNLOCK(lfs2->cfg); @@ -6172,7 +6390,7 @@ int lfs2_dir_seek(lfs2_t *lfs2, lfs2_dir_t *dir, lfs2_off_t off) { LFS2_TRACE("lfs2_dir_seek(%p, %p, %"PRIu32")", (void*)lfs2, (void*)dir, off); - err = lfs2_dir_rawseek(lfs2, dir, off); + err = lfs2_dir_seek_(lfs2, dir, off); LFS2_TRACE("lfs2_dir_seek -> %d", err); LFS2_UNLOCK(lfs2->cfg); @@ -6186,7 +6404,7 @@ lfs2_soff_t lfs2_dir_tell(lfs2_t *lfs2, lfs2_dir_t *dir) { } LFS2_TRACE("lfs2_dir_tell(%p, %p)", (void*)lfs2, (void*)dir); - lfs2_soff_t res = lfs2_dir_rawtell(lfs2, dir); + lfs2_soff_t res = lfs2_dir_tell_(lfs2, dir); LFS2_TRACE("lfs2_dir_tell -> %"PRId32, res); LFS2_UNLOCK(lfs2->cfg); @@ -6200,7 +6418,7 @@ int lfs2_dir_rewind(lfs2_t *lfs2, lfs2_dir_t *dir) { } LFS2_TRACE("lfs2_dir_rewind(%p, %p)", (void*)lfs2, (void*)dir); - err = lfs2_dir_rawrewind(lfs2, dir); + err = lfs2_dir_rewind_(lfs2, dir); LFS2_TRACE("lfs2_dir_rewind -> %d", err); LFS2_UNLOCK(lfs2->cfg); @@ -6214,7 +6432,7 @@ int lfs2_fs_stat(lfs2_t *lfs2, struct lfs2_fsinfo *fsinfo) { } LFS2_TRACE("lfs2_fs_stat(%p, %p)", (void*)lfs2, (void*)fsinfo); - err = lfs2_fs_rawstat(lfs2, fsinfo); + err = lfs2_fs_stat_(lfs2, fsinfo); LFS2_TRACE("lfs2_fs_stat -> %d", err); LFS2_UNLOCK(lfs2->cfg); @@ -6228,7 +6446,7 @@ lfs2_ssize_t lfs2_fs_size(lfs2_t *lfs2) { } LFS2_TRACE("lfs2_fs_size(%p)", (void*)lfs2); - lfs2_ssize_t res = lfs2_fs_rawsize(lfs2); + lfs2_ssize_t res = lfs2_fs_size_(lfs2); LFS2_TRACE("lfs2_fs_size -> %"PRId32, res); LFS2_UNLOCK(lfs2->cfg); @@ -6243,7 +6461,7 @@ int lfs2_fs_traverse(lfs2_t *lfs2, int (*cb)(void *, lfs2_block_t), void *data) LFS2_TRACE("lfs2_fs_traverse(%p, %p, %p)", (void*)lfs2, (void*)(uintptr_t)cb, data); - err = lfs2_fs_rawtraverse(lfs2, cb, data, true); + err = lfs2_fs_traverse_(lfs2, cb, data, true); LFS2_TRACE("lfs2_fs_traverse -> %d", err); LFS2_UNLOCK(lfs2->cfg); @@ -6251,32 +6469,32 @@ int lfs2_fs_traverse(lfs2_t *lfs2, int (*cb)(void *, lfs2_block_t), void *data) } #ifndef LFS2_READONLY -int lfs2_fs_gc(lfs2_t *lfs2) { +int lfs2_fs_mkconsistent(lfs2_t *lfs2) { int err = LFS2_LOCK(lfs2->cfg); if (err) { return err; } - LFS2_TRACE("lfs2_fs_gc(%p)", (void*)lfs2); + LFS2_TRACE("lfs2_fs_mkconsistent(%p)", (void*)lfs2); - err = lfs2_fs_rawgc(lfs2); + err = lfs2_fs_mkconsistent_(lfs2); - LFS2_TRACE("lfs2_fs_gc -> %d", err); + LFS2_TRACE("lfs2_fs_mkconsistent -> %d", err); LFS2_UNLOCK(lfs2->cfg); return err; } #endif #ifndef LFS2_READONLY -int lfs2_fs_mkconsistent(lfs2_t *lfs2) { +int lfs2_fs_gc(lfs2_t *lfs2) { int err = LFS2_LOCK(lfs2->cfg); if (err) { return err; } - LFS2_TRACE("lfs2_fs_mkconsistent(%p)", (void*)lfs2); + LFS2_TRACE("lfs2_fs_gc(%p)", (void*)lfs2); - err = lfs2_fs_rawmkconsistent(lfs2); + err = lfs2_fs_gc_(lfs2); - LFS2_TRACE("lfs2_fs_mkconsistent -> %d", err); + LFS2_TRACE("lfs2_fs_gc -> %d", err); LFS2_UNLOCK(lfs2->cfg); return err; } @@ -6290,7 +6508,7 @@ int lfs2_fs_grow(lfs2_t *lfs2, lfs2_size_t block_count) { } LFS2_TRACE("lfs2_fs_grow(%p, %"PRIu32")", (void*)lfs2, block_count); - err = lfs2_fs_rawgrow(lfs2, block_count); + err = lfs2_fs_grow_(lfs2, block_count); LFS2_TRACE("lfs2_fs_grow -> %d", err); LFS2_UNLOCK(lfs2->cfg); @@ -6308,7 +6526,7 @@ int lfs2_migrate(lfs2_t *lfs2, const struct lfs2_config *cfg) { ".read=%p, .prog=%p, .erase=%p, .sync=%p, " ".read_size=%"PRIu32", .prog_size=%"PRIu32", " ".block_size=%"PRIu32", .block_count=%"PRIu32", " - ".block_cycles=%"PRIu32", .cache_size=%"PRIu32", " + ".block_cycles=%"PRId32", .cache_size=%"PRIu32", " ".lookahead_size=%"PRIu32", .read_buffer=%p, " ".prog_buffer=%p, .lookahead_buffer=%p, " ".name_max=%"PRIu32", .file_max=%"PRIu32", " @@ -6321,7 +6539,7 @@ int lfs2_migrate(lfs2_t *lfs2, const struct lfs2_config *cfg) { cfg->read_buffer, cfg->prog_buffer, cfg->lookahead_buffer, cfg->name_max, cfg->file_max, cfg->attr_max); - err = lfs2_rawmigrate(lfs2, cfg); + err = lfs2_migrate_(lfs2, cfg); LFS2_TRACE("lfs2_migrate -> %d", err); LFS2_UNLOCK(cfg); diff --git a/lib/littlefs/lfs2.h b/lib/littlefs/lfs2.h index 559ccebac9269..aee0619e9326a 100644 --- a/lib/littlefs/lfs2.h +++ b/lib/littlefs/lfs2.h @@ -21,7 +21,7 @@ extern "C" // Software library version // Major (top-nibble), incremented on backwards incompatible changes // Minor (bottom-nibble), incremented on feature additions -#define LFS2_VERSION 0x00020008 +#define LFS2_VERSION 0x0002000b #define LFS2_VERSION_MAJOR (0xffff & (LFS2_VERSION >> 16)) #define LFS2_VERSION_MINOR (0xffff & (LFS2_VERSION >> 0)) @@ -52,16 +52,15 @@ typedef uint32_t lfs2_block_t; #endif // Maximum size of a file in bytes, may be redefined to limit to support other -// drivers. Limited on disk to <= 4294967296. However, above 2147483647 the -// functions lfs2_file_seek, lfs2_file_size, and lfs2_file_tell will return -// incorrect values due to using signed integers. Stored in superblock and -// must be respected by other littlefs drivers. +// drivers. Limited on disk to <= 2147483647. Stored in superblock and must be +// respected by other littlefs drivers. #ifndef LFS2_FILE_MAX #define LFS2_FILE_MAX 2147483647 #endif // Maximum size of custom attributes in bytes, may be redefined, but there is -// no real benefit to using a smaller LFS2_ATTR_MAX. Limited to <= 1022. +// no real benefit to using a smaller LFS2_ATTR_MAX. Limited to <= 1022. Stored +// in superblock and must be respected by other littlefs drivers. #ifndef LFS2_ATTR_MAX #define LFS2_ATTR_MAX 1022 #endif @@ -205,7 +204,8 @@ struct lfs2_config { // program sizes. lfs2_size_t block_size; - // Number of erasable blocks on the device. + // Number of erasable blocks on the device. Defaults to block_count stored + // on disk when zero. lfs2_size_t block_count; // Number of erase cycles before littlefs evicts metadata logs and moves @@ -226,9 +226,20 @@ struct lfs2_config { // Size of the lookahead buffer in bytes. A larger lookahead buffer // increases the number of blocks found during an allocation pass. The // lookahead buffer is stored as a compact bitmap, so each byte of RAM - // can track 8 blocks. Must be a multiple of 8. + // can track 8 blocks. lfs2_size_t lookahead_size; + // Threshold for metadata compaction during lfs2_fs_gc in bytes. Metadata + // pairs that exceed this threshold will be compacted during lfs2_fs_gc. + // Defaults to ~88% block_size when zero, though the default may change + // in the future. + // + // Note this only affects lfs2_fs_gc. Normal compactions still only occur + // when full. + // + // Set to -1 to disable metadata compaction during lfs2_fs_gc. + lfs2_size_t compact_thresh; + // Optional statically allocated read buffer. Must be cache_size. // By default lfs2_malloc is used to allocate this buffer. void *read_buffer; @@ -237,25 +248,24 @@ struct lfs2_config { // By default lfs2_malloc is used to allocate this buffer. void *prog_buffer; - // Optional statically allocated lookahead buffer. Must be lookahead_size - // and aligned to a 32-bit boundary. By default lfs2_malloc is used to - // allocate this buffer. + // Optional statically allocated lookahead buffer. Must be lookahead_size. + // By default lfs2_malloc is used to allocate this buffer. void *lookahead_buffer; // Optional upper limit on length of file names in bytes. No downside for // larger names except the size of the info struct which is controlled by - // the LFS2_NAME_MAX define. Defaults to LFS2_NAME_MAX when zero. Stored in - // superblock and must be respected by other littlefs drivers. + // the LFS2_NAME_MAX define. Defaults to LFS2_NAME_MAX or name_max stored on + // disk when zero. lfs2_size_t name_max; // Optional upper limit on files in bytes. No downside for larger files - // but must be <= LFS2_FILE_MAX. Defaults to LFS2_FILE_MAX when zero. Stored - // in superblock and must be respected by other littlefs drivers. + // but must be <= LFS2_FILE_MAX. Defaults to LFS2_FILE_MAX or file_max stored + // on disk when zero. lfs2_size_t file_max; // Optional upper limit on custom attributes in bytes. No downside for // larger attributes size but must be <= LFS2_ATTR_MAX. Defaults to - // LFS2_ATTR_MAX when zero. + // LFS2_ATTR_MAX or attr_max stored on disk when zero. lfs2_size_t attr_max; // Optional upper limit on total space given to metadata pairs in bytes. On @@ -264,6 +274,15 @@ struct lfs2_config { // Defaults to block_size when zero. lfs2_size_t metadata_max; + // Optional upper limit on inlined files in bytes. Inlined files live in + // metadata and decrease storage requirements, but may be limited to + // improve metadata-related performance. Must be <= cache_size, <= + // attr_max, and <= block_size/8. Defaults to the largest possible + // inline_max when zero. + // + // Set to -1 to disable inlined files. + lfs2_size_t inline_max; + #ifdef LFS2_MULTIVERSION // On-disk version to use when writing in the form of 16-bit major version // + 16-bit minor version. This limiting metadata to what is supported by @@ -430,19 +449,20 @@ typedef struct lfs2 { lfs2_gstate_t gdisk; lfs2_gstate_t gdelta; - struct lfs2_free { - lfs2_block_t off; + struct lfs2_lookahead { + lfs2_block_t start; lfs2_block_t size; - lfs2_block_t i; - lfs2_block_t ack; - uint32_t *buffer; - } free; + lfs2_block_t next; + lfs2_block_t ckpoint; + uint8_t *buffer; + } lookahead; const struct lfs2_config *cfg; lfs2_size_t block_count; lfs2_size_t name_max; lfs2_size_t file_max; lfs2_size_t attr_max; + lfs2_size_t inline_max; #ifdef LFS2_MIGRATE struct lfs21 *lfs21; @@ -712,18 +732,6 @@ lfs2_ssize_t lfs2_fs_size(lfs2_t *lfs2); // Returns a negative error code on failure. int lfs2_fs_traverse(lfs2_t *lfs2, int (*cb)(void*, lfs2_block_t), void *data); -// Attempt to proactively find free blocks -// -// Calling this function is not required, but may allowing the offloading of -// the expensive block allocation scan to a less time-critical code path. -// -// Note: littlefs currently does not persist any found free blocks to disk. -// This may change in the future. -// -// Returns a negative error code on failure. Finding no free blocks is -// not an error. -int lfs2_fs_gc(lfs2_t *lfs2); - #ifndef LFS2_READONLY // Attempt to make the filesystem consistent and ready for writing // @@ -736,11 +744,33 @@ int lfs2_fs_gc(lfs2_t *lfs2); int lfs2_fs_mkconsistent(lfs2_t *lfs2); #endif +#ifndef LFS2_READONLY +// Attempt any janitorial work +// +// This currently: +// 1. Calls mkconsistent if not already consistent +// 2. Compacts metadata > compact_thresh +// 3. Populates the block allocator +// +// Though additional janitorial work may be added in the future. +// +// Calling this function is not required, but may allow the offloading of +// expensive janitorial work to a less time-critical code path. +// +// Returns a negative error code on failure. Accomplishing nothing is not +// an error. +int lfs2_fs_gc(lfs2_t *lfs2); +#endif + #ifndef LFS2_READONLY // Grows the filesystem to a new size, updating the superblock with the new // block count. // -// Note: This is irreversible. +// If LFS2_SHRINKNONRELOCATING is defined, this function will also accept +// block_counts smaller than the current configuration, after checking +// that none of the blocks that are being removed are in use. +// Note that littlefs's pseudorandom block allocation means that +// this is very unlikely to work in the general case. // // Returns a negative error code on failure. int lfs2_fs_grow(lfs2_t *lfs2, lfs2_size_t block_count); diff --git a/lib/littlefs/lfs2_util.c b/lib/littlefs/lfs2_util.c index c9850e78869c3..4fe7e5340ce77 100644 --- a/lib/littlefs/lfs2_util.c +++ b/lib/littlefs/lfs2_util.c @@ -11,6 +11,8 @@ #ifndef LFS2_CONFIG +// If user provides their own CRC impl we don't need this +#ifndef LFS2_CRC // Software CRC implementation with small lookup table uint32_t lfs2_crc(uint32_t crc, const void *buffer, size_t size) { static const uint32_t rtable[16] = { @@ -29,6 +31,7 @@ uint32_t lfs2_crc(uint32_t crc, const void *buffer, size_t size) { return crc; } +#endif #endif diff --git a/lib/littlefs/lfs2_util.h b/lib/littlefs/lfs2_util.h index dd2cbcc106d84..12c82a630b079 100644 --- a/lib/littlefs/lfs2_util.h +++ b/lib/littlefs/lfs2_util.h @@ -8,6 +8,9 @@ #ifndef LFS2_UTIL_H #define LFS2_UTIL_H +#define LFS2_STRINGIZE(x) LFS2_STRINGIZE2(x) +#define LFS2_STRINGIZE2(x) #x + // Users can override lfs2_util.h with their own configuration by defining // LFS2_CONFIG as a header file to include (-DLFS2_CONFIG=lfs2_config.h). // @@ -15,11 +18,26 @@ // provided by the config file. To start, I would suggest copying lfs2_util.h // and modifying as needed. #ifdef LFS2_CONFIG -#define LFS2_STRINGIZE(x) LFS2_STRINGIZE2(x) -#define LFS2_STRINGIZE2(x) #x #include LFS2_STRINGIZE(LFS2_CONFIG) #else +// Alternatively, users can provide a header file which defines +// macros and other things consumed by littlefs. +// +// For example, provide my_defines.h, which contains +// something like: +// +// #include +// extern void *my_malloc(size_t sz); +// #define LFS2_MALLOC(sz) my_malloc(sz) +// +// And build littlefs with the header by defining LFS2_DEFINES. +// (-DLFS2_DEFINES=my_defines.h) + +#ifdef LFS2_DEFINES +#include LFS2_STRINGIZE(LFS2_DEFINES) +#endif + // System includes #include #include @@ -177,10 +195,10 @@ static inline uint32_t lfs2_fromle32(uint32_t a) { (defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)) return __builtin_bswap32(a); #else - return (((uint8_t*)&a)[0] << 0) | - (((uint8_t*)&a)[1] << 8) | - (((uint8_t*)&a)[2] << 16) | - (((uint8_t*)&a)[3] << 24); + return ((uint32_t)((uint8_t*)&a)[0] << 0) | + ((uint32_t)((uint8_t*)&a)[1] << 8) | + ((uint32_t)((uint8_t*)&a)[2] << 16) | + ((uint32_t)((uint8_t*)&a)[3] << 24); #endif } @@ -200,10 +218,10 @@ static inline uint32_t lfs2_frombe32(uint32_t a) { (defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) return a; #else - return (((uint8_t*)&a)[0] << 24) | - (((uint8_t*)&a)[1] << 16) | - (((uint8_t*)&a)[2] << 8) | - (((uint8_t*)&a)[3] << 0); + return ((uint32_t)((uint8_t*)&a)[0] << 24) | + ((uint32_t)((uint8_t*)&a)[1] << 16) | + ((uint32_t)((uint8_t*)&a)[2] << 8) | + ((uint32_t)((uint8_t*)&a)[3] << 0); #endif } @@ -212,12 +230,22 @@ static inline uint32_t lfs2_tobe32(uint32_t a) { } // Calculate CRC-32 with polynomial = 0x04c11db7 +#ifdef LFS2_CRC +static inline uint32_t lfs2_crc(uint32_t crc, const void *buffer, size_t size) { + return LFS2_CRC(crc, buffer, size); +} +#else uint32_t lfs2_crc(uint32_t crc, const void *buffer, size_t size); +#endif // Allocate memory, only used if buffers are not provided to littlefs -// Note, memory must be 64-bit aligned +// +// littlefs current has no alignment requirements, as it only allocates +// byte-level buffers. static inline void *lfs2_malloc(size_t size) { -#ifndef LFS2_NO_MALLOC +#if defined(LFS2_MALLOC) + return LFS2_MALLOC(size); +#elif !defined(LFS2_NO_MALLOC) return malloc(size); #else (void)size; @@ -227,7 +255,9 @@ static inline void *lfs2_malloc(size_t size) { // Deallocate memory, only used if buffers are not provided to littlefs static inline void lfs2_free(void *p) { -#ifndef LFS2_NO_MALLOC +#if defined(LFS2_FREE) + LFS2_FREE(p); +#elif !defined(LFS2_NO_MALLOC) free(p); #else (void)p; diff --git a/lib/micropython-lib b/lib/micropython-lib index 5b496e944ec04..6ae440a8a1442 160000 --- a/lib/micropython-lib +++ b/lib/micropython-lib @@ -1 +1 @@ -Subproject commit 5b496e944ec045177afa1620920a168410b7f60b +Subproject commit 6ae440a8a144233e6e703f6759b7e7a0afaa37a4 diff --git a/locale/circuitpython.pot b/locale/circuitpython.pot index 07623ea94a078..833991ac476e0 100644 --- a/locale/circuitpython.pot +++ b/locale/circuitpython.pot @@ -78,6 +78,11 @@ msgid "" "%d address pins, %d rgb pins and %d tiles indicate a height of %d, not %d" msgstr "" +#: py/emitinlinextensa.c +#, c-format +msgid "%d is not a multiple of %d" +msgstr "" + #: shared-bindings/microcontroller/Pin.c msgid "%q and %q contain duplicate pins" msgstr "" @@ -262,10 +267,6 @@ msgstr "" msgid "%q out of range" msgstr "" -#: py/objmodule.c -msgid "%q renamed %q" -msgstr "" - #: py/objrange.c py/objslice.c shared-bindings/random/__init__.c msgid "%q step cannot be zero" msgstr "" @@ -963,6 +964,14 @@ msgstr "" msgid "ECB only operates on 16 bytes at a time" msgstr "" +#: py/asmxtensa.c +msgid "ERROR: %q %q not word-aligned" +msgstr "" + +#: py/asmxtensa.c +msgid "ERROR: xtensa %q out of range" +msgstr "" + #: ports/espressif/common-hal/busio/SPI.c #: ports/espressif/common-hal/canio/CAN.c #: ports/espressif/common-hal/qspibus/QSPIBus.c @@ -2627,10 +2636,6 @@ msgstr "" msgid "array/bytes required on right side" msgstr "" -#: py/asmxtensa.c -msgid "asm overflow" -msgstr "" - #: py/compile.c msgid "async for/with outside async function" msgstr "" @@ -2844,6 +2849,10 @@ msgstr "" msgid "can't create '%q' instances" msgstr "" +#: py/objtype.c +msgid "can't create instance" +msgstr "" + #: py/compile.c msgid "can't declare nonlocal in outer code" msgstr "" @@ -2942,14 +2951,14 @@ msgstr "" msgid "cannot convert complex type" msgstr "" -#: py/objtype.c -msgid "cannot create instance" -msgstr "" - #: extmod/ulab/code/ndarray.c msgid "cannot delete array elements" msgstr "" +#: py/compile.c +msgid "cannot emit native code for this architecture" +msgstr "" + #: extmod/ulab/code/ndarray.c msgid "cannot reshape array" msgstr "" @@ -3104,7 +3113,7 @@ msgstr "" msgid "div/mod not implemented for uint" msgstr "" -#: extmod/ulab/code/numpy/create.c +#: extmod/ulab/code/numpy/create.c py/objint_longlong.c py/objint_mpz.c msgid "divide by zero" msgstr "" @@ -3480,6 +3489,10 @@ msgstr "" msgid "interval must be in range %s-%s" msgstr "" +#: py/emitinlinerv32.c +msgid "invalid RV32 instruction '%q'" +msgstr "" + #: py/compile.c msgid "invalid arch" msgstr "" @@ -3674,6 +3687,10 @@ msgstr "" msgid "mode must be complete, or reduced" msgstr "" +#: py/runtime.c +msgid "module '%q' has no attribute '%q'" +msgstr "" + #: py/builtinimport.c msgid "module not found" msgstr "" @@ -3722,10 +3739,6 @@ msgstr "" msgid "native code in .mpy unsupported" msgstr "" -#: py/asmthumb.c -msgid "native method too big" -msgstr "" - #: py/emitnative.c msgid "native yield" msgstr "" @@ -3747,7 +3760,7 @@ msgstr "" msgid "negative power with no float support" msgstr "" -#: py/objint_mpz.c py/runtime.c +#: py/objint_longlong.c py/objint_mpz.c py/runtime.c msgid "negative shift count" msgstr "" @@ -4102,6 +4115,10 @@ msgstr "" msgid "real and imaginary parts must be of equal length" msgstr "" +#: extmod/modre.c +msgid "regex too complex" +msgstr "" + #: py/builtinimport.c msgid "relative import" msgstr "" @@ -4111,6 +4128,10 @@ msgstr "" msgid "requested length %d but object has length %d" msgstr "" +#: py/objint_longlong.c py/parsenum.c +msgid "result overflows long long storage" +msgstr "" + #: extmod/ulab/code/ndarray_operators.c msgid "results cannot be cast to specified type" msgstr "" @@ -4375,10 +4396,6 @@ msgstr "" msgid "type takes 1 or 3 arguments" msgstr "" -#: py/objint_longlong.c -msgid "ulonglong too large" -msgstr "" - #: py/parse.c msgid "unexpected indent" msgstr "" @@ -4400,10 +4417,6 @@ msgstr "" msgid "unindent doesn't match any outer indent level" msgstr "" -#: py/emitinlinerv32.c -msgid "unknown RV32 instruction '%q'" -msgstr "" - #: py/objstr.c #, c-format msgid "unknown conversion specifier %c" diff --git a/main.c b/main.c index c9ed036bf865d..ce058cc49c4b5 100644 --- a/main.c +++ b/main.c @@ -1149,7 +1149,7 @@ int __attribute__((used)) main(void) { } else { skip_repl = false; } - } else if (exit_code != 0) { + } else if (exit_code != PYEXEC_NORMAL_EXIT) { break; } @@ -1206,7 +1206,7 @@ size_t gc_get_max_new_split(void) { return port_heap_get_largest_free_size(); } -void NORETURN nlr_jump_fail(void *val) { +void MP_NORETURN nlr_jump_fail(void *val) { reset_into_safe_mode(SAFE_MODE_NLR_JUMP_FAIL); while (true) { } @@ -1217,7 +1217,7 @@ bool vm_is_running(void) { } #ifndef NDEBUG -static void NORETURN __fatal_error(const char *msg) { +static void MP_NORETURN __fatal_error(const char *msg) { #if CIRCUITPY_DEBUG == 0 reset_into_safe_mode(SAFE_MODE_HARD_FAULT); #endif diff --git a/mpy-cross/main.c b/mpy-cross/main.c index 989aec68bb519..add07c3d49b3e 100644 --- a/mpy-cross/main.c +++ b/mpy-cross/main.c @@ -34,13 +34,19 @@ #include "py/persistentcode.h" #include "py/runtime.h" #include "py/gc.h" -#include "py/stackctrl.h" +#include "py/parsenum.h" #include "genhdr/mpversion.h" #ifdef _WIN32 // CIRCUITPY-CHANGE #include "fmode.h" #endif +#if MICROPY_EMIT_NATIVE && MICROPY_EMIT_RV32 +#include "py/asmrv32.h" + +static asm_rv32_backend_options_t rv32_options = { 0 }; +#endif + // Command line options, with their defaults static uint emit_opt = MP_EMIT_OPT_NONE; mp_uint_t mp_verbose_flag = 0; @@ -82,13 +88,20 @@ static int compile_and_save(const char *file, const char *output_file, const cha source_name = qstr_from_str(source_file); } - #if MICROPY_PY___FILE__ + #if MICROPY_MODULE___FILE__ mp_store_global(MP_QSTR___file__, MP_OBJ_NEW_QSTR(source_name)); #endif mp_parse_tree_t parse_tree = mp_parse(lex, MP_PARSE_FILE_INPUT); mp_compiled_module_t cm; cm.context = m_new_obj(mp_module_context_t); + cm.arch_flags = 0; + #if MICROPY_EMIT_NATIVE && MICROPY_EMIT_RV32 + if (mp_dynamic_compiler.native_arch == MP_NATIVE_ARCH_RV32IMC && mp_dynamic_compiler.backend_options != NULL) { + cm.arch_flags = ((asm_rv32_backend_options_t *)mp_dynamic_compiler.backend_options)->allowed_extensions; + } + #endif + mp_compile_to_raw_code(&parse_tree, source_name, false, &cm); if ((output_file != NULL && strcmp(output_file, "-") == 0) || @@ -131,7 +144,10 @@ static int usage(char **argv) { "Target specific options:\n" "-msmall-int-bits=number : set the maximum bits used to encode a small-int\n" "-march= : set architecture for native emitter;\n" - " x86, x64, armv6, armv6m, armv7m, armv7em, armv7emsp, armv7emdp, xtensa, xtensawin, rv32imc, debug\n" + " x86, x64, armv6, armv6m, armv7m, armv7em, armv7emsp,\n" + " armv7emdp, xtensa, xtensawin, rv32imc, rv64imc, host, debug\n" + "-march-flags= : set architecture-specific flags (can be either a dec/hex/bin value or a string)\n" + " supported flags for rv32imc: zba\n" "\n" "Implementation specific options:\n", argv[0] ); @@ -212,9 +228,39 @@ static char *backslash_to_forwardslash(char *path) { return path; } -MP_NOINLINE int main_(int argc, char **argv) { - mp_stack_set_limit(40000 * (sizeof(void *) / 4)); +// This will need to be reworked in case mpy-cross needs to set more bits than +// what its small int representation allows to fit in there. +static bool parse_integer(const char *value, mp_uint_t *integer) { + assert(value && "Attempting to parse a NULL string"); + assert(integer && "Attempting to store into a NULL integer buffer"); + + size_t value_length = strlen(value); + int base = 10; + if (value_length > 2 && value[0] == '0') { + if ((value[1] | 0x20) == 'b') { + base = 2; + } else if ((value[1] | 0x20) == 'x') { + base = 16; + } else { + return false; + } + } + + bool valid = false; + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + mp_obj_t parsed = mp_parse_num_integer(value, value_length, base, NULL); + if (mp_obj_is_small_int(parsed)) { + *integer = MP_OBJ_SMALL_INT_VALUE(parsed); + valid = true; + } + nlr_pop(); + } + return valid; +} + +MP_NOINLINE int main_(int argc, char **argv) { pre_process_options(argc, argv); char *heap = malloc(heap_size); @@ -237,11 +283,13 @@ MP_NOINLINE int main_(int argc, char **argv) { // don't support native emitter unless -march is specified mp_dynamic_compiler.native_arch = MP_NATIVE_ARCH_NONE; mp_dynamic_compiler.nlr_buf_num_regs = 0; + mp_dynamic_compiler.backend_options = NULL; const char *input_file = NULL; const char *output_file = NULL; const char *source_file = NULL; bool option_parsing_active = true; + const char *arch_flags = NULL; // parse main options for (int a = 1; a < argc; a++) { @@ -318,6 +366,9 @@ MP_NOINLINE int main_(int argc, char **argv) { } else if (strcmp(arch, "rv32imc") == 0) { mp_dynamic_compiler.native_arch = MP_NATIVE_ARCH_RV32IMC; mp_dynamic_compiler.nlr_buf_num_regs = MICROPY_NLR_NUM_REGS_RV32I; + } else if (strcmp(arch, "rv64imc") == 0) { + mp_dynamic_compiler.native_arch = MP_NATIVE_ARCH_RV64IMC; + mp_dynamic_compiler.nlr_buf_num_regs = MICROPY_NLR_NUM_REGS_RV64I; } else if (strcmp(arch, "debug") == 0) { mp_dynamic_compiler.native_arch = MP_NATIVE_ARCH_DEBUG; mp_dynamic_compiler.nlr_buf_num_regs = 0; @@ -331,6 +382,9 @@ MP_NOINLINE int main_(int argc, char **argv) { #elif defined(__arm__) && !defined(__thumb2__) mp_dynamic_compiler.native_arch = MP_NATIVE_ARCH_ARMV6; mp_dynamic_compiler.nlr_buf_num_regs = MICROPY_NLR_NUM_REGS_ARM_THUMB_FP; + #elif defined(__riscv) && (__riscv_xlen == 64) + mp_dynamic_compiler.native_arch = MP_NATIVE_ARCH_RV64IMC; + mp_dynamic_compiler.nlr_buf_num_regs = MICROPY_NLR_NUM_REGS_RV64I; #else mp_printf(&mp_stderr_print, "unable to determine host architecture for -march=host\n"); exit(1); @@ -338,6 +392,8 @@ MP_NOINLINE int main_(int argc, char **argv) { } else { return usage(argv); } + } else if (strncmp(argv[a], "-march-flags=", sizeof("-march-flags=") - 1) == 0) { + arch_flags = argv[a] + sizeof("-march-flags=") - 1; } else if (strcmp(argv[a], "--") == 0) { option_parsing_active = false; } else { @@ -352,6 +408,38 @@ MP_NOINLINE int main_(int argc, char **argv) { } } + if (arch_flags && mp_dynamic_compiler.native_arch != MP_NATIVE_ARCH_NONE) { + bool processed = false; + #if MICROPY_EMIT_NATIVE && MICROPY_EMIT_RV32 + if (mp_dynamic_compiler.native_arch == MP_NATIVE_ARCH_RV32IMC) { + mp_dynamic_compiler.backend_options = (void *)&rv32_options; + mp_uint_t raw_flags = 0; + if (parse_integer(arch_flags, &raw_flags)) { + if ((raw_flags & ~((mp_uint_t)RV32_EXT_ALL)) == 0) { + rv32_options.allowed_extensions = raw_flags; + processed = true; + } + } else if (strncmp(arch_flags, "zba", sizeof("zba") - 1) == 0) { + rv32_options.allowed_extensions |= RV32_EXT_ZBA; + processed = true; + } + } + #endif + if (!processed) { + mp_printf(&mp_stderr_print, "unrecognised arch flags\n"); + exit(1); + } + } + + #if MICROPY_EMIT_NATIVE + if ((MP_STATE_VM(default_emit_opt) == MP_EMIT_OPT_NATIVE_PYTHON + || MP_STATE_VM(default_emit_opt) == MP_EMIT_OPT_VIPER) + && mp_dynamic_compiler.native_arch == MP_NATIVE_ARCH_NONE) { + mp_printf(&mp_stderr_print, "arch not specified\n"); + exit(1); + } + #endif + if (input_file == NULL) { mp_printf(&mp_stderr_print, "no input file\n"); exit(1); @@ -371,7 +459,7 @@ MP_NOINLINE int main_(int argc, char **argv) { } int main(int argc, char **argv) { - mp_stack_ctrl_init(); + mp_cstack_init_with_sp_here(40000 * (sizeof(void *) / 4)); return main_(argc, argv); } diff --git a/mpy-cross/mpconfigport.h b/mpy-cross/mpconfigport.h index edbd9f87c0558..36bcbc113941c 100644 --- a/mpy-cross/mpconfigport.h +++ b/mpy-cross/mpconfigport.h @@ -59,6 +59,7 @@ #define MICROPY_COMP_CONST_FOLDING (1) #define MICROPY_COMP_MODULE_CONST (1) #define MICROPY_COMP_CONST (1) +#define MICROPY_COMP_CONST_FLOAT (1) #define MICROPY_COMP_DOUBLE_TUPLE_ASSIGN (1) #define MICROPY_COMP_TRIPLE_TUPLE_ASSIGN (1) #define MICROPY_COMP_RETURN_IF_EXPR (1) @@ -90,11 +91,12 @@ #define MICROPY_GCREGS_SETJMP (1) #endif -#define MICROPY_PY___FILE__ (0) +#define MICROPY_MODULE___FILE__ (0) #define MICROPY_PY_ARRAY (0) #define MICROPY_PY_ATTRTUPLE (0) #define MICROPY_PY_COLLECTIONS (0) -#define MICROPY_PY_MATH (0) +#define MICROPY_PY_MATH (MICROPY_COMP_CONST_FLOAT) +#define MICROPY_PY_MATH_CONSTANTS (MICROPY_COMP_CONST_FLOAT) #define MICROPY_PY_CMATH (0) #define MICROPY_PY_GC (0) #define MICROPY_PY_IO (0) @@ -102,23 +104,6 @@ // type definitions for the specific machine -#ifdef __LP64__ -typedef long mp_int_t; // must be pointer size -typedef unsigned long mp_uint_t; // must be pointer size -#elif defined(__MINGW32__) && defined(_WIN64) -#include -typedef __int64 mp_int_t; -typedef unsigned __int64 mp_uint_t; -#elif defined(_MSC_VER) && defined(_WIN64) -typedef __int64 mp_int_t; -typedef unsigned __int64 mp_uint_t; -#else -// These are definitions for machines where sizeof(int) == sizeof(void*), -// regardless for actual size. -typedef int mp_int_t; // must be pointer size -typedef unsigned int mp_uint_t; // must be pointer size -#endif - // Cannot include , as it may lead to symbol name clashes #if _FILE_OFFSET_BITS == 64 && !defined(__LP64__) typedef long long mp_off_t; @@ -143,7 +128,7 @@ typedef long mp_off_t; #ifdef _MSC_VER #define MP_ENDIANNESS_LITTLE (1) -#define NORETURN __declspec(noreturn) +#define MP_NORETURN __declspec(noreturn) #define MP_NOINLINE __declspec(noinline) #define MP_ALWAYSINLINE __forceinline #define MP_LIKELY(x) (x) diff --git a/mpy-cross/mpy_cross/__init__.py b/mpy-cross/mpy_cross/__init__.py index 91cd6f99335b3..6d7002a3b89d2 100644 --- a/mpy-cross/mpy_cross/__init__.py +++ b/mpy-cross/mpy_cross/__init__.py @@ -25,7 +25,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -from __future__ import print_function import os import re import stat @@ -44,6 +43,7 @@ "NATIVE_ARCH_XTENSA": "xtensa", "NATIVE_ARCH_XTENSAWIN": "xtensawin", "NATIVE_ARCH_RV32IMC": "rv32imc", + "NATIVE_ARCH_RV64IMC": "rv64imc", } globals().update(NATIVE_ARCHS) diff --git a/mpy-cross/mpy_cross/__main__.py b/mpy-cross/mpy_cross/__main__.py index 2b6b81c333362..fe78a9e077e77 100644 --- a/mpy-cross/mpy_cross/__main__.py +++ b/mpy-cross/mpy_cross/__main__.py @@ -25,7 +25,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -from __future__ import print_function import argparse import sys diff --git a/ports/analog/supervisor/port.c b/ports/analog/supervisor/port.c index 86d96bc251ba2..ee410a38148e9 100644 --- a/ports/analog/supervisor/port.c +++ b/ports/analog/supervisor/port.c @@ -67,7 +67,7 @@ static uint32_t subsec, sec = 0; static uint32_t tick_flag = 0; // defined by cmsis core files -extern void NVIC_SystemReset(void) NORETURN; +extern void NVIC_SystemReset(void) MP_NORETURN; volatile uint32_t system_ticks = 0; diff --git a/ports/atmel-samd/common-hal/alarm/__init__.c b/ports/atmel-samd/common-hal/alarm/__init__.c index 483aa1a9679ad..38ef58bea7659 100644 --- a/ports/atmel-samd/common-hal/alarm/__init__.c +++ b/ports/atmel-samd/common-hal/alarm/__init__.c @@ -133,7 +133,7 @@ void common_hal_alarm_set_deep_sleep_alarms(size_t n_alarms, const mp_obj_t *ala _setup_sleep_alarms(true, n_alarms, alarms); } -void NORETURN common_hal_alarm_enter_deep_sleep(void) { +void MP_NORETURN common_hal_alarm_enter_deep_sleep(void) { alarm_pin_pinalarm_prepare_for_deep_sleep(); alarm_time_timealarm_prepare_for_deep_sleep(); // port_disable_tick(); // TODO: Required for SAMD? diff --git a/ports/atmel-samd/common-hal/alarm/touch/TouchAlarm.c b/ports/atmel-samd/common-hal/alarm/touch/TouchAlarm.c index 3c1fbb5ba7b32..de98b7ec85ae6 100644 --- a/ports/atmel-samd/common-hal/alarm/touch/TouchAlarm.c +++ b/ports/atmel-samd/common-hal/alarm/touch/TouchAlarm.c @@ -7,6 +7,6 @@ #include "shared-bindings/alarm/touch/TouchAlarm.h" #include "shared-bindings/microcontroller/__init__.h" -NORETURN void common_hal_alarm_touch_touchalarm_construct(alarm_touch_touchalarm_obj_t *self, const mcu_pin_obj_t *pin) { +MP_NORETURN void common_hal_alarm_touch_touchalarm_construct(alarm_touch_touchalarm_obj_t *self, const mcu_pin_obj_t *pin) { mp_raise_NotImplementedError_varg(MP_ERROR_TEXT("%q"), MP_QSTR_TouchAlarm); } diff --git a/ports/atmel-samd/reset.c b/ports/atmel-samd/reset.c index 9cacd4ab951ae..816e23e3251db 100644 --- a/ports/atmel-samd/reset.c +++ b/ports/atmel-samd/reset.c @@ -9,7 +9,7 @@ #include "reset.h" #include "supervisor/filesystem.h" -void NVIC_SystemReset(void) NORETURN; +void NVIC_SystemReset(void) MP_NORETURN; void reset(void) { filesystem_flush(); diff --git a/ports/atmel-samd/reset.h b/ports/atmel-samd/reset.h index c74d25fa01ea5..248d43779fdc9 100644 --- a/ports/atmel-samd/reset.h +++ b/ports/atmel-samd/reset.h @@ -16,6 +16,6 @@ extern uint32_t _bootloader_dbl_tap; -void reset_to_bootloader(void) NORETURN; -void reset(void) NORETURN; +void reset_to_bootloader(void) MP_NORETURN; +void reset(void) MP_NORETURN; bool bootloader_available(void); diff --git a/ports/atmel-samd/supervisor/port.c b/ports/atmel-samd/supervisor/port.c index fe7b4a3676c79..288446cf62e71 100644 --- a/ports/atmel-samd/supervisor/port.c +++ b/ports/atmel-samd/supervisor/port.c @@ -685,7 +685,7 @@ void port_idle_until_interrupt(void) { /** * \brief Default interrupt handler for unused IRQs. */ -__attribute__((used)) NORETURN void HardFault_Handler(void) { +__attribute__((used)) MP_NORETURN void HardFault_Handler(void) { #ifdef ENABLE_MICRO_TRACE_BUFFER // Turn off the micro trace buffer so we don't fill it up in the infinite // loop below. diff --git a/ports/espressif/bindings/espidf/__init__.c b/ports/espressif/bindings/espidf/__init__.c index 012ed7d9f9adf..897da0eb5717b 100644 --- a/ports/espressif/bindings/espidf/__init__.c +++ b/ports/espressif/bindings/espidf/__init__.c @@ -106,7 +106,7 @@ MP_DEFINE_CONST_OBJ_TYPE( //| ... //| //| -NORETURN void mp_raise_espidf_MemoryError(void) { +MP_NORETURN void mp_raise_espidf_MemoryError(void) { nlr_raise(mp_obj_new_exception(&mp_type_espidf_MemoryError)); } diff --git a/ports/espressif/bindings/espidf/__init__.h b/ports/espressif/bindings/espidf/__init__.h index 7c112abf7368c..92d2e0c276931 100644 --- a/ports/espressif/bindings/espidf/__init__.h +++ b/ports/espressif/bindings/espidf/__init__.h @@ -15,9 +15,9 @@ extern const mp_obj_type_t mp_type_espidf_IDFError; extern const mp_obj_type_t mp_type_espidf_MemoryError; -NORETURN void mp_raise_espidf_MemoryError(void); +MP_NORETURN void mp_raise_espidf_MemoryError(void); -void raise_esp_error(esp_err_t err) NORETURN; +void raise_esp_error(esp_err_t err) MP_NORETURN; #define CHECK_ESP_RESULT(x) do { int res = (x); if (res != ESP_OK) raise_esp_error(res); } while (0) size_t common_hal_espidf_get_total_psram(void); diff --git a/ports/espressif/common-hal/alarm/__init__.c b/ports/espressif/common-hal/alarm/__init__.c index 629f976039fd1..4f8e7e530425a 100644 --- a/ports/espressif/common-hal/alarm/__init__.c +++ b/ports/espressif/common-hal/alarm/__init__.c @@ -189,7 +189,7 @@ void common_hal_alarm_set_deep_sleep_alarms(size_t n_alarms, const mp_obj_t *ala _setup_sleep_alarms(true, n_alarms, alarms); } -void NORETURN common_hal_alarm_enter_deep_sleep(void) { +void MP_NORETURN common_hal_alarm_enter_deep_sleep(void) { alarm_pin_pinalarm_prepare_for_deep_sleep(); #if CIRCUITPY_ALARM_TOUCH alarm_touch_touchalarm_prepare_for_deep_sleep(); diff --git a/ports/espressif/common-hal/microcontroller/__init__.c b/ports/espressif/common-hal/microcontroller/__init__.c index d23afce4d999e..9b50cd7b5d286 100644 --- a/ports/espressif/common-hal/microcontroller/__init__.c +++ b/ports/espressif/common-hal/microcontroller/__init__.c @@ -142,7 +142,7 @@ void common_hal_mcu_on_next_reset(mcu_runmode_t runmode) { } } -void NORETURN common_hal_mcu_reset(void) { +void MP_NORETURN common_hal_mcu_reset(void) { filesystem_flush(); // TODO: implement as part of flash improvements esp_restart(); } diff --git a/ports/espressif/supervisor/port.c b/ports/espressif/supervisor/port.c index b3045026e12d9..d2aafb99f0bd6 100644 --- a/ports/espressif/supervisor/port.c +++ b/ports/espressif/supervisor/port.c @@ -103,7 +103,7 @@ static esp_timer_handle_t _sleep_timer; TaskHandle_t circuitpython_task = NULL; -extern void esp_restart(void) NORETURN; +extern void esp_restart(void) MP_NORETURN; static void tick_on_cp_core(void *arg) { supervisor_tick(); diff --git a/ports/espressif/tools/update_sdkconfig.py b/ports/espressif/tools/update_sdkconfig.py index 209b976b881e9..d40614edb75d8 100644 --- a/ports/espressif/tools/update_sdkconfig.py +++ b/ports/espressif/tools/update_sdkconfig.py @@ -157,7 +157,7 @@ def sym_default(sym): default=False, help="Updates the sdkconfigs outside of the board directory.", ) -def update(debug, board, update_all): # noqa: C901: too complex +def update(debug, board, update_all): # noqa: C901 too complex """Updates related sdkconfig files based on the build directory version that was likely modified by menuconfig.""" diff --git a/ports/mimxrt10xx/reset.h b/ports/mimxrt10xx/reset.h index 04c951221e9e5..cae838556bce3 100644 --- a/ports/mimxrt10xx/reset.h +++ b/ports/mimxrt10xx/reset.h @@ -16,6 +16,6 @@ #define DBL_TAP_MAGIC 0xf01669ef // Randomly selected, adjusted to have first and last bit set #define DBL_TAP_MAGIC_QUICK_BOOT 0xf02669ef -void reset_to_bootloader(void) NORETURN; -void reset(void) NORETURN; +void reset_to_bootloader(void) MP_NORETURN; +void reset(void) MP_NORETURN; bool bootloader_available(void); diff --git a/ports/nordic/common-hal/alarm/__init__.c b/ports/nordic/common-hal/alarm/__init__.c index 9753b0321fd06..026e117b9a67a 100644 --- a/ports/nordic/common-hal/alarm/__init__.c +++ b/ports/nordic/common-hal/alarm/__init__.c @@ -240,7 +240,7 @@ void common_hal_alarm_set_deep_sleep_alarms(size_t n_alarms, const mp_obj_t *ala #define PRESCALER_VALUE_IN_DEEP_SLEEP (1024) -void NORETURN common_hal_alarm_enter_deep_sleep(void) { +void MP_NORETURN common_hal_alarm_enter_deep_sleep(void) { alarm_pin_pinalarm_prepare_for_deep_sleep(); alarm_time_timealarm_prepare_for_deep_sleep(); diff --git a/ports/raspberrypi/common-hal/alarm/__init__.c b/ports/raspberrypi/common-hal/alarm/__init__.c index c47ddc3150515..a72b3a368d407 100644 --- a/ports/raspberrypi/common-hal/alarm/__init__.c +++ b/ports/raspberrypi/common-hal/alarm/__init__.c @@ -197,7 +197,7 @@ void common_hal_alarm_set_deep_sleep_alarms(size_t n_alarms, const mp_obj_t *ala _setup_sleep_alarms(true, n_alarms, alarms); } -void NORETURN common_hal_alarm_enter_deep_sleep(void) { +void MP_NORETURN common_hal_alarm_enter_deep_sleep(void) { bool timealarm_set = alarm_time_timealarm_is_set(); #if CIRCUITPY_CYW43 diff --git a/ports/raspberrypi/common-hal/wifi/Radio.c b/ports/raspberrypi/common-hal/wifi/Radio.c index 3c22c3548d191..73e7794ecc2dc 100644 --- a/ports/raspberrypi/common-hal/wifi/Radio.c +++ b/ports/raspberrypi/common-hal/wifi/Radio.c @@ -48,7 +48,7 @@ static inline void nw_put_le32(uint8_t *buf, uint32_t x) { buf[3] = x >> 24; } -NORETURN static void ro_attribute(qstr attr) { +MP_NORETURN static void ro_attribute(qstr attr) { mp_raise_NotImplementedError_varg(MP_ERROR_TEXT("%q is read-only for this board"), attr); } diff --git a/ports/raspberrypi/common-hal/wifi/__init__.h b/ports/raspberrypi/common-hal/wifi/__init__.h index 73988e8437987..4005c48a3cde2 100644 --- a/ports/raspberrypi/common-hal/wifi/__init__.h +++ b/ports/raspberrypi/common-hal/wifi/__init__.h @@ -11,7 +11,7 @@ #include "lwip/ip_addr.h" void wifi_reset(void); -NORETURN void raise_cyw_error(int err); +MP_NORETURN void raise_cyw_error(int err); #define CHECK_CYW_RESULT(x) do { int res = (x); if (res != 0) raise_cyw_error(res); } while (0) void ipaddress_ipaddress_to_lwip(mp_obj_t ip_address, ip_addr_t *lwip_ip_address); diff --git a/ports/raspberrypi/supervisor/port.c b/ports/raspberrypi/supervisor/port.c index bb6482240a697..0316e34ad951c 100644 --- a/ports/raspberrypi/supervisor/port.c +++ b/ports/raspberrypi/supervisor/port.c @@ -588,7 +588,7 @@ void port_idle_until_interrupt(void) { /** * \brief Default interrupt handler for unused IRQs. */ -extern NORETURN void isr_hardfault(void); // provide a prototype to avoid a missing-prototypes diagnostic +extern MP_NORETURN void isr_hardfault(void); // provide a prototype to avoid a missing-prototypes diagnostic __attribute__((used)) void __not_in_flash_func(isr_hardfault)(void) { // Only safe mode from core 0 which is running CircuitPython. Core 1 faulting // should not be fatal to CP. (Fingers crossed.) diff --git a/ports/stm/common-hal/alarm/__init__.c b/ports/stm/common-hal/alarm/__init__.c index 3d6c4466882b4..1be8f8dc10d09 100644 --- a/ports/stm/common-hal/alarm/__init__.c +++ b/ports/stm/common-hal/alarm/__init__.c @@ -135,7 +135,7 @@ void common_hal_alarm_set_deep_sleep_alarms(size_t n_alarms, const mp_obj_t *ala _setup_sleep_alarms(true, n_alarms, alarms); } -void NORETURN common_hal_alarm_enter_deep_sleep(void) { +void MP_NORETURN common_hal_alarm_enter_deep_sleep(void) { alarm_set_wakeup_reason(STM_WAKEUP_UNDEF); alarm_pin_pinalarm_prepare_for_deep_sleep(); alarm_time_timealarm_prepare_for_deep_sleep(); diff --git a/ports/stm/mpconfigport_nanbox.h b/ports/stm/mpconfigport_nanbox.h deleted file mode 100644 index 164850112e01f..0000000000000 --- a/ports/stm/mpconfigport_nanbox.h +++ /dev/null @@ -1,25 +0,0 @@ -// This file is part of the CircuitPython project: https://circuitpython.org -// -// SPDX-FileCopyrightText: Copyright (c) 2016 Damien P. George -// -// SPDX-License-Identifier: MIT - -#pragma once - -// select nan-boxing object model -#define MICROPY_OBJ_REPR (MICROPY_OBJ_REPR_D) - -// native emitters don't work with nan-boxing -#define MICROPY_EMIT_X86 (0) -#define MICROPY_EMIT_X64 (0) -#define MICROPY_EMIT_THUMB (0) -#define MICROPY_EMIT_ARM (0) - -#include - -typedef int64_t mp_int_t; -typedef uint64_t mp_uint_t; -#define UINT_FMT "%llu" -#define INT_FMT "%lld" - -#include diff --git a/ports/stm/supervisor/port.c b/ports/stm/supervisor/port.c index 3820a046fc4af..1ffc8656a888e 100644 --- a/ports/stm/supervisor/port.c +++ b/ports/stm/supervisor/port.c @@ -39,7 +39,7 @@ #include STM32_HAL_H -void NVIC_SystemReset(void) NORETURN; +void NVIC_SystemReset(void) MP_NORETURN; #if (CPY_STM32H7) || (CPY_STM32F7) #if defined(CIRCUITPY_HW_SDRAM_SIZE) diff --git a/ports/unix/Makefile b/ports/unix/Makefile index 6461c310eeb68..32e3b44b2273a 100644 --- a/ports/unix/Makefile +++ b/ports/unix/Makefile @@ -27,6 +27,9 @@ FROZEN_MANIFEST ?= variants/manifest.py # This should be configured by the mpconfigvariant.mk PROG ?= micropython +# For use in test rules below +ABS_PROG = $(abspath $(BUILD)/$(PROG)) + # qstr definitions (must come before including py.mk) QSTR_DEFS += qstrdefsport.h QSTR_GLOBAL_DEPENDENCIES += $(VARIANT_DIR)/mpconfigvariant.h @@ -46,6 +49,7 @@ INC += -I$(BUILD) # compiler settings CWARN = -Wall -Werror +// CIRCUITPY-CHANGE: add -Wno-missing-field-initializers CWARN += -Wextra -Wno-unused-parameter -Wpointer-arith -Wdouble-promotion -Wfloat-conversion -Wno-missing-field-initializers CFLAGS += $(INC) $(CWARN) -std=gnu99 -DUNIX $(COPT) -I$(VARIANT_DIR) $(CFLAGS_EXTRA) @@ -53,6 +57,10 @@ CFLAGS += $(INC) $(CWARN) -std=gnu99 -DUNIX $(COPT) -I$(VARIANT_DIR) $(CFLAGS_EX # This option has no effect on 64-bit builds. CFLAGS += -D_FILE_OFFSET_BITS=64 +# Force the use of 64-bits for time_t in C library functions on 32-bit platforms. +# This option has no effect on 64-bit builds. +CFLAGS += -D_TIME_BITS=64 + # Debugging/Optimization ifdef DEBUG COPT ?= -Og @@ -139,7 +147,11 @@ ifeq ($(MICROPY_PY_SOCKET),1) CFLAGS += -DMICROPY_PY_SOCKET=1 endif ifeq ($(MICROPY_PY_THREAD),1) +ifeq ($(MICROPY_PY_THREAD_GIL),1) +CFLAGS += -DMICROPY_PY_THREAD=1 -DMICROPY_PY_THREAD_GIL=1 +else CFLAGS += -DMICROPY_PY_THREAD=1 -DMICROPY_PY_THREAD_GIL=0 +endif LDFLAGS += $(LIBPTHREAD) endif @@ -225,6 +237,7 @@ $(BUILD)/supervisor/shared/translate/translate.o: $(HEADER_BUILD)/qstrdefs.gener SHARED_SRC_C += $(addprefix shared/,\ runtime/gchelper_generic.c \ + runtime/pyexec.c \ timeutils/timeutils.c \ $(SHARED_SRC_C_EXTRA) \ ) @@ -280,7 +293,7 @@ print-failures clean-failures: TEST_ARGS ?= test: $(BUILD)/$(PROG) $(TOP)/tests/run-tests.py $(eval DIRNAME=ports/$(notdir $(CURDIR))) - cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(BUILD)/$(PROG) ./run-tests.py $(TEST_ARGS) + cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(BUILD)/$(PROG) ./run-tests.py test_full: $(BUILD)/$(PROG) $(TOP)/tests/run-tests.py $(eval DIRNAME=ports/$(notdir $(CURDIR))) diff --git a/ports/unix/README.md b/ports/unix/README.md index b7aa6e3fef76d..ee983a882cc83 100644 --- a/ports/unix/README.md +++ b/ports/unix/README.md @@ -72,6 +72,14 @@ To run the complete testsuite, use: $ make test +There are other make targets to interact with the testsuite: + + $ make test//int # Run all tests matching the pattern "int" + $ make test/ports/unix # Run all tests in ports/unix + $ make test-failures # Re-run only the failed tests + $ make print-failures # print the differences for failed tests + $ make clean-failures # delete the .exp and .out files from failed tests + The Unix port comes with a built-in package manager called `mip`, e.g.: $ ./build-standard/micropython -m mip install hmac @@ -155,3 +163,21 @@ The default compiler optimisation level is -Os, or -Og if `DEBUG=1` is set. Setting the variable `COPT` will explicitly set the optimisation level. For example `make [other arguments] COPT=-O0 DEBUG=1` will build a binary with no optimisations, assertions enabled, and debug symbols. + +### Sanitizers + +Sanitizers are extra runtime checks supported by gcc and clang. The CI process +supports building with the "undefined behavior" (UBSan) or "address" (ASan) +sanitizers. The script `tools/ci.sh` is the source of truth about how to build +and run in these modes. + +Several classes of checks are disabled via compiler flags: + +* In the undefined behavior sanitizer, checks based on the presence of the + `non_null` attribute are disabled because the code makes technically incorrect + calls like `memset(NULL, 0, 0)`. A future C standard is likely to permit such + calls. +* In the address sanitizer, `detect_stack_use_after_return` is disabled. This + check is intended to make sure locals in a "returned from" stack frame are not + used. However, this mode interferes with various assumptions that + MicroPython's stack checking, NLR, and GC rely on. diff --git a/ports/unix/coverage.c b/ports/unix/coverage.c index 7f13f9756f177..49426f0f3e865 100644 --- a/ports/unix/coverage.c +++ b/ports/unix/coverage.c @@ -97,6 +97,7 @@ static const mp_rom_map_elem_t rawfile_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_write), MP_ROM_PTR(&mp_stream_write_obj) }, { MP_ROM_QSTR(MP_QSTR_write1), MP_ROM_PTR(&mp_stream_write1_obj) }, { MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) }, + { MP_ROM_QSTR(MP_QSTR_readinto1), MP_ROM_PTR(&mp_stream_readinto1_obj) }, { MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) }, { MP_ROM_QSTR(MP_QSTR_ioctl), MP_ROM_PTR(&mp_stream_ioctl_obj) }, }; @@ -171,19 +172,34 @@ static void pairheap_test(size_t nops, int *ops) { if (mp_pairheap_is_empty(pairheap_lt, heap)) { mp_printf(&mp_plat_print, " -"); } else { - mp_printf(&mp_plat_print, " %d", mp_pairheap_peek(pairheap_lt, heap) - &node[0]); + mp_printf(&mp_plat_print, " %d", (int)(mp_pairheap_peek(pairheap_lt, heap) - &node[0])); ; } } mp_printf(&mp_plat_print, "\npop all:"); while (!mp_pairheap_is_empty(pairheap_lt, heap)) { - mp_printf(&mp_plat_print, " %d", mp_pairheap_peek(pairheap_lt, heap) - &node[0]); + mp_printf(&mp_plat_print, " %d", (int)(mp_pairheap_peek(pairheap_lt, heap) - &node[0])); ; heap = mp_pairheap_pop(pairheap_lt, heap); } mp_printf(&mp_plat_print, "\n"); } +// CIRCUITPY-CHANGE: not turned on in CircuitPython +#if MICROPY_SCHEDULER_STATIC_NODES +static mp_sched_node_t mp_coverage_sched_node; +static bool coverage_sched_function_continue; + +static void coverage_sched_function(mp_sched_node_t *node) { + (void)node; + mp_printf(&mp_plat_print, "scheduled function\n"); + if (coverage_sched_function_continue) { + // Re-scheduling node will cause it to run again next time scheduled functions are run + mp_sched_schedule_node(&mp_coverage_sched_node, coverage_sched_function); + } +} +#endif + // function to run extra tests for things that can't be checked by scripts static mp_obj_t extra_coverage(void) { // mp_printf (used by ports that don't have a native printf) @@ -191,10 +207,22 @@ static mp_obj_t extra_coverage(void) { mp_printf(&mp_plat_print, "# mp_printf\n"); mp_printf(&mp_plat_print, "%d %+d % d\n", -123, 123, 123); // sign mp_printf(&mp_plat_print, "%05d\n", -123); // negative number with zero padding - mp_printf(&mp_plat_print, "%ld\n", 123); // long - mp_printf(&mp_plat_print, "%lx\n", 0x123); // long hex - mp_printf(&mp_plat_print, "%X\n", 0x1abcdef); // capital hex - mp_printf(&mp_plat_print, "%.2s %.3s '%4.4s' '%5.5q' '%.3q'\n", "abc", "abc", "abc", MP_QSTR_True, MP_QSTR_True); // fixed string precision + mp_printf(&mp_plat_print, "%ld\n", 123l); // long + mp_printf(&mp_plat_print, "%lx\n", 0x123fl); // long hex + mp_printf(&mp_plat_print, "%lX\n", 0x123fl); // capital long hex + if (sizeof(mp_int_t) == 8) { + mp_printf(&mp_plat_print, "%llx\n", LLONG_MAX); // long long hex + mp_printf(&mp_plat_print, "%llX\n", LLONG_MAX); // capital long long hex + mp_printf(&mp_plat_print, "%llu\n", ULLONG_MAX); // unsigned long long + } else { + // fake for platforms without narrower mp_int_t + mp_printf(&mp_plat_print, "7fffffffffffffff\n"); + mp_printf(&mp_plat_print, "7FFFFFFFFFFFFFFF\n"); + mp_printf(&mp_plat_print, "18446744073709551615\n"); + } + mp_printf(&mp_plat_print, "%p\n", (void *)0x789f); // pointer + mp_printf(&mp_plat_print, "%P\n", (void *)0x789f); // pointer uppercase + mp_printf(&mp_plat_print, "%.2s %.3s '%4.4s' '%5.5q' '%.3q'\n", "abc", "abc", "abc", (qstr)MP_QSTR_True, (qstr)MP_QSTR_True); // fixed string precision mp_printf(&mp_plat_print, "%.*s\n", -1, "abc"); // negative string precision mp_printf(&mp_plat_print, "%b %b\n", 0, 1); // bools #ifndef NDEBUG @@ -204,10 +232,36 @@ static mp_obj_t extra_coverage(void) { #endif mp_printf(&mp_plat_print, "%d\n", 0x80000000); // should print signed mp_printf(&mp_plat_print, "%u\n", 0x80000000); // should print unsigned - mp_printf(&mp_plat_print, "%x\n", 0x80000000); // should print unsigned - mp_printf(&mp_plat_print, "%X\n", 0x80000000); // should print unsigned - mp_printf(&mp_plat_print, "abc\n%"); // string ends in middle of format specifier + mp_printf(&mp_plat_print, "%x\n", 0x8000000f); // should print unsigned + mp_printf(&mp_plat_print, "%X\n", 0x8000000f); // should print unsigned + // note: storing the string in a variable is enough to prevent the + // format string checker from checking this format string. Otherwise, + // it would be a compile time diagnostic under the format string + // checker. + const char msg[] = "abc\n%"; + mp_printf(&mp_plat_print, msg); // string ends in middle of format specifier mp_printf(&mp_plat_print, "%%\n"); // literal % character + mp_printf(&mp_plat_print, ".%-3s.\n", "a"); // left adjust + + // Check that all kinds of mp_printf arguments are parsed out + // correctly, by having a char argument before and after each main type + // of value that can be formatted. + mp_printf(&mp_plat_print, "%c%%%c\n", '<', '>'); + mp_printf(&mp_plat_print, "%c%p%c\n", '<', (void *)0xaaaa, '>'); + mp_printf(&mp_plat_print, "%c%b%c\n", '<', true, '>'); + mp_printf(&mp_plat_print, "%c%d%c\n", '<', 0xaaaa, '>'); + mp_printf(&mp_plat_print, "%c%ld%c\n", '<', 0xaaaal, '>'); + mp_printf(&mp_plat_print, "%c" INT_FMT "%c\n", '<', (mp_int_t)0xaaaa, '>'); + mp_printf(&mp_plat_print, "%c%s%c\n", '<', "test", '>'); + mp_printf(&mp_plat_print, "%c%f%c\n", '<', MICROPY_FLOAT_CONST(1000.), '>'); + mp_printf(&mp_plat_print, "%c%q%c\n", '<', (qstr)MP_QSTR_True, '>'); + if (sizeof(mp_int_t) == 8) { + mp_printf(&mp_plat_print, "%c%lld%c\n", '<', LLONG_MAX, '>'); + } else { + mp_printf(&mp_plat_print, "<9223372036854775807>\n"); + } + + } // GC @@ -220,11 +274,11 @@ static mp_obj_t extra_coverage(void) { gc_unlock(); // using gc_realloc to resize to 0, which means free the memory - void *p = gc_alloc(4, false); + void *p = gc_alloc(4, 0); mp_printf(&mp_plat_print, "%p\n", gc_realloc(p, 0, false)); // calling gc_nbytes with a non-heap pointer - mp_printf(&mp_plat_print, "%p\n", gc_nbytes(NULL)); + mp_printf(&mp_plat_print, "%d\n", (int)gc_nbytes(NULL)); } // GC initialisation and allocation stress test, to check the logic behind ALLOC_TABLE_GAP_BYTE @@ -285,7 +339,7 @@ static mp_obj_t extra_coverage(void) { } ptrs[i][j] = j; } - mp_printf(&mp_plat_print, "%d %d\n", i, all_zero); + mp_printf(&mp_plat_print, "%d %d\n", (int)i, (int)all_zero); // hide the pointer from the GC and collect ptrs[i] = FLIP_POINTER(ptrs[i]); @@ -301,7 +355,7 @@ static mp_obj_t extra_coverage(void) { break; } } - mp_printf(&mp_plat_print, "%d %d\n", i, correct_contents); + mp_printf(&mp_plat_print, "%d %d\n", (int)i, (int)correct_contents); } // free the memory blocks @@ -400,7 +454,7 @@ static mp_obj_t extra_coverage(void) { // create a bytearray via mp_obj_new_bytearray mp_buffer_info_t bufinfo; mp_get_buffer_raise(mp_obj_new_bytearray(4, "data"), &bufinfo, MP_BUFFER_RW); - mp_printf(&mp_plat_print, "%.*s\n", bufinfo.len, bufinfo.buf); + mp_printf(&mp_plat_print, "%.*s\n", (int)bufinfo.len, bufinfo.buf); } // mpz @@ -464,6 +518,38 @@ static mp_obj_t extra_coverage(void) { mp_int_t value_signed; mpz_as_int_checked(&mpz, &value_signed); mp_printf(&mp_plat_print, "%d\n", (int)value_signed); + + // hash the zero mpz integer + mpz_set_from_int(&mpz, 0); + mp_printf(&mp_plat_print, "%d\n", (int)mpz_hash(&mpz)); + + // convert the mpz zero integer to int + mp_printf(&mp_plat_print, "%d\n", mpz_as_int_checked(&mpz, &value_signed)); + mp_printf(&mp_plat_print, "%d\n", (int)value_signed); + + // mpz_set_from_float with 0 as argument + mpz_set_from_float(&mpz, 0); + mp_printf(&mp_plat_print, "%f\n", mpz_as_float(&mpz)); + + // convert a large integer value (stored in a mpz) to mp_uint_t and to ll; + mp_obj_t obj_bigint = mp_obj_new_int_from_uint((mp_uint_t)0xdeadbeef); + mp_printf(&mp_plat_print, "%x\n", (int)mp_obj_get_uint(obj_bigint)); + obj_bigint = mp_obj_new_int_from_ll(0xc0ffee777c0ffeell); + long long value_ll = mp_obj_get_ll(obj_bigint); + mp_printf(&mp_plat_print, "%x%08x\n", (uint32_t)(value_ll >> 32), (uint32_t)value_ll); + + // convert a large integer value (stored via a struct object) to uint and to ll + // `deadbeef` global is an uctypes.struct defined by extra_coverage.py + obj_bigint = mp_load_global(MP_QSTR_deadbeef); + mp_printf(&mp_plat_print, "%x\n", (int)mp_obj_get_uint(obj_bigint)); + value_ll = mp_obj_get_ll(obj_bigint); + mp_printf(&mp_plat_print, "%x%08x\n", (uint32_t)(value_ll >> 32), (uint32_t)value_ll); + + // convert a smaller integer value to mp_uint_t and to ll + obj_bigint = mp_obj_new_int_from_uint(0xc0ffee); + mp_printf(&mp_plat_print, "%x\n", (int)mp_obj_get_uint(obj_bigint)); + value_ll = mp_obj_get_ll(obj_bigint); + mp_printf(&mp_plat_print, "%x%08x\n", (uint32_t)(value_ll >> 32), (uint32_t)value_ll); } // runtime utils @@ -481,7 +567,7 @@ static mp_obj_t extra_coverage(void) { mp_call_function_2_protected(MP_OBJ_FROM_PTR(&mp_builtin_divmod_obj), mp_obj_new_str_from_cstr("abc"), mp_obj_new_str_from_cstr("abc")); // mp_obj_int_get_checked with mp_obj_int_t that has a value that is a small integer - mp_printf(&mp_plat_print, "%d\n", mp_obj_int_get_checked(mp_obj_int_new_mpz())); + mp_printf(&mp_plat_print, "%d\n", (int)mp_obj_int_get_checked(MP_OBJ_FROM_PTR(mp_obj_int_new_mpz()))); // mp_obj_int_get_uint_checked with non-negative small-int mp_printf(&mp_plat_print, "%d\n", (int)mp_obj_int_get_uint_checked(MP_OBJ_NEW_SMALL_INT(1))); @@ -506,6 +592,22 @@ static mp_obj_t extra_coverage(void) { mp_obj_print_exception(&mp_plat_print, MP_OBJ_FROM_PTR(nlr.ret_val)); } + // mp_obj_get_uint from a non-int object (should raise exception) + if (nlr_push(&nlr) == 0) { + mp_obj_get_uint(mp_const_none); + nlr_pop(); + } else { + mp_obj_print_exception(&mp_plat_print, MP_OBJ_FROM_PTR(nlr.ret_val)); + } + + // mp_obj_int_get_ll from a non-int object (should raise exception) + if (nlr_push(&nlr) == 0) { + mp_obj_get_ll(mp_const_none); + nlr_pop(); + } else { + mp_obj_print_exception(&mp_plat_print, MP_OBJ_FROM_PTR(nlr.ret_val)); + } + // call mp_obj_new_exception_args (it's a part of the public C API and not used in the core) mp_obj_print_exception(&mp_plat_print, mp_obj_new_exception_args(&mp_type_ValueError, 0, NULL)); } @@ -515,26 +617,6 @@ static mp_obj_t extra_coverage(void) { mp_emitter_warning(MP_PASS_CODE_SIZE, "test"); } - // format float - { - mp_printf(&mp_plat_print, "# format float\n"); - - // format with inadequate buffer size - char buf[5]; - mp_format_float(1, buf, sizeof(buf), 'g', 0, '+'); - mp_printf(&mp_plat_print, "%s\n", buf); - - // format with just enough buffer so that precision must be - // set from 0 to 1 twice - char buf2[8]; - mp_format_float(1, buf2, sizeof(buf2), 'g', 0, '+'); - mp_printf(&mp_plat_print, "%s\n", buf2); - - // format where precision is trimmed to avoid buffer overflow - mp_format_float(1, buf2, sizeof(buf2), 'e', 0, '+'); - mp_printf(&mp_plat_print, "%s\n", buf2); - } - // binary { mp_printf(&mp_plat_print, "# binary\n"); @@ -558,14 +640,26 @@ static mp_obj_t extra_coverage(void) { fun_bc.context = &context; fun_bc.child_table = NULL; fun_bc.bytecode = (const byte *)"\x01"; // just needed for n_state + #if MICROPY_PY_SYS_SETTRACE + struct _mp_raw_code_t rc = {}; + fun_bc.rc = &rc; + #endif mp_code_state_t *code_state = m_new_obj_var(mp_code_state_t, state, mp_obj_t, 1); code_state->fun_bc = &fun_bc; code_state->ip = (const byte *)"\x00"; // just needed for an invalid opcode code_state->sp = &code_state->state[0]; code_state->exc_sp_idx = 0; code_state->old_globals = NULL; + #if MICROPY_STACKLESS + code_state->prev = NULL; + #endif + #if MICROPY_PY_SYS_SETTRACE + code_state->prev_state = NULL; + code_state->frame = NULL; + #endif + mp_vm_return_kind_t ret = mp_execute_bytecode(code_state, MP_OBJ_NULL); - mp_printf(&mp_plat_print, "%d %d\n", ret, mp_obj_get_type(code_state->state[0]) == &mp_type_NotImplementedError); + mp_printf(&mp_plat_print, "%d %d\n", (int)ret, mp_obj_get_type(code_state->state[0]) == &mp_type_NotImplementedError); } // scheduler @@ -622,9 +716,25 @@ static mp_obj_t extra_coverage(void) { mp_obj_print_exception(&mp_plat_print, MP_OBJ_FROM_PTR(nlr.ret_val)); } mp_handle_pending(true); + + // CIRCUITPY-CHANGE: not turned on in CircuitPython + #if MICROPY_SCHEDULER_STATIC_NODES + coverage_sched_function_continue = true; + mp_sched_schedule_node(&mp_coverage_sched_node, coverage_sched_function); + for (int i = 0; i < 3; ++i) { + mp_printf(&mp_plat_print, "loop\n"); + mp_handle_pending(true); + } + // Clear this flag to prevent the function scheduling itself again + coverage_sched_function_continue = false; + // Will only run the first time through this loop, then not scheduled again + for (int i = 0; i < 3; ++i) { + mp_handle_pending(true); + } + #endif } - // CIRCUITPY-CHANGE: ringbuf is different + // CIRCUITPY-CHANGE: ringbuf is quite different // ringbuf { #define RINGBUF_SIZE 99 @@ -636,36 +746,36 @@ static mp_obj_t extra_coverage(void) { mp_printf(&mp_plat_print, "# ringbuf\n"); // Single-byte put/get with empty ringbuf. - mp_printf(&mp_plat_print, "%d %d\n", ringbuf_num_empty(&ringbuf), ringbuf_num_filled(&ringbuf)); + mp_printf(&mp_plat_print, "%d %d\n", (int)ringbuf_num_empty(&ringbuf), (int)ringbuf_num_filled(&ringbuf)); ringbuf_put(&ringbuf, 22); - mp_printf(&mp_plat_print, "%d %d\n", ringbuf_num_empty(&ringbuf), ringbuf_num_filled(&ringbuf)); + mp_printf(&mp_plat_print, "%d %d\n", (int)ringbuf_num_empty(&ringbuf), (int)ringbuf_num_filled(&ringbuf)); mp_printf(&mp_plat_print, "%d\n", ringbuf_get(&ringbuf)); - mp_printf(&mp_plat_print, "%d %d\n", ringbuf_num_empty(&ringbuf), ringbuf_num_filled(&ringbuf)); + mp_printf(&mp_plat_print, "%d %d\n", (int)ringbuf_num_empty(&ringbuf), (int)ringbuf_num_filled(&ringbuf)); // Two-byte put/get with empty ringbuf. ringbuf_put16(&ringbuf, 0xaa55); - mp_printf(&mp_plat_print, "%d %d\n", ringbuf_num_empty(&ringbuf), ringbuf_num_filled(&ringbuf)); + mp_printf(&mp_plat_print, "%d %d\n", (int)ringbuf_num_empty(&ringbuf), (int)ringbuf_num_filled(&ringbuf)); mp_printf(&mp_plat_print, "%04x\n", ringbuf_get16(&ringbuf)); - mp_printf(&mp_plat_print, "%d %d\n", ringbuf_num_empty(&ringbuf), ringbuf_num_filled(&ringbuf)); + mp_printf(&mp_plat_print, "%d %d\n", (int)ringbuf_num_empty(&ringbuf), (int)ringbuf_num_filled(&ringbuf)); // Two-byte put with full ringbuf. for (int i = 0; i < RINGBUF_SIZE; ++i) { ringbuf_put(&ringbuf, i); } - mp_printf(&mp_plat_print, "%d %d\n", ringbuf_num_empty(&ringbuf), ringbuf_num_filled(&ringbuf)); + mp_printf(&mp_plat_print, "%d %d\n", (int)ringbuf_num_empty(&ringbuf), (int)ringbuf_num_filled(&ringbuf)); mp_printf(&mp_plat_print, "%d\n", ringbuf_put16(&ringbuf, 0x11bb)); // Two-byte put with one byte free. ringbuf_get(&ringbuf); - mp_printf(&mp_plat_print, "%d %d\n", ringbuf_num_empty(&ringbuf), ringbuf_num_filled(&ringbuf)); + mp_printf(&mp_plat_print, "%d %d\n", (int)ringbuf_num_empty(&ringbuf), (int)ringbuf_num_filled(&ringbuf)); mp_printf(&mp_plat_print, "%d\n", ringbuf_put16(&ringbuf, 0x3377)); ringbuf_get(&ringbuf); - mp_printf(&mp_plat_print, "%d %d\n", ringbuf_num_empty(&ringbuf), ringbuf_num_filled(&ringbuf)); + mp_printf(&mp_plat_print, "%d %d\n", (int)ringbuf_num_empty(&ringbuf), (int)ringbuf_num_filled(&ringbuf)); mp_printf(&mp_plat_print, "%d\n", ringbuf_put16(&ringbuf, 0xcc99)); for (int i = 0; i < RINGBUF_SIZE - 2; ++i) { ringbuf_get(&ringbuf); } mp_printf(&mp_plat_print, "%04x\n", ringbuf_get16(&ringbuf)); - mp_printf(&mp_plat_print, "%d %d\n", ringbuf_num_empty(&ringbuf), ringbuf_num_filled(&ringbuf)); + mp_printf(&mp_plat_print, "%d %d\n", (int)ringbuf_num_empty(&ringbuf), (int)ringbuf_num_filled(&ringbuf)); // Two-byte put with wrap around on first byte: ringbuf_clear(&ringbuf); @@ -777,7 +887,7 @@ static mp_obj_t extra_coverage(void) { mp_obj_streamtest_t *s2 = mp_obj_malloc(mp_obj_streamtest_t, &mp_type_stest_textio2); // return a tuple of data for testing on the Python side - mp_obj_t items[] = {(mp_obj_t)&str_no_hash_obj, (mp_obj_t)&bytes_no_hash_obj, MP_OBJ_FROM_PTR(s), MP_OBJ_FROM_PTR(s2)}; + mp_obj_t items[] = {MP_OBJ_FROM_PTR(&str_no_hash_obj), MP_OBJ_FROM_PTR(&bytes_no_hash_obj), MP_OBJ_FROM_PTR(s), MP_OBJ_FROM_PTR(s2)}; return mp_obj_new_tuple(MP_ARRAY_SIZE(items), items); } MP_DEFINE_CONST_FUN_OBJ_0(extra_coverage_obj, extra_coverage); diff --git a/ports/unix/coveragecpp.cpp b/ports/unix/coveragecpp.cpp index 377b5acf763ce..8ba308f6468f3 100644 --- a/ports/unix/coveragecpp.cpp +++ b/ports/unix/coveragecpp.cpp @@ -1,12 +1,77 @@ extern "C" { -// CIRCUITPY-CHANGE: do not include everything: it causes compilation warnings -#include "py/obj.h" +// Include the complete public API to verify everything compiles as C++. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include } +// Invoke all (except one, see below) public API macros which initialize structs to make sure +// they are C++-compatible, meaning they explicitly initialize all struct members. +mp_obj_t f0(); +MP_DEFINE_CONST_FUN_OBJ_0(f0_obj, f0); +mp_obj_t f1(mp_obj_t); +MP_DEFINE_CONST_FUN_OBJ_1(f1_obj, f1); +mp_obj_t f2(mp_obj_t, mp_obj_t); +MP_DEFINE_CONST_FUN_OBJ_2(f2_obj, f2); +mp_obj_t f3(mp_obj_t, mp_obj_t, mp_obj_t); +MP_DEFINE_CONST_FUN_OBJ_3(f3_obj, f3); +mp_obj_t fvar(size_t, const mp_obj_t *); +MP_DEFINE_CONST_FUN_OBJ_VAR(fvar_obj, 1, fvar); +mp_obj_t fvarbetween(size_t, const mp_obj_t *); +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(fvarbetween_obj, 1, 2, fvarbetween); +mp_obj_t fkw(size_t, const mp_obj_t *, mp_map_t *); +MP_DEFINE_CONST_FUN_OBJ_KW(fkw_obj, 1, fkw); + +static const mp_rom_map_elem_t table[] = { + { MP_ROM_QSTR(MP_QSTR_f0), MP_ROM_PTR(&f0_obj) }, +}; +MP_DEFINE_CONST_MAP(map, table); +MP_DEFINE_CONST_DICT(dict, table); + +static const qstr attrtuple_fields[] = { + MP_QSTR_f0, +}; +MP_DEFINE_ATTRTUPLE(attrtuple, attrtuple_fields, 1, MP_ROM_PTR(&f0_obj)); + +void nlr_cb(void *); +void nlr_cb(void *){ +} + +// The MP_DEFINE_CONST_OBJ_TYPE macro is not C++-compatible because each of the +// MP_DEFINE_CONST_OBJ_TYPE_NARGS_X macros only initializes some of _mp_obj_type_t's +// .slot_index_xxx members but that cannot be fixed to be done in a deterministic way. + + #if defined(MICROPY_UNIX_COVERAGE) // Just to test building of C++ code. static mp_obj_t extra_cpp_coverage_impl() { + MP_DEFINE_NLR_JUMP_CALLBACK_FUNCTION_1(ctx, nlr_cb, (void *) nlr_cb); + + // To avoid 'error: unused variable [-Werror,-Wunused-const-variable]'. + (void) ctx; + (void) f0_obj; + (void) f1_obj; + (void) f2_obj; + (void) f3_obj; + (void) fvar_obj; + (void) fvarbetween_obj; + (void) fkw_obj; + (void) map; + (void) dict; + (void) attrtuple; return mp_const_none; } diff --git a/ports/unix/input.c b/ports/unix/input.c index 31926a5a8e1af..260e9eac8c9db 100644 --- a/ports/unix/input.c +++ b/ports/unix/input.c @@ -104,6 +104,9 @@ void prompt_write_history(void) { #if MICROPY_USE_READLINE == 1 char *home = getenv("HOME"); if (home != NULL) { + if (MP_STATE_THREAD(gc_lock_depth) != 0) { + return; + } vstr_t vstr; vstr_init(&vstr, 50); vstr_printf(&vstr, "%s/.micropython.history", home); diff --git a/ports/unix/main.c b/ports/unix/main.c index 259b183eb766d..e42400fe5f9a3 100644 --- a/ports/unix/main.c +++ b/ports/unix/main.c @@ -54,9 +54,11 @@ #include "extmod/vfs_posix.h" #include "genhdr/mpversion.h" #include "input.h" +#include "stack_size.h" +#include "shared/runtime/pyexec.h" // Command line options, with their defaults -static bool compile_only = false; +bool mp_compile_only = false; static uint emit_opt = MP_EMIT_OPT_NONE; #if MICROPY_ENABLE_GC @@ -90,7 +92,8 @@ static void stderr_print_strn(void *env, const char *str, size_t len) { const mp_print_t mp_stderr_print = {NULL, stderr_print_strn}; -#define FORCED_EXIT (0x100) +// CIRCUITPY-CHANGE: be consistent about using PYEXEC_FORCED_EXIT +// #define FORCED_EXIT (0x100) // If exc is SystemExit, return value where FORCED_EXIT bit set, // and lower 8 bits are SystemExit value. For all other exceptions, // return 1. @@ -103,7 +106,8 @@ static int handle_uncaught_exception(mp_obj_base_t *exc) { if (exit_val != mp_const_none && !mp_obj_get_int_maybe(exit_val, &val)) { val = 1; } - return FORCED_EXIT | (val & 255); + // CIRCUITPY-CHANGE: be consistent about using PYEXEC_FORCED_EXIT + return PYEXEC_FORCED_EXIT | (val & 255); } // Report all other exceptions @@ -112,8 +116,6 @@ static int handle_uncaught_exception(mp_obj_base_t *exc) { } #define LEX_SRC_STR (1) -#define LEX_SRC_VSTR (2) -#define LEX_SRC_FILENAME (3) #define LEX_SRC_STDIN (4) // Returns standard error codes: 0 for success, 1 for all other errors, @@ -129,19 +131,13 @@ static int execute_from_lexer(int source_kind, const void *source, mp_parse_inpu if (source_kind == LEX_SRC_STR) { const char *line = source; lex = mp_lexer_new_from_str_len(MP_QSTR__lt_stdin_gt_, line, strlen(line), false); - } else if (source_kind == LEX_SRC_VSTR) { - const vstr_t *vstr = source; - lex = mp_lexer_new_from_str_len(MP_QSTR__lt_stdin_gt_, vstr->buf, vstr->len, false); - } else if (source_kind == LEX_SRC_FILENAME) { - const char *filename = (const char *)source; - lex = mp_lexer_new_from_file(qstr_from_str(filename)); } else { // LEX_SRC_STDIN lex = mp_lexer_new_from_fd(MP_QSTR__lt_stdin_gt_, 0, false); } qstr source_name = lex->source_name; - #if MICROPY_PY___FILE__ + #if MICROPY_MODULE___FILE__ if (input_kind == MP_PARSE_FILE_INPUT) { mp_store_global(MP_QSTR___file__, MP_OBJ_NEW_QSTR(source_name)); } @@ -160,7 +156,7 @@ static int execute_from_lexer(int source_kind, const void *source, mp_parse_inpu mp_obj_t module_fun = mp_compile(&parse_tree, source_name, is_repl); - if (!compile_only) { + if (!mp_compile_only) { // execute it mp_call_function_0(module_fun); } @@ -197,92 +193,32 @@ static char *strjoin(const char *s1, int sep_char, const char *s2) { #endif static int do_repl(void) { - mp_hal_stdout_tx_str(MICROPY_BANNER_NAME_AND_VERSION); - mp_hal_stdout_tx_str("; " MICROPY_BANNER_MACHINE); - mp_hal_stdout_tx_str("\nUse Ctrl-D to exit, Ctrl-E for paste mode\n"); - #if MICROPY_USE_READLINE == 1 - // use MicroPython supplied readline + // use MicroPython supplied readline-based REPL - vstr_t line; - vstr_init(&line, 16); + int ret = 0; for (;;) { - mp_hal_stdio_mode_raw(); - - input_restart: - vstr_reset(&line); - int ret = readline(&line, mp_repl_get_ps1()); - mp_parse_input_kind_t parse_input_kind = MP_PARSE_SINGLE_INPUT; - - if (ret == CHAR_CTRL_C) { - // cancel input - mp_hal_stdout_tx_str("\r\n"); - goto input_restart; - } else if (ret == CHAR_CTRL_D) { - // EOF - printf("\n"); - mp_hal_stdio_mode_orig(); - vstr_clear(&line); - return 0; - } else if (ret == CHAR_CTRL_E) { - // paste mode - mp_hal_stdout_tx_str("\npaste mode; Ctrl-C to cancel, Ctrl-D to finish\n=== "); - vstr_reset(&line); - for (;;) { - char c = mp_hal_stdin_rx_chr(); - if (c == CHAR_CTRL_C) { - // cancel everything - mp_hal_stdout_tx_str("\n"); - goto input_restart; - } else if (c == CHAR_CTRL_D) { - // end of input - mp_hal_stdout_tx_str("\n"); - break; - } else { - // add char to buffer and echo - vstr_add_byte(&line, c); - if (c == '\r') { - mp_hal_stdout_tx_str("\n=== "); - } else { - mp_hal_stdout_tx_strn(&c, 1); - } - } - } - parse_input_kind = MP_PARSE_FILE_INPUT; - } else if (line.len == 0) { - if (ret != 0) { - printf("\n"); + if (pyexec_mode_kind == PYEXEC_MODE_RAW_REPL) { + if ((ret = pyexec_raw_repl()) != 0) { + break; } - goto input_restart; } else { - // got a line with non-zero length, see if it needs continuing - while (mp_repl_continue_with_input(vstr_null_terminated_str(&line))) { - vstr_add_byte(&line, '\n'); - ret = readline(&line, mp_repl_get_ps2()); - if (ret == CHAR_CTRL_C) { - // cancel everything - printf("\n"); - goto input_restart; - } else if (ret == CHAR_CTRL_D) { - // stop entering compound statement - break; - } + if ((ret = pyexec_friendly_repl()) != 0) { + break; } } - - mp_hal_stdio_mode_orig(); - - ret = execute_from_lexer(LEX_SRC_VSTR, &line, parse_input_kind, true); - if (ret & FORCED_EXIT) { - return ret; - } } + return ret; #else // use simple readline + mp_hal_stdout_tx_str(MICROPY_BANNER_NAME_AND_VERSION); + mp_hal_stdout_tx_str("; " MICROPY_BANNER_MACHINE); + mp_hal_stdout_tx_str("\nUse Ctrl-D to exit, Ctrl-E for paste mode\n"); + for (;;) { char *line = prompt((char *)mp_repl_get_ps1()); if (line == NULL) { @@ -302,7 +238,8 @@ static int do_repl(void) { int ret = execute_from_lexer(LEX_SRC_STR, line, MP_PARSE_SINGLE_INPUT, true); free(line); - if (ret & FORCED_EXIT) { + // CIRCUITPY-CHANGE: be consistent about using PYEXEC_FORCED_EXIT + if (ret & PYEXEC_FORCED_EXIT) { return ret; } } @@ -310,12 +247,44 @@ static int do_repl(void) { #endif } +static inline int convert_pyexec_result(int ret) { + #if MICROPY_PYEXEC_ENABLE_EXIT_CODE_HANDLING + // With exit code handling enabled: + // pyexec returns exit code with PYEXEC_FORCED_EXIT flag set for SystemExit + // Unix port expects: 0 for success, non-zero for error/exit + if (ret & PYEXEC_FORCED_EXIT) { + // SystemExit: extract exit code from lower bits + return ret & 0xFF; + } + // Normal execution or exception: return as-is (0 for success, 1 for exception) + return ret; + #else + // pyexec returns 1 for success, 0 for exception, PYEXEC_FORCED_EXIT for SystemExit + // Convert to unix port's expected codes: 0 for success, 1 for exception, FORCED_EXIT|val for SystemExit + if (ret == 1) { + return 0; // success + } else if (ret & PYEXEC_FORCED_EXIT) { + return ret; // SystemExit with exit value in lower 8 bits + } else { + return 1; // exception + } + #endif +} + static int do_file(const char *file) { - return execute_from_lexer(LEX_SRC_FILENAME, file, MP_PARSE_FILE_INPUT, false); + // CIRCUITPY-CHANGE: pyexec_file result arg + pyexec_result_t pyexec_result; + return convert_pyexec_result(pyexec_file(file, &pyexec_result)); } static int do_str(const char *str) { - return execute_from_lexer(LEX_SRC_STR, str, MP_PARSE_FILE_INPUT, false); + vstr_t vstr; + vstr.buf = (char *)str; + vstr.len = strlen(str); + // CIRCUITPY-CHANGE: pyexec_vstr result arg + pyexec_result_t pyexec_result; + int ret = pyexec_vstr(&vstr, true, &pyexec_result); + return convert_pyexec_result(ret); } static void print_help(char **argv) { @@ -384,7 +353,7 @@ static void pre_process_options(int argc, char **argv) { } if (0) { } else if (strcmp(argv[a + 1], "compile-only") == 0) { - compile_only = true; + mp_compile_only = true; } else if (strcmp(argv[a + 1], "emit=bytecode") == 0) { emit_opt = MP_EMIT_OPT_BYTECODE; #if MICROPY_EMIT_NATIVE @@ -456,10 +425,13 @@ static void set_sys_argv(char *argv[], int argc, int start_arg) { #if MICROPY_PY_SYS_EXECUTABLE extern mp_obj_str_t mp_sys_executable_obj; -static char executable_path[MICROPY_ALLOC_PATH_MAX]; +static char *executable_path = NULL; static void sys_set_excecutable(char *argv0) { - if (realpath(argv0, executable_path)) { + if (executable_path == NULL) { + executable_path = realpath(argv0, NULL); + } + if (executable_path != NULL) { mp_obj_str_set_data(&mp_sys_executable_obj, (byte *)executable_path, strlen(executable_path)); } } @@ -479,11 +451,7 @@ int main(int argc, char **argv) { #endif // Define a reasonable stack limit to detect stack overflow. - mp_uint_t stack_size = 40000 * (sizeof(void *) / 4); - #if defined(__arm__) && !defined(__thumb2__) - // ARM (non-Thumb) architectures require more stack. - stack_size *= 2; - #endif + mp_uint_t stack_size = 40000 * UNIX_STACK_MULTIPLIER; // We should capture stack top ASAP after start, and it should be // captured guaranteedly before any other stack variables are allocated. @@ -619,19 +587,6 @@ MP_NOINLINE int main_(int argc, char **argv) { } #endif - // Here is some example code to create a class and instance of that class. - // First is the Python, then the C code. - // - // class TestClass: - // pass - // test_obj = TestClass() - // test_obj.attr = 42 - // - // mp_obj_t test_class_type, test_class_instance; - // test_class_type = mp_obj_new_type(qstr_from_str("TestClass"), mp_const_empty_tuple, mp_obj_new_dict(0)); - // mp_store_name(qstr_from_str("test_obj"), test_class_instance = mp_call_function_0(test_class_type)); - // mp_store_attr(test_class_instance, qstr_from_str("attr"), mp_obj_new_int(42)); - /* printf("bytes:\n"); printf(" total %d\n", m_get_total_bytes_allocated()); @@ -685,12 +640,18 @@ MP_NOINLINE int main_(int argc, char **argv) { subpkg_tried = false; reimport: + mp_hal_set_interrupt_char(CHAR_CTRL_C); if (nlr_push(&nlr) == 0) { mod = mp_builtin___import__(MP_ARRAY_SIZE(import_args), import_args); + mp_hal_set_interrupt_char(-1); + mp_handle_pending(true); nlr_pop(); } else { // uncaught exception - return handle_uncaught_exception(nlr.ret_val) & 0xff; + mp_hal_set_interrupt_char(-1); + mp_handle_pending(false); + ret = handle_uncaught_exception(nlr.ret_val) & 0xff; + break; } // If this module is a package, see if it has a `__main__.py`. @@ -727,11 +688,9 @@ MP_NOINLINE int main_(int argc, char **argv) { return invalid_args(); } } else { - char *pathbuf = malloc(PATH_MAX); - char *basedir = realpath(argv[a], pathbuf); + char *basedir = realpath(argv[a], NULL); if (basedir == NULL) { mp_printf(&mp_stderr_print, "%s: can't open file '%s': [Errno %d] %s\n", argv[0], argv[a], errno, strerror(errno)); - free(pathbuf); // CPython exits with 2 in such case ret = 2; break; @@ -740,7 +699,7 @@ MP_NOINLINE int main_(int argc, char **argv) { // Set base dir of the script as first entry in sys.path. char *p = strrchr(basedir, '/'); mp_obj_list_store(mp_sys_path, MP_OBJ_NEW_SMALL_INT(0), mp_obj_new_str_via_qstr(basedir, p - basedir)); - free(pathbuf); + free(basedir); set_sys_argv(argv, argc, a); ret = do_file(argv[a]); @@ -780,7 +739,7 @@ MP_NOINLINE int main_(int argc, char **argv) { #endif #if MICROPY_PY_BLUETOOTH - void mp_bluetooth_deinit(void); + int mp_bluetooth_deinit(void); mp_bluetooth_deinit(); #endif @@ -806,8 +765,20 @@ MP_NOINLINE int main_(int argc, char **argv) { #endif #endif + #if MICROPY_PY_SYS_EXECUTABLE && !defined(NDEBUG) + // Again, make memory leak detector happy + free(executable_path); + #endif + // printf("total bytes = %d\n", m_get_total_bytes_allocated()); - return ret & 0xff; + + // CIRCUITPY-CHANGE: handle PYEXEC_EXCEPTION + if (ret & PYEXEC_EXCEPTION) { + // Return exit status code 1 so the invoker knows there was an uncaught exception. + return 1; + } else { + return ret & 0xff; + } } void nlr_jump_fail(void *val) { diff --git a/ports/unix/modtime.c b/ports/unix/modtime.c index fbd94b5ecd129..4f0550dbea765 100644 --- a/ports/unix/modtime.c +++ b/ports/unix/modtime.c @@ -34,6 +34,7 @@ #include "py/mphal.h" #include "py/runtime.h" +#include "shared/timeutils/timeutils.h" #ifdef _WIN32 static inline int msec_sleep_tv(struct timeval *tv) { @@ -130,12 +131,7 @@ static mp_obj_t mod_time_gm_local_time(size_t n_args, const mp_obj_t *args, stru if (n_args == 0) { t = time(NULL); } else { - #if MICROPY_PY_BUILTINS_FLOAT && MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_DOUBLE - mp_float_t val = mp_obj_get_float(args[0]); - t = (time_t)MICROPY_FLOAT_C_FUN(trunc)(val); - #else - t = mp_obj_get_int(args[0]); - #endif + t = (time_t)timeutils_obj_get_timestamp(args[0]); } struct tm *tm = time_func(&t); @@ -193,10 +189,10 @@ static mp_obj_t mod_time_mktime(mp_obj_t tuple) { time.tm_isdst = -1; // auto-detect } time_t ret = mktime(&time); - if (ret == -1) { + if (ret == (time_t)-1) { mp_raise_msg(&mp_type_OverflowError, MP_ERROR_TEXT("invalid mktime usage")); } - return mp_obj_new_int(ret); + return timeutils_obj_from_timestamp(ret); } MP_DEFINE_CONST_FUN_OBJ_1(mod_time_mktime_obj, mod_time_mktime); diff --git a/ports/unix/mpconfigport.h b/ports/unix/mpconfigport.h index 4d9fe9f1dc4a2..815be76b4e90c 100644 --- a/ports/unix/mpconfigport.h +++ b/ports/unix/mpconfigport.h @@ -32,6 +32,9 @@ // features to work on Unix-like systems, see mpconfigvariant.h (and // mpconfigvariant_common.h) for feature enabling. +// For time_t, needed by MICROPY_TIMESTAMP_IMPL_TIME_T. +#include + // For size_t and ssize_t #include @@ -41,6 +44,7 @@ // CIRCUITPY-CHANGE #define CIRCUITPY_MICROPYTHON_ADVANCED (1) #define MICROPY_PY_ASYNC_AWAIT (1) +#define MICROPY_PY_DOUBLE_TYPECODE (1) #define MICROPY_PY_UCTYPES (0) #ifndef MICROPY_CONFIG_ROM_LEVEL @@ -78,21 +82,6 @@ #define MICROPY_EMIT_ARM (1) #endif -// Type definitions for the specific machine based on the word size. -#ifndef MICROPY_OBJ_REPR -#ifdef __LP64__ -typedef long mp_int_t; // must be pointer size -typedef unsigned long mp_uint_t; // must be pointer size -#else -// These are definitions for machines where sizeof(int) == sizeof(void*), -// regardless of actual size. -typedef int mp_int_t; // must be pointer size -typedef unsigned int mp_uint_t; // must be pointer size -#endif -#else -// Assume that if we already defined the obj repr then we also defined types. -#endif - // Cannot include , as it may lead to symbol name clashes #if _FILE_OFFSET_BITS == 64 && !defined(__LP64__) typedef long long mp_off_t; @@ -135,6 +124,9 @@ typedef long mp_off_t; // VFS stat functions should return time values relative to 1970/1/1 #define MICROPY_EPOCH_IS_1970 (1) +// port modtime functions use time_t +#define MICROPY_TIMESTAMP_IMPL (MICROPY_TIMESTAMP_IMPL_TIME_T) + // Assume that select() call, interrupted with a signal, and erroring // with EINTR, updates remaining timeout value. #define MICROPY_SELECT_REMAINING_TIME (1) @@ -145,6 +137,9 @@ typedef long mp_off_t; #define MICROPY_STACKLESS_STRICT (0) #endif +// Recursive mutex is needed when threading is enabled, regardless of GIL setting. +#define MICROPY_PY_THREAD_RECURSIVE_MUTEX (MICROPY_PY_THREAD) + // Implementation of the machine module. #define MICROPY_PY_MACHINE_INCLUDEFILE "ports/unix/modmachine.c" @@ -172,6 +167,12 @@ typedef long mp_off_t; // Enable sys.executable. #define MICROPY_PY_SYS_EXECUTABLE (1) +// Enable support for compile-only mode. +#define MICROPY_PYEXEC_COMPILE_ONLY (1) + +// Enable handling of sys.exit() exit codes. +#define MICROPY_PYEXEC_ENABLE_EXIT_CODE_HANDLING (1) + #define MICROPY_PY_SOCKET_LISTEN_BACKLOG_DEFAULT (SOMAXCONN < 128 ? SOMAXCONN : 128) // Bare-metal ports don't have stderr. Printing debug to stderr may give tests diff --git a/ports/unix/mpconfigport.mk b/ports/unix/mpconfigport.mk index 26c04faf4c5ca..c836663cd2339 100644 --- a/ports/unix/mpconfigport.mk +++ b/ports/unix/mpconfigport.mk @@ -14,6 +14,7 @@ MICROPY_PY_BTREE = 0 # _thread module using pthreads MICROPY_PY_THREAD = 1 +MICROPY_PY_THREAD_GIL = 0 # Subset of CPython termios module MICROPY_PY_TERMIOS = 1 diff --git a/ports/unix/mphalport.h b/ports/unix/mphalport.h index 7f71217632a8e..d9cd05b3de82b 100644 --- a/ports/unix/mphalport.h +++ b/ports/unix/mphalport.h @@ -25,7 +25,6 @@ */ #include #include -// CIRCUITPY-CHANGE: extra include #include #ifndef CHAR_CTRL_C @@ -38,6 +37,20 @@ #define MICROPY_END_ATOMIC_SECTION(x) (void)x; mp_thread_unix_end_atomic_section() #endif +// In lieu of a WFI(), slow down polling from being a tight loop. +// +// Note that we don't delay for the full TIMEOUT_MS, as execution +// can't be woken from the delay. +#define MICROPY_INTERNAL_WFE(TIMEOUT_MS) \ + do { \ + MP_THREAD_GIL_EXIT(); \ + mp_hal_delay_us(500); \ + MP_THREAD_GIL_ENTER(); \ + } while (0) + +// The port provides `mp_hal_stdio_mode_raw()` and `mp_hal_stdio_mode_orig()`. +#define MICROPY_HAL_HAS_STDIO_MODE_SWITCH (1) + // CIRCUITPY-CHANGE: mp_hal_set_interrupt_char(int) instead of char void mp_hal_set_interrupt_char(int c); bool mp_hal_is_interrupted(void); @@ -110,3 +123,6 @@ enum { void mp_hal_get_mac(int idx, uint8_t buf[6]); #endif + +// Global variable to control compile-only mode. +extern bool mp_compile_only; diff --git a/ports/unix/mpthreadport.c b/ports/unix/mpthreadport.c index 5172645bc147a..a41b3ec9f4701 100644 --- a/ports/unix/mpthreadport.c +++ b/ports/unix/mpthreadport.c @@ -31,6 +31,7 @@ #include "py/runtime.h" #include "py/mpthread.h" #include "py/gc.h" +#include "stack_size.h" #if MICROPY_PY_THREAD @@ -45,8 +46,14 @@ // potential conflict with other uses of the more commonly used SIGUSR1. #ifdef SIGRTMIN #define MP_THREAD_GC_SIGNAL (SIGRTMIN + 5) +#ifdef __ANDROID__ +#define MP_THREAD_TERMINATE_SIGNAL (SIGRTMIN + 6) +#endif #else #define MP_THREAD_GC_SIGNAL (SIGUSR1) +#ifdef __ANDROID__ +#define MP_THREAD_TERMINATE_SIGNAL (SIGUSR2) +#endif #endif // This value seems to be about right for both 32-bit and 64-bit builds. @@ -107,6 +114,18 @@ static void mp_thread_gc(int signo, siginfo_t *info, void *context) { } } +// On Android, pthread_cancel and pthread_setcanceltype are not implemented. +// To achieve that result a new signal handler responding on either +// (SIGRTMIN + 6) or SIGUSR2 is installed on every child thread. The sole +// purpose of this new signal handler is to terminate the thread in a safe +// asynchronous manner. + +#ifdef __ANDROID__ +static void mp_thread_terminate(int signo, siginfo_t *info, void *context) { + pthread_exit(NULL); +} +#endif + void mp_thread_init(void) { pthread_key_create(&tls_key, NULL); pthread_setspecific(tls_key, &mp_state_ctx.thread); @@ -135,6 +154,14 @@ void mp_thread_init(void) { sa.sa_sigaction = mp_thread_gc; sigemptyset(&sa.sa_mask); sigaction(MP_THREAD_GC_SIGNAL, &sa, NULL); + + // Install a signal handler for asynchronous termination if needed. + #if defined(__ANDROID__) + sa.sa_flags = SA_SIGINFO; + sa.sa_sigaction = mp_thread_terminate; + sigemptyset(&sa.sa_mask); + sigaction(MP_THREAD_TERMINATE_SIGNAL, &sa, NULL); + #endif } void mp_thread_deinit(void) { @@ -142,7 +169,11 @@ void mp_thread_deinit(void) { while (thread->next != NULL) { mp_thread_t *th = thread; thread = thread->next; + #if defined(__ANDROID__) + pthread_kill(th->id, MP_THREAD_TERMINATE_SIGNAL); + #else pthread_cancel(th->id); + #endif free(th); } mp_thread_unix_end_atomic_section(); @@ -200,7 +231,9 @@ void mp_thread_start(void) { } #endif + #if !defined(__ANDROID__) pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); + #endif mp_thread_unix_begin_atomic_section(); for (mp_thread_t *th = thread; th != NULL; th = th->next) { if (th->id == pthread_self()) { @@ -212,14 +245,14 @@ void mp_thread_start(void) { } mp_uint_t mp_thread_create(void *(*entry)(void *), void *arg, size_t *stack_size) { - // default stack size is 8k machine-words + // default stack size if (*stack_size == 0) { - *stack_size = 8192 * sizeof(void *); + *stack_size = 32768 * UNIX_STACK_MULTIPLIER; } // minimum stack size is set by pthreads - if (*stack_size < PTHREAD_STACK_MIN) { - *stack_size = PTHREAD_STACK_MIN; + if (*stack_size < (size_t)PTHREAD_STACK_MIN) { + *stack_size = (size_t)PTHREAD_STACK_MIN; } // ensure there is enough stack to include a stack-overflow margin diff --git a/ports/unix/stack_size.h b/ports/unix/stack_size.h new file mode 100644 index 0000000000000..f6159bb69d529 --- /dev/null +++ b/ports/unix/stack_size.h @@ -0,0 +1,54 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2025 Angus Gratton + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#ifndef MICROPY_INCLUDED_UNIX_STACK_SIZE_H +#define MICROPY_INCLUDED_UNIX_STACK_SIZE_H + +#include "py/misc.h" + +// Define scaling factors for the stack size (also applies to main thread) +#ifndef UNIX_STACK_MULTIPLIER + +#if defined(__arm__) && !defined(__thumb2__) +// ARM (non-Thumb) architectures require more stack. +#define UNIX_STACK_MUL_ARM 2 +#else +#define UNIX_STACK_MUL_ARM 1 +#endif + +#if MP_SANITIZER_BUILD +// Sanitizer features consume significant stack in some cases +// This multiplier can probably be removed when using GCC 12 or newer. +#define UNIX_STACK_MUL_SANITIZERS 4 +#else +#define UNIX_STACK_MUL_SANITIZERS 1 +#endif + +// Double the stack size for 64-bit builds, plus additional scaling +#define UNIX_STACK_MULTIPLIER ((sizeof(void *) / 4) * UNIX_STACK_MUL_ARM * UNIX_STACK_MUL_SANITIZERS) + +#endif // UNIX_STACK_MULTIPLIER + +#endif // MICROPY_INCLUDED_UNIX_STACK_SIZE_H diff --git a/ports/unix/variants/coverage/mpconfigvariant.h b/ports/unix/variants/coverage/mpconfigvariant.h index ca79d3d0d2b61..03f77ec14c6db 100644 --- a/ports/unix/variants/coverage/mpconfigvariant.h +++ b/ports/unix/variants/coverage/mpconfigvariant.h @@ -39,11 +39,22 @@ // Enable additional features. #define MICROPY_DEBUG_PARSE_RULE_NAME (1) +// CIRCUITPY-CHANGE: off +#define MICROPY_PY_SYS_SETTRACE (0) #define MICROPY_TRACKED_ALLOC (1) #define MICROPY_WARNINGS_CATEGORY (1) #undef MICROPY_VFS_ROM_IOCTL #define MICROPY_VFS_ROM_IOCTL (1) #define MICROPY_PY_CRYPTOLIB_CTR (1) +// CIRCUITPY-CHANGE: off +#define MICROPY_SCHEDULER_STATIC_NODES (0) + +// Enable os.uname for attrtuple coverage test +#define MICROPY_PY_OS_UNAME (1) +#define MICROPY_HW_BOARD_NAME "a machine" +#define MICROPY_HW_MCU_NAME MICROPY_PY_SYS_PLATFORM +// Keep the standard banner message +#define MICROPY_BANNER_MACHINE MICROPY_PY_SYS_PLATFORM " [" MICROPY_PLATFORM_COMPILER "] version" // CIRCUITPY-CHANGE: Disable things never used in circuitpython #define MICROPY_PY_CRYPTOLIB (0) diff --git a/ports/unix/variants/longlong/mpconfigvariant.h b/ports/unix/variants/longlong/mpconfigvariant.h new file mode 100644 index 0000000000000..d50d360b1fe5c --- /dev/null +++ b/ports/unix/variants/longlong/mpconfigvariant.h @@ -0,0 +1,44 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2016 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +// This config exists to test that the MICROPY_LONGINT_IMPL_LONGLONG variant +// (i.e. minimal form of "big integer" that's backed by 64-bit int only) builds +// and passes tests. + +#define MICROPY_LONGINT_IMPL (MICROPY_LONGINT_IMPL_LONGLONG) + +// We build it on top of REPR C, which uses memory-efficient floating point +// objects encoded directly mp_obj_t (30 bits only). +// Therefore this variant should be built using MICROPY_FORCE_32BIT=1 + +#define MICROPY_OBJ_REPR (MICROPY_OBJ_REPR_C) +#define MICROPY_FLOAT_IMPL (MICROPY_FLOAT_IMPL_FLOAT) + +// Set base feature level. +#define MICROPY_CONFIG_ROM_LEVEL (MICROPY_CONFIG_ROM_LEVEL_EXTRA_FEATURES) + +// Enable extra Unix features. +#include "../mpconfigvariant_common.h" diff --git a/ports/unix/variants/longlong/mpconfigvariant.mk b/ports/unix/variants/longlong/mpconfigvariant.mk new file mode 100644 index 0000000000000..2d2c3706469fb --- /dev/null +++ b/ports/unix/variants/longlong/mpconfigvariant.mk @@ -0,0 +1,8 @@ +# build interpreter with "bigints" implemented as "longlong" + +# otherwise, small int is essentially 64-bit +MICROPY_FORCE_32BIT := 1 + +MICROPY_PY_FFI := 0 + +MPY_TOOL_FLAGS = -mlongint-impl longlong diff --git a/ports/unix/variants/minimal/mpconfigvariant.h b/ports/unix/variants/minimal/mpconfigvariant.h index 97ed786b8f409..0e280a8f73058 100644 --- a/ports/unix/variants/minimal/mpconfigvariant.h +++ b/ports/unix/variants/minimal/mpconfigvariant.h @@ -49,6 +49,7 @@ #define MICROPY_COMP_DOUBLE_TUPLE_ASSIGN (1) #define MICROPY_ENABLE_COMPILER (1) #define MICROPY_ENABLE_EXTERNAL_IMPORT (1) +#define MICROPY_STACK_CHECK (1) #define MICROPY_FULL_CHECKS (1) #define MICROPY_HELPER_REPL (1) #define MICROPY_KBD_EXCEPTION (1) @@ -61,6 +62,5 @@ #define MICROPY_PY_BUILTINS_RANGE_ATTRS (1) #define MICROPY_PY_GENERATOR_PEND_THROW (1) -// Enable just the sys and os built-in modules. -#define MICROPY_PY_SYS (1) +// Add just the os built-in module. #define MICROPY_PY_OS (1) diff --git a/ports/unix/variants/mpconfigvariant_common.h b/ports/unix/variants/mpconfigvariant_common.h index 9eeed8797366c..1ac59c95572dd 100644 --- a/ports/unix/variants/mpconfigvariant_common.h +++ b/ports/unix/variants/mpconfigvariant_common.h @@ -29,7 +29,7 @@ // Send raise KeyboardInterrupt directly from the signal handler rather than // scheduling it into the VM. -#define MICROPY_ASYNC_KBD_INTR (1) +#define MICROPY_ASYNC_KBD_INTR (!MICROPY_PY_THREAD_GIL) // Enable helpers for printing debugging information. #ifndef MICROPY_DEBUG_PRINTERS @@ -104,12 +104,6 @@ #define MICROPY_PY_TIME_CUSTOM_SLEEP (1) #define MICROPY_PY_TIME_INCLUDEFILE "ports/unix/modtime.c" -#if MICROPY_PY_SSL -#define MICROPY_PY_HASHLIB_MD5 (1) -#define MICROPY_PY_HASHLIB_SHA1 (1) -#define MICROPY_PY_CRYPTOLIB (1) -#endif - // The "select" module is enabled by default, but disable select.select(). #define MICROPY_PY_SELECT_POSIX_OPTIMISATIONS (1) #define MICROPY_PY_SELECT_SELECT (0) diff --git a/ports/unix/variants/nanbox/mpconfigvariant.h b/ports/unix/variants/nanbox/mpconfigvariant.h index 7b13b7dc6ce4e..c9e9c1f63d95d 100644 --- a/ports/unix/variants/nanbox/mpconfigvariant.h +++ b/ports/unix/variants/nanbox/mpconfigvariant.h @@ -41,10 +41,3 @@ #define MICROPY_EMIT_X64 (0) #define MICROPY_EMIT_THUMB (0) #define MICROPY_EMIT_ARM (0) - -#include - -typedef int64_t mp_int_t; -typedef uint64_t mp_uint_t; -#define UINT_FMT "%llu" -#define INT_FMT "%lld" diff --git a/py/argcheck.c b/py/argcheck.c index 9302dec96c396..155e29bdc15bf 100644 --- a/py/argcheck.c +++ b/py/argcheck.c @@ -170,13 +170,13 @@ void mp_arg_parse_all_kw_array(size_t n_pos, size_t n_kw, const mp_obj_t *args, } #if MICROPY_ERROR_REPORTING <= MICROPY_ERROR_REPORTING_TERSE -NORETURN void mp_arg_error_terse_mismatch(void) { +MP_NORETURN void mp_arg_error_terse_mismatch(void) { mp_raise_TypeError(MP_ERROR_TEXT("argument num/types mismatch")); } #endif #if MICROPY_CPYTHON_COMPAT -NORETURN void mp_arg_error_unimpl_kw(void) { +MP_NORETURN void mp_arg_error_unimpl_kw(void) { mp_raise_NotImplementedError(MP_ERROR_TEXT("keyword argument(s) not implemented - use normal args instead")); } #endif @@ -312,6 +312,6 @@ mp_int_t mp_arg_validate_type_int(mp_obj_t obj, qstr arg_name) { return an_int; } -NORETURN void mp_arg_error_invalid(qstr arg_name) { +MP_NORETURN void mp_arg_error_invalid(qstr arg_name) { mp_raise_ValueError_varg(MP_ERROR_TEXT("Invalid %q"), arg_name); } diff --git a/py/asmarm.c b/py/asmarm.c index 6fa751b32eb7c..15bc73b61eca6 100644 --- a/py/asmarm.c +++ b/py/asmarm.c @@ -36,7 +36,7 @@ #include "py/asmarm.h" -#define SIGNED_FIT24(x) (((x) & 0xff800000) == 0) || (((x) & 0xff000000) == 0xff000000) +#define REG_TEMP ASM_ARM_REG_R8 // Insert word into instruction flow static void emit(asm_arm_t *as, uint op) { @@ -171,8 +171,8 @@ void asm_arm_entry(asm_arm_t *as, int num_locals) { if (as->stack_adjust < 0x100) { emit_al(as, asm_arm_op_sub_imm(ASM_ARM_REG_SP, ASM_ARM_REG_SP, as->stack_adjust)); } else { - asm_arm_mov_reg_i32_optimised(as, ASM_ARM_REG_R8, as->stack_adjust); - emit_al(as, asm_arm_op_sub_reg(ASM_ARM_REG_SP, ASM_ARM_REG_SP, ASM_ARM_REG_R8)); + asm_arm_mov_reg_i32_optimised(as, REG_TEMP, as->stack_adjust); + emit_al(as, asm_arm_op_sub_reg(ASM_ARM_REG_SP, ASM_ARM_REG_SP, REG_TEMP)); } } } @@ -182,8 +182,8 @@ void asm_arm_exit(asm_arm_t *as) { if (as->stack_adjust < 0x100) { emit_al(as, asm_arm_op_add_imm(ASM_ARM_REG_SP, ASM_ARM_REG_SP, as->stack_adjust)); } else { - asm_arm_mov_reg_i32_optimised(as, ASM_ARM_REG_R8, as->stack_adjust); - emit_al(as, asm_arm_op_add_reg(ASM_ARM_REG_SP, ASM_ARM_REG_SP, ASM_ARM_REG_R8)); + asm_arm_mov_reg_i32_optimised(as, REG_TEMP, as->stack_adjust); + emit_al(as, asm_arm_op_add_reg(ASM_ARM_REG_SP, ASM_ARM_REG_SP, REG_TEMP)); } } @@ -293,10 +293,10 @@ void asm_arm_orr_reg_reg_reg(asm_arm_t *as, uint rd, uint rn, uint rm) { void asm_arm_mov_reg_local_addr(asm_arm_t *as, uint rd, int local_num) { if (local_num >= 0x40) { - // mov r8, #local_num*4 - // add rd, sp, r8 - asm_arm_mov_reg_i32_optimised(as, ASM_ARM_REG_R8, local_num << 2); - emit_al(as, asm_arm_op_add_reg(rd, ASM_ARM_REG_SP, ASM_ARM_REG_R8)); + // mov temp, #local_num*4 + // add rd, sp, temp + asm_arm_mov_reg_i32_optimised(as, REG_TEMP, local_num << 2); + emit_al(as, asm_arm_op_add_reg(rd, ASM_ARM_REG_SP, REG_TEMP)); } else { // add rd, sp, #local_num*4 emit_al(as, asm_arm_op_add_imm(rd, ASM_ARM_REG_SP, local_num << 2)); @@ -333,14 +333,22 @@ void asm_arm_asr_reg_reg(asm_arm_t *as, uint rd, uint rs) { emit_al(as, 0x1a00050 | (rd << 12) | (rs << 8) | rd); } -void asm_arm_ldr_reg_reg(asm_arm_t *as, uint rd, uint rn, uint byte_offset) { - // ldr rd, [rn, #off] - emit_al(as, 0x5900000 | (rn << 16) | (rd << 12) | byte_offset); +void asm_arm_ldr_reg_reg_offset(asm_arm_t *as, uint rd, uint rn, uint byte_offset) { + if (byte_offset < 0x1000) { + // ldr rd, [rn, #off] + emit_al(as, 0x5900000 | (rn << 16) | (rd << 12) | byte_offset); + } else { + // mov temp, #off + // ldr rd, [rn, temp] + asm_arm_mov_reg_i32_optimised(as, REG_TEMP, byte_offset); + emit_al(as, 0x7900000 | (rn << 16) | (rd << 12) | REG_TEMP); + } } -void asm_arm_ldrh_reg_reg(asm_arm_t *as, uint rd, uint rn) { - // ldrh rd, [rn] - emit_al(as, 0x1d000b0 | (rn << 16) | (rd << 12)); +void asm_arm_ldrh_reg_reg_reg(asm_arm_t *as, uint rd, uint rm, uint rn) { + // ldrh doesn't support scaled register index + emit_al(as, 0x1a00080 | (REG_TEMP << 12) | rn); // mov temp, rn, lsl #1 + emit_al(as, 0x19000b0 | (rm << 16) | (rd << 12) | REG_TEMP); // ldrh rd, [rm, temp]; } void asm_arm_ldrh_reg_reg_offset(asm_arm_t *as, uint rd, uint rn, uint byte_offset) { @@ -348,31 +356,69 @@ void asm_arm_ldrh_reg_reg_offset(asm_arm_t *as, uint rd, uint rn, uint byte_offs // ldrh rd, [rn, #off] emit_al(as, 0x1d000b0 | (rn << 16) | (rd << 12) | ((byte_offset & 0xf0) << 4) | (byte_offset & 0xf)); } else { - // mov r8, #off - // ldrh rd, [rn, r8] - asm_arm_mov_reg_i32_optimised(as, ASM_ARM_REG_R8, byte_offset); - emit_al(as, 0x19000b0 | (rn << 16) | (rd << 12) | ASM_ARM_REG_R8); + // mov temp, #off + // ldrh rd, [rn, temp] + asm_arm_mov_reg_i32_optimised(as, REG_TEMP, byte_offset); + emit_al(as, 0x19000b0 | (rn << 16) | (rd << 12) | REG_TEMP); } } -void asm_arm_ldrb_reg_reg(asm_arm_t *as, uint rd, uint rn) { - // ldrb rd, [rn] - emit_al(as, 0x5d00000 | (rn << 16) | (rd << 12)); +void asm_arm_ldrb_reg_reg_reg(asm_arm_t *as, uint rd, uint rm, uint rn) { + // ldrb rd, [rm, rn] + emit_al(as, 0x7d00000 | (rm << 16) | (rd << 12) | rn); +} + +void asm_arm_ldrb_reg_reg_offset(asm_arm_t *as, uint rd, uint rn, uint byte_offset) { + if (byte_offset < 0x1000) { + // ldrb rd, [rn, #off] + emit_al(as, 0x5d00000 | (rn << 16) | (rd << 12) | byte_offset); + } else { + // mov temp, #off + // ldrb rd, [rn, temp] + asm_arm_mov_reg_i32_optimised(as, REG_TEMP, byte_offset); + emit_al(as, 0x7d00000 | (rn << 16) | (rd << 12) | REG_TEMP); + } } -void asm_arm_str_reg_reg(asm_arm_t *as, uint rd, uint rm, uint byte_offset) { - // str rd, [rm, #off] - emit_al(as, 0x5800000 | (rm << 16) | (rd << 12) | byte_offset); +void asm_arm_ldr_reg_reg_reg(asm_arm_t *as, uint rd, uint rm, uint rn) { + // ldr rd, [rm, rn, lsl #2] + emit_al(as, 0x7900100 | (rm << 16) | (rd << 12) | rn); } -void asm_arm_strh_reg_reg(asm_arm_t *as, uint rd, uint rm) { - // strh rd, [rm] - emit_al(as, 0x1c000b0 | (rm << 16) | (rd << 12)); +void asm_arm_str_reg_reg_offset(asm_arm_t *as, uint rd, uint rm, uint byte_offset) { + if (byte_offset < 0x1000) { + // str rd, [rm, #off] + emit_al(as, 0x5800000 | (rm << 16) | (rd << 12) | byte_offset); + } else { + // mov temp, #off + // str rd, [rm, temp] + asm_arm_mov_reg_i32_optimised(as, REG_TEMP, byte_offset); + emit_al(as, 0x7800000 | (rm << 16) | (rd << 12) | REG_TEMP); + } } -void asm_arm_strb_reg_reg(asm_arm_t *as, uint rd, uint rm) { - // strb rd, [rm] - emit_al(as, 0x5c00000 | (rm << 16) | (rd << 12)); +void asm_arm_strh_reg_reg_offset(asm_arm_t *as, uint rd, uint rn, uint byte_offset) { + if (byte_offset < 0x100) { + // strh rd, [rn, #off] + emit_al(as, 0x1c000b0 | (rn << 16) | (rd << 12) | ((byte_offset & 0xf0) << 4) | (byte_offset & 0xf)); + } else { + // mov temp, #off + // strh rd, [rn, temp] + asm_arm_mov_reg_i32_optimised(as, REG_TEMP, byte_offset); + emit_al(as, 0x18000b0 | (rn << 16) | (rd << 12) | REG_TEMP); + } +} + +void asm_arm_strb_reg_reg_offset(asm_arm_t *as, uint rd, uint rm, uint byte_offset) { + if (byte_offset < 0x1000) { + // strb rd, [rm, #off] + emit_al(as, 0x5c00000 | (rm << 16) | (rd << 12) | byte_offset); + } else { + // mov temp, #off + // strb rd, [rm, temp] + asm_arm_mov_reg_i32_optimised(as, REG_TEMP, byte_offset); + emit_al(as, 0x7c00000 | (rm << 16) | (rd << 12) | REG_TEMP); + } } void asm_arm_str_reg_reg_reg(asm_arm_t *as, uint rd, uint rm, uint rn) { @@ -382,8 +428,8 @@ void asm_arm_str_reg_reg_reg(asm_arm_t *as, uint rd, uint rm, uint rn) { void asm_arm_strh_reg_reg_reg(asm_arm_t *as, uint rd, uint rm, uint rn) { // strh doesn't support scaled register index - emit_al(as, 0x1a00080 | (ASM_ARM_REG_R8 << 12) | rn); // mov r8, rn, lsl #1 - emit_al(as, 0x18000b0 | (rm << 16) | (rd << 12) | ASM_ARM_REG_R8); // strh rd, [rm, r8] + emit_al(as, 0x1a00080 | (REG_TEMP << 12) | rn); // mov temp, rn, lsl #1 + emit_al(as, 0x18000b0 | (rm << 16) | (rd << 12) | REG_TEMP); // strh rd, [rm, temp] } void asm_arm_strb_reg_reg_reg(asm_arm_t *as, uint rd, uint rm, uint rn) { @@ -398,7 +444,7 @@ void asm_arm_bcc_label(asm_arm_t *as, int cond, uint label) { rel -= 8; // account for instruction prefetch, PC is 8 bytes ahead of this instruction rel >>= 2; // in ARM mode the branch target is 32-bit aligned, so the 2 LSB are omitted - if (SIGNED_FIT24(rel)) { + if (MP_FIT_SIGNED(24, rel)) { emit(as, cond | 0xa000000 | (rel & 0xffffff)); } else { printf("asm_arm_bcc: branch does not fit in 24 bits\n"); diff --git a/py/asmarm.h b/py/asmarm.h index a0e057fce4450..f3fd586a37530 100644 --- a/py/asmarm.h +++ b/py/asmarm.h @@ -109,13 +109,18 @@ void asm_arm_lsr_reg_reg(asm_arm_t *as, uint rd, uint rs); void asm_arm_asr_reg_reg(asm_arm_t *as, uint rd, uint rs); // memory -void asm_arm_ldr_reg_reg(asm_arm_t *as, uint rd, uint rn, uint byte_offset); -void asm_arm_ldrh_reg_reg(asm_arm_t *as, uint rd, uint rn); +void asm_arm_ldr_reg_reg_offset(asm_arm_t *as, uint rd, uint rn, uint byte_offset); void asm_arm_ldrh_reg_reg_offset(asm_arm_t *as, uint rd, uint rn, uint byte_offset); -void asm_arm_ldrb_reg_reg(asm_arm_t *as, uint rd, uint rn); -void asm_arm_str_reg_reg(asm_arm_t *as, uint rd, uint rm, uint byte_offset); -void asm_arm_strh_reg_reg(asm_arm_t *as, uint rd, uint rm); -void asm_arm_strb_reg_reg(asm_arm_t *as, uint rd, uint rm); +void asm_arm_ldrb_reg_reg_offset(asm_arm_t *as, uint rd, uint rn, uint byte_offset); +void asm_arm_str_reg_reg_offset(asm_arm_t *as, uint rd, uint rm, uint byte_offset); +void asm_arm_strh_reg_reg_offset(asm_arm_t *as, uint rd, uint rm, uint byte_offset); +void asm_arm_strb_reg_reg_offset(asm_arm_t *as, uint rd, uint rm, uint byte_offset); + +// load from array +void asm_arm_ldr_reg_reg_reg(asm_arm_t *as, uint rd, uint rm, uint rn); +void asm_arm_ldrh_reg_reg_reg(asm_arm_t *as, uint rd, uint rm, uint rn); +void asm_arm_ldrb_reg_reg_reg(asm_arm_t *as, uint rd, uint rm, uint rn); + // store to array void asm_arm_str_reg_reg_reg(asm_arm_t *as, uint rd, uint rm, uint rn); void asm_arm_strh_reg_reg_reg(asm_arm_t *as, uint rd, uint rm, uint rn); @@ -160,12 +165,12 @@ void asm_arm_bx_reg(asm_arm_t *as, uint reg_src); // Holds a pointer to mp_fun_table #define REG_FUN_TABLE ASM_ARM_REG_FUN_TABLE -#define ASM_T asm_arm_t -#define ASM_END_PASS asm_arm_end_pass -#define ASM_ENTRY asm_arm_entry -#define ASM_EXIT asm_arm_exit +#define ASM_T asm_arm_t +#define ASM_END_PASS asm_arm_end_pass +#define ASM_ENTRY(as, num_locals, name) asm_arm_entry((as), (num_locals)) +#define ASM_EXIT asm_arm_exit -#define ASM_JUMP asm_arm_b_label +#define ASM_JUMP asm_arm_b_label #define ASM_JUMP_IF_REG_ZERO(as, reg, label, bool_test) \ do { \ asm_arm_cmp_reg_i8(as, reg, 0); \ @@ -203,18 +208,28 @@ void asm_arm_bx_reg(asm_arm_t *as, uint reg_src); #define ASM_SUB_REG_REG(as, reg_dest, reg_src) asm_arm_sub_reg_reg_reg((as), (reg_dest), (reg_dest), (reg_src)) #define ASM_MUL_REG_REG(as, reg_dest, reg_src) asm_arm_mul_reg_reg_reg((as), (reg_dest), (reg_dest), (reg_src)) -#define ASM_LOAD_REG_REG(as, reg_dest, reg_base) asm_arm_ldr_reg_reg((as), (reg_dest), (reg_base), 0) -#define ASM_LOAD_REG_REG_OFFSET(as, reg_dest, reg_base, word_offset) asm_arm_ldr_reg_reg((as), (reg_dest), (reg_base), 4 * (word_offset)) -#define ASM_LOAD8_REG_REG(as, reg_dest, reg_base) asm_arm_ldrb_reg_reg((as), (reg_dest), (reg_base)) -#define ASM_LOAD16_REG_REG(as, reg_dest, reg_base) asm_arm_ldrh_reg_reg((as), (reg_dest), (reg_base)) -#define ASM_LOAD16_REG_REG_OFFSET(as, reg_dest, reg_base, uint16_offset) asm_arm_ldrh_reg_reg_offset((as), (reg_dest), (reg_base), 2 * (uint16_offset)) -#define ASM_LOAD32_REG_REG(as, reg_dest, reg_base) asm_arm_ldr_reg_reg((as), (reg_dest), (reg_base), 0) - -#define ASM_STORE_REG_REG(as, reg_value, reg_base) asm_arm_str_reg_reg((as), (reg_value), (reg_base), 0) -#define ASM_STORE_REG_REG_OFFSET(as, reg_dest, reg_base, word_offset) asm_arm_str_reg_reg((as), (reg_dest), (reg_base), 4 * (word_offset)) -#define ASM_STORE8_REG_REG(as, reg_value, reg_base) asm_arm_strb_reg_reg((as), (reg_value), (reg_base)) -#define ASM_STORE16_REG_REG(as, reg_value, reg_base) asm_arm_strh_reg_reg((as), (reg_value), (reg_base)) -#define ASM_STORE32_REG_REG(as, reg_value, reg_base) asm_arm_str_reg_reg((as), (reg_value), (reg_base), 0) +#define ASM_LOAD_REG_REG_OFFSET(as, reg_dest, reg_base, word_offset) ASM_LOAD32_REG_REG_OFFSET((as), (reg_dest), (reg_base), (word_offset)) +#define ASM_LOAD8_REG_REG(as, reg_dest, reg_base) ASM_LOAD8_REG_REG_OFFSET((as), (reg_dest), (reg_base), 0) +#define ASM_LOAD8_REG_REG_OFFSET(as, reg_dest, reg_base, byte_offset) asm_arm_ldrb_reg_reg_offset((as), (reg_dest), (reg_base), (byte_offset)) +#define ASM_LOAD16_REG_REG(as, reg_dest, reg_base) ASM_LOAD16_REG_REG_OFFSET((as), (reg_dest), (reg_base), 0) +#define ASM_LOAD16_REG_REG_OFFSET(as, reg_dest, reg_base, halfword_offset) asm_arm_ldrh_reg_reg_offset((as), (reg_dest), (reg_base), 2 * (halfword_offset)) +#define ASM_LOAD32_REG_REG(as, reg_dest, reg_base) ASM_LOAD32_REG_REG_OFFSET((as), (reg_dest), (reg_base), 0) +#define ASM_LOAD32_REG_REG_OFFSET(as, reg_dest, reg_base, word_offset) asm_arm_ldr_reg_reg_offset((as), (reg_dest), (reg_base), 4 * (word_offset)) + +#define ASM_STORE_REG_REG_OFFSET(as, reg_value, reg_base, word_offset) ASM_STORE32_REG_REG_OFFSET((as), (reg_value), (reg_base), (word_offset)) +#define ASM_STORE8_REG_REG(as, reg_value, reg_base) ASM_STORE8_REG_REG_OFFSET((as), (reg_value), (reg_base), 0) +#define ASM_STORE8_REG_REG_OFFSET(as, reg_value, reg_base, byte_offset) asm_arm_strb_reg_reg_offset((as), (reg_value), (reg_base), (byte_offset)) +#define ASM_STORE16_REG_REG(as, reg_value, reg_base) ASM_STORE16_REG_REG_OFFSET((as), (reg_value), (reg_base), 0) +#define ASM_STORE16_REG_REG_OFFSET(as, reg_value, reg_base, halfword_offset) asm_arm_strh_reg_reg_offset((as), (reg_value), (reg_base), 2 * (halfword_offset)) +#define ASM_STORE32_REG_REG(as, reg_value, reg_base) ASM_STORE32_REG_REG_OFFSET((as), (reg_value), (reg_base), 0) +#define ASM_STORE32_REG_REG_OFFSET(as, reg_value, reg_base, word_offset) asm_arm_str_reg_reg_offset((as), (reg_value), (reg_base), 4 * (word_offset)) + +#define ASM_LOAD8_REG_REG_REG(as, reg_dest, reg_base, reg_index) asm_arm_ldrb_reg_reg_reg((as), (reg_dest), (reg_base), (reg_index)) +#define ASM_LOAD16_REG_REG_REG(as, reg_dest, reg_base, reg_index) asm_arm_ldrh_reg_reg_reg((as), (reg_dest), (reg_base), (reg_index)) +#define ASM_LOAD32_REG_REG_REG(as, reg_dest, reg_base, reg_index) asm_arm_ldr_reg_reg_reg((as), (reg_dest), (reg_base), (reg_index)) +#define ASM_STORE8_REG_REG_REG(as, reg_val, reg_base, reg_index) asm_arm_strb_reg_reg_reg((as), (reg_val), (reg_base), (reg_index)) +#define ASM_STORE16_REG_REG_REG(as, reg_val, reg_base, reg_index) asm_arm_strh_reg_reg_reg((as), (reg_val), (reg_base), (reg_index)) +#define ASM_STORE32_REG_REG_REG(as, reg_val, reg_base, reg_index) asm_arm_str_reg_reg_reg((as), (reg_val), (reg_base), (reg_index)) #endif // GENERIC_ASM_API diff --git a/py/asmbase.c b/py/asmbase.c index 3fce543a7f485..07dbf4430f9db 100644 --- a/py/asmbase.c +++ b/py/asmbase.c @@ -53,7 +53,7 @@ void mp_asm_base_start_pass(mp_asm_base_t *as, int pass) { } else { // allocating executable RAM is platform specific MP_PLAT_ALLOC_EXEC(as->code_offset, (void **)&as->code_base, &as->code_size); - assert(as->code_base != NULL); + assert(as->code_size == 0 || as->code_base != NULL); } as->pass = pass; as->suppress = false; @@ -102,7 +102,7 @@ void mp_asm_base_label_assign(mp_asm_base_t *as, size_t label) { // align must be a multiple of 2 void mp_asm_base_align(mp_asm_base_t *as, unsigned int align) { - as->code_offset = (as->code_offset + align - 1) & (~(align - 1)); + as->code_offset = (as->code_offset + align - 1) & (~(size_t)(align - 1)); } // this function assumes a little endian machine diff --git a/py/asmrv32.c b/py/asmrv32.c index c24d05a1384d4..1d0cea6c02616 100644 --- a/py/asmrv32.c +++ b/py/asmrv32.c @@ -36,6 +36,8 @@ #if MICROPY_EMIT_RV32 #include "py/asmrv32.h" +#include "py/mpstate.h" +#include "py/persistentcode.h" #if MICROPY_DEBUG_VERBOSE #define DEBUG_PRINT (1) @@ -450,18 +452,24 @@ void asm_rv32_emit_mov_reg_local_addr(asm_rv32_t *state, mp_uint_t rd, mp_uint_t asm_rv32_opcode_cadd(state, rd, ASM_RV32_REG_SP); } -void asm_rv32_emit_load_reg_reg_offset(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, mp_int_t offset) { - mp_int_t scaled_offset = offset * sizeof(ASM_WORD_SIZE); +static const uint8_t RV32_LOAD_OPCODE_TABLE[3] = { + 0x04, 0x05, 0x02 +}; - if (scaled_offset >= 0 && RV32_IS_IN_C_REGISTER_WINDOW(rd) && RV32_IS_IN_C_REGISTER_WINDOW(rs) && FIT_UNSIGNED(scaled_offset, 6)) { +void asm_rv32_emit_load_reg_reg_offset(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, int32_t offset, mp_uint_t operation_size) { + assert(operation_size <= 2 && "Operation size value out of range."); + + int32_t scaled_offset = offset << operation_size; + + if (scaled_offset >= 0 && operation_size == 2 && RV32_IS_IN_C_REGISTER_WINDOW(rd) && RV32_IS_IN_C_REGISTER_WINDOW(rs) && MP_FIT_UNSIGNED(6, scaled_offset)) { // c.lw rd', offset(rs') asm_rv32_opcode_clw(state, RV32_MAP_IN_C_REGISTER_WINDOW(rd), RV32_MAP_IN_C_REGISTER_WINDOW(rs), scaled_offset); return; } - if (FIT_SIGNED(scaled_offset, 12)) { - // lw rd, offset(rs) - asm_rv32_opcode_lw(state, rd, rs, scaled_offset); + if (MP_FIT_SIGNED(12, scaled_offset)) { + // lbu|lhu|lw rd, offset(rs) + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_I(0x03, RV32_LOAD_OPCODE_TABLE[operation_size], rd, rs, scaled_offset)); return; } @@ -469,12 +477,12 @@ void asm_rv32_emit_load_reg_reg_offset(asm_rv32_t *state, mp_uint_t rd, mp_uint_ mp_uint_t lower = 0; split_immediate(scaled_offset, &upper, &lower); - // lui rd, HI(offset) ; Or c.lui if possible - // c.add rd, rs - // lw rd, LO(offset)(rd) + // lui rd, HI(offset) ; Or c.lui if possible + // c.add rd, rs + // lbu|lhu|lw rd, LO(offset)(rd) load_upper_immediate(state, rd, upper); asm_rv32_opcode_cadd(state, rd, rs); - asm_rv32_opcode_lw(state, rd, rd, lower); + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_I(0x03, RV32_LOAD_OPCODE_TABLE[operation_size], rd, rd, lower)); } void asm_rv32_emit_jump(asm_rv32_t *state, mp_uint_t label) { @@ -497,12 +505,20 @@ void asm_rv32_emit_jump(asm_rv32_t *state, mp_uint_t label) { asm_rv32_opcode_jalr(state, ASM_RV32_REG_ZERO, REG_TEMP2, lower); } -void asm_rv32_emit_store_reg_reg_offset(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, mp_int_t offset) { - mp_int_t scaled_offset = offset * ASM_WORD_SIZE; +void asm_rv32_emit_store_reg_reg_offset(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, int32_t offset, mp_uint_t operation_size) { + assert(operation_size <= 2 && "Operation size value out of range."); - if (FIT_SIGNED(scaled_offset, 12)) { - // sw rd, offset(rs) - asm_rv32_opcode_sw(state, rd, rs, scaled_offset); + int32_t scaled_offset = offset << operation_size; + + if (scaled_offset >= 0 && operation_size == 2 && RV32_IS_IN_C_REGISTER_WINDOW(rd) && RV32_IS_IN_C_REGISTER_WINDOW(rs) && MP_FIT_UNSIGNED(6, scaled_offset)) { + // c.sw rd', offset(rs') + asm_rv32_opcode_csw(state, RV32_MAP_IN_C_REGISTER_WINDOW(rd), RV32_MAP_IN_C_REGISTER_WINDOW(rs), scaled_offset); + return; + } + + if (MP_FIT_SIGNED(12, scaled_offset)) { + // sb|sh|sw rd, offset(rs) + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_S(0x23, operation_size, rs, rd, scaled_offset)); return; } @@ -510,12 +526,12 @@ void asm_rv32_emit_store_reg_reg_offset(asm_rv32_t *state, mp_uint_t rd, mp_uint mp_uint_t lower = 0; split_immediate(scaled_offset, &upper, &lower); - // lui temporary, HI(offset) ; Or c.lui if possible - // c.add temporary, rs - // sw rd, LO(offset)(temporary) + // lui temporary, HI(offset) ; Or c.lui if possible + // c.add temporary, rs + // sb|sh|sw rd, LO(offset)(temporary) load_upper_immediate(state, REG_TEMP2, upper); asm_rv32_opcode_cadd(state, REG_TEMP2, rs); - asm_rv32_opcode_sw(state, rd, REG_TEMP2, lower); + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_S(0x23, operation_size, REG_TEMP2, rd, lower)); } void asm_rv32_emit_mov_reg_pcrel(asm_rv32_t *state, mp_uint_t rd, mp_uint_t label) { @@ -530,27 +546,6 @@ void asm_rv32_emit_mov_reg_pcrel(asm_rv32_t *state, mp_uint_t rd, mp_uint_t labe asm_rv32_opcode_addi(state, rd, rd, lower); } -void asm_rv32_emit_load16_reg_reg_offset(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, mp_int_t offset) { - mp_int_t scaled_offset = offset * sizeof(uint16_t); - - if (FIT_SIGNED(scaled_offset, 12)) { - // lhu rd, offset(rs) - asm_rv32_opcode_lhu(state, rd, rs, scaled_offset); - return; - } - - mp_uint_t upper = 0; - mp_uint_t lower = 0; - split_immediate(scaled_offset, &upper, &lower); - - // lui rd, HI(offset) ; Or c.lui if possible - // c.add rd, rs - // lhu rd, LO(offset)(rd) - load_upper_immediate(state, rd, upper); - asm_rv32_opcode_cadd(state, rd, rs); - asm_rv32_opcode_lhu(state, rd, rd, lower); -} - void asm_rv32_emit_optimised_xor(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs) { if (rs == rd) { // c.li rd, 0 @@ -562,14 +557,39 @@ void asm_rv32_emit_optimised_xor(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs) asm_rv32_opcode_xor(state, rd, rd, rs); } +static bool asm_rv32_allow_zba_opcodes(void) { + return asm_rv32_allowed_extensions() & RV32_EXT_ZBA; +} + +static void asm_rv32_fix_up_scaled_reg_reg_reg(asm_rv32_t *state, mp_uint_t rs1, mp_uint_t rs2, mp_uint_t operation_size) { + assert(operation_size <= 2 && "Operation size value out of range."); + + if (operation_size > 0 && asm_rv32_allow_zba_opcodes()) { + // sh{1,2}add rs1, rs2, rs1 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_R(0x33, 1 << operation_size, 0x10, rs1, rs2, rs1)); + } else { + if (operation_size > 0) { + asm_rv32_opcode_cslli(state, rs2, operation_size); + } + asm_rv32_opcode_cadd(state, rs1, rs2); + } +} + +void asm_rv32_emit_load_reg_reg_reg(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs1, mp_uint_t rs2, mp_uint_t operation_size) { + asm_rv32_fix_up_scaled_reg_reg_reg(state, rs1, rs2, operation_size); + asm_rv32_emit_load_reg_reg_offset(state, rd, rs1, 0, operation_size); +} + +void asm_rv32_emit_store_reg_reg_reg(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs1, mp_uint_t rs2, mp_uint_t operation_size) { + asm_rv32_fix_up_scaled_reg_reg_reg(state, rs1, rs2, operation_size); + asm_rv32_emit_store_reg_reg_offset(state, rd, rs1, 0, operation_size); +} + void asm_rv32_meta_comparison_eq(asm_rv32_t *state, mp_uint_t rs1, mp_uint_t rs2, mp_uint_t rd) { - // c.li rd, 1 ; - // beq rs1, rs2, 6 ; PC + 0 - // c.li rd, 0 ; PC + 4 - // ... ; PC + 6 - asm_rv32_opcode_cli(state, rd, 1); - asm_rv32_opcode_beq(state, rs1, rs2, 6); - asm_rv32_opcode_cli(state, rd, 0); + // sub rd, rs1, rs2 + // sltiu rd, rd, 1 + asm_rv32_opcode_sub(state, rd, rs1, rs2); + asm_rv32_opcode_sltiu(state, rd, rd, 1); } void asm_rv32_meta_comparison_ne(asm_rv32_t *state, mp_uint_t rs1, mp_uint_t rs2, mp_uint_t rd) { @@ -580,26 +600,15 @@ void asm_rv32_meta_comparison_ne(asm_rv32_t *state, mp_uint_t rs1, mp_uint_t rs2 } void asm_rv32_meta_comparison_lt(asm_rv32_t *state, mp_uint_t rs1, mp_uint_t rs2, mp_uint_t rd, bool unsigned_comparison) { - // slt(u) rd, rs1, rs2 - if (unsigned_comparison) { - asm_rv32_opcode_sltu(state, rd, rs1, rs2); - } else { - asm_rv32_opcode_slt(state, rd, rs1, rs2); - } + // slt|sltu rd, rs1, rs2 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_R(0x33, (0x02 | (unsigned_comparison ? 1 : 0)), 0x00, rd, rs1, rs2)); } void asm_rv32_meta_comparison_le(asm_rv32_t *state, mp_uint_t rs1, mp_uint_t rs2, mp_uint_t rd, bool unsigned_comparison) { - // c.li rd, 1 ; - // beq rs1, rs2, 8 ; PC + 0 - // slt(u) rd, rs1, rs2 ; PC + 4 - // ... ; PC + 8 - asm_rv32_opcode_cli(state, rd, 1); - asm_rv32_opcode_beq(state, rs1, rs2, 8); - if (unsigned_comparison) { - asm_rv32_opcode_sltu(state, rd, rs1, rs2); - } else { - asm_rv32_opcode_slt(state, rd, rs1, rs2); - } + // slt[u] rd, rs2, rs1 + // xori rd, rd, 1 + asm_rv32_meta_comparison_lt(state, rs2, rs1, rd, unsigned_comparison); + asm_rv32_opcode_xori(state, rd, rd, 1); } #endif // MICROPY_EMIT_RV32 diff --git a/py/asmrv32.h b/py/asmrv32.h index b09f48eb12f66..6f709daa11bce 100644 --- a/py/asmrv32.h +++ b/py/asmrv32.h @@ -122,6 +122,18 @@ typedef struct _asm_rv32_t { mp_uint_t locals_stack_offset; } asm_rv32_t; +enum { + RV32_EXT_NONE = 0, + RV32_EXT_ZBA = 1 << 0, + + RV32_EXT_ALL = RV32_EXT_ZBA +}; + +typedef struct _asm_rv32_backend_options_t { + // This is a bitmask holding a combination of RV32_EXT_* entries. + uint8_t allowed_extensions; +} asm_rv32_backend_options_t; + void asm_rv32_entry(asm_rv32_t *state, mp_uint_t locals); void asm_rv32_exit(asm_rv32_t *state); void asm_rv32_end_pass(asm_rv32_t *state); @@ -583,6 +595,24 @@ static inline void asm_rv32_opcode_remu(asm_rv32_t *state, mp_uint_t rd, mp_uint asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_R(0x33, 0x07, 0x01, rd, rs1, rs2)); } +// SH1ADD RD, RS1, RS2 +static inline void asm_rv32_opcode_sh1add(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs1, mp_uint_t rs2) { + // R: 0010000 ..... ..... 010 ..... 0110011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_R(0x33, 0x02, 0x10, rd, rs1, rs2)); +} + +// SH2ADD RD, RS1, RS2 +static inline void asm_rv32_opcode_sh2add(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs1, mp_uint_t rs2) { + // R: 0010000 ..... ..... 100 ..... 0110011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_R(0x33, 0x04, 0x10, rd, rs1, rs2)); +} + +// SH3ADD RD, RS1, RS2 +static inline void asm_rv32_opcode_sh3add(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs1, mp_uint_t rs2) { + // R: 0010000 ..... ..... 110 ..... 0110011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_R(0x33, 0x06, 0x10, rd, rs1, rs2)); +} + // SLL RD, RS1, RS2 static inline void asm_rv32_opcode_sll(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs1, mp_uint_t rs2) { // R: 0000000 ..... ..... 001 ..... 0110011 @@ -679,6 +709,19 @@ static inline void asm_rv32_opcode_xori(asm_rv32_t *state, mp_uint_t rd, mp_uint asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_I(0x13, 0x04, rd, rs, immediate)); } +#define MICROPY_RV32_EXTENSIONS \ + (MICROPY_EMIT_RV32_ZBA ? RV32_EXT_ZBA : 0) + +static inline uint8_t asm_rv32_allowed_extensions(void) { + uint8_t extensions = MICROPY_RV32_EXTENSIONS; + #if MICROPY_DYNAMIC_COMPILER + if (mp_dynamic_compiler.backend_options != NULL) { + extensions |= ((asm_rv32_backend_options_t *)mp_dynamic_compiler.backend_options)->allowed_extensions; + } + #endif + return extensions; +} + #define ASM_WORD_SIZE (4) #define ASM_HALFWORD_SIZE (2) @@ -702,6 +745,8 @@ void asm_rv32_meta_comparison_lt(asm_rv32_t *state, mp_uint_t rs1, mp_uint_t rs2 void asm_rv32_meta_comparison_le(asm_rv32_t *state, mp_uint_t rs1, mp_uint_t rs2, mp_uint_t rd, bool unsigned_comparison); void asm_rv32_emit_optimised_load_immediate(asm_rv32_t *state, mp_uint_t rd, mp_int_t immediate); +void asm_rv32_emit_load_reg_reg_reg(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs1, mp_uint_t rs2, mp_uint_t operation_size); +void asm_rv32_emit_store_reg_reg_reg(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs1, mp_uint_t rs2, mp_uint_t operation_size); #ifdef GENERIC_ASM_API @@ -709,17 +754,16 @@ void asm_rv32_emit_call_ind(asm_rv32_t *state, mp_uint_t index); void asm_rv32_emit_jump(asm_rv32_t *state, mp_uint_t label); void asm_rv32_emit_jump_if_reg_eq(asm_rv32_t *state, mp_uint_t rs1, mp_uint_t rs2, mp_uint_t label); void asm_rv32_emit_jump_if_reg_nonzero(asm_rv32_t *state, mp_uint_t rs, mp_uint_t label); -void asm_rv32_emit_load16_reg_reg_offset(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, mp_int_t offset); -void asm_rv32_emit_load_reg_reg_offset(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, mp_int_t offset); +void asm_rv32_emit_load_reg_reg_offset(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, int32_t offset, mp_uint_t operation_size); void asm_rv32_emit_mov_local_reg(asm_rv32_t *state, mp_uint_t local, mp_uint_t rs); void asm_rv32_emit_mov_reg_local_addr(asm_rv32_t *state, mp_uint_t rd, mp_uint_t local); void asm_rv32_emit_mov_reg_local(asm_rv32_t *state, mp_uint_t rd, mp_uint_t local); void asm_rv32_emit_mov_reg_pcrel(asm_rv32_t *state, mp_uint_t rd, mp_uint_t label); void asm_rv32_emit_optimised_xor(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs); -void asm_rv32_emit_store_reg_reg_offset(asm_rv32_t *state, mp_uint_t source, mp_uint_t base, mp_int_t offset); +void asm_rv32_emit_store_reg_reg_offset(asm_rv32_t *state, mp_uint_t source, mp_uint_t base, int32_t offset, mp_uint_t operation_size); #define ASM_T asm_rv32_t -#define ASM_ENTRY(state, labels) asm_rv32_entry(state, labels) +#define ASM_ENTRY(state, labels, name) asm_rv32_entry(state, labels) #define ASM_EXIT(state) asm_rv32_exit(state) #define ASM_END_PASS(state) asm_rv32_end_pass(state) @@ -732,12 +776,13 @@ void asm_rv32_emit_store_reg_reg_offset(asm_rv32_t *state, mp_uint_t source, mp_ #define ASM_JUMP_IF_REG_NONZERO(state, rs, label, bool_test) asm_rv32_emit_jump_if_reg_nonzero(state, rs, label) #define ASM_JUMP_IF_REG_ZERO(state, rs, label, bool_test) asm_rv32_emit_jump_if_reg_eq(state, rs, ASM_RV32_REG_ZERO, label) #define ASM_JUMP_REG(state, rs) asm_rv32_opcode_cjr(state, rs) -#define ASM_LOAD16_REG_REG_OFFSET(state, rd, rs, offset) asm_rv32_emit_load16_reg_reg_offset(state, rd, rs, offset) -#define ASM_LOAD16_REG_REG(state, rd, rs) asm_rv32_opcode_lhu(state, rd, rs, 0) -#define ASM_LOAD32_REG_REG(state, rd, rs) ASM_LOAD_REG_REG_OFFSET(state, rd, rs, 0) -#define ASM_LOAD8_REG_REG(state, rd, rs) asm_rv32_opcode_lbu(state, rd, rs, 0) -#define ASM_LOAD_REG_REG_OFFSET(state, rd, rs, offset) asm_rv32_emit_load_reg_reg_offset(state, rd, rs, offset) -#define ASM_LOAD_REG_REG(state, rd, rs) ASM_LOAD32_REG_REG(state, rd, rs) +#define ASM_LOAD_REG_REG_OFFSET(state, rd, rs, offset) ASM_LOAD32_REG_REG_OFFSET(state, rd, rs, offset) +#define ASM_LOAD8_REG_REG(state, rd, rs) ASM_LOAD8_REG_REG_OFFSET(state, rd, rs, 0) +#define ASM_LOAD16_REG_REG(state, rd, rs) ASM_LOAD16_REG_REG_OFFSET(state, rd, rs, 0) +#define ASM_LOAD32_REG_REG(state, rd, rs) ASM_LOAD32_REG_REG_OFFSET(state, rd, rs, 0) +#define ASM_LOAD8_REG_REG_OFFSET(state, rd, rs, offset) asm_rv32_emit_load_reg_reg_offset(state, rd, rs, offset, 0) +#define ASM_LOAD16_REG_REG_OFFSET(state, rd, rs, offset) asm_rv32_emit_load_reg_reg_offset(state, rd, rs, offset, 1) +#define ASM_LOAD32_REG_REG_OFFSET(state, rd, rs, offset) asm_rv32_emit_load_reg_reg_offset(state, rd, rs, offset, 2) #define ASM_LSL_REG_REG(state, rd, rs) asm_rv32_opcode_sll(state, rd, rd, rs) #define ASM_LSR_REG_REG(state, rd, rs) asm_rv32_opcode_srl(state, rd, rd, rs) #define ASM_MOV_LOCAL_REG(state, local, rs) asm_rv32_emit_mov_local_reg(state, local, rs) @@ -750,14 +795,22 @@ void asm_rv32_emit_store_reg_reg_offset(asm_rv32_t *state, mp_uint_t source, mp_ #define ASM_NEG_REG(state, rd) asm_rv32_opcode_sub(state, rd, ASM_RV32_REG_ZERO, rd) #define ASM_NOT_REG(state, rd) asm_rv32_opcode_xori(state, rd, rd, -1) #define ASM_OR_REG_REG(state, rd, rs) asm_rv32_opcode_or(state, rd, rd, rs) -#define ASM_STORE16_REG_REG(state, rs1, rs2) asm_rv32_opcode_sh(state, rs1, rs2, 0) -#define ASM_STORE32_REG_REG(state, rs1, rs2) ASM_STORE_REG_REG_OFFSET(state, rs1, rs2, 0) -#define ASM_STORE8_REG_REG(state, rs1, rs2) asm_rv32_opcode_sb(state, rs1, rs2, 0) -#define ASM_STORE_REG_REG_OFFSET(state, rd, rs, offset) asm_rv32_emit_store_reg_reg_offset(state, rd, rs, offset) -#define ASM_STORE_REG_REG(state, rs1, rs2) ASM_STORE32_REG_REG(state, rs1, rs2) +#define ASM_STORE_REG_REG_OFFSET(state, rd, rs, offset) ASM_STORE32_REG_REG_OFFSET(state, rd, rs, offset) +#define ASM_STORE8_REG_REG(state, rs1, rs2) ASM_STORE8_REG_REG_OFFSET(state, rs1, rs2, 0) +#define ASM_STORE16_REG_REG(state, rs1, rs2) ASM_STORE16_REG_REG_OFFSET(state, rs1, rs2, 0) +#define ASM_STORE32_REG_REG(state, rs1, rs2) ASM_STORE32_REG_REG_OFFSET(state, rs1, rs2, 0) +#define ASM_STORE8_REG_REG_OFFSET(state, rd, rs, offset) asm_rv32_emit_store_reg_reg_offset(state, rd, rs, offset, 0) +#define ASM_STORE16_REG_REG_OFFSET(state, rd, rs, offset) asm_rv32_emit_store_reg_reg_offset(state, rd, rs, offset, 1) +#define ASM_STORE32_REG_REG_OFFSET(state, rd, rs, offset) asm_rv32_emit_store_reg_reg_offset(state, rd, rs, offset, 2) #define ASM_SUB_REG_REG(state, rd, rs) asm_rv32_opcode_sub(state, rd, rd, rs) #define ASM_XOR_REG_REG(state, rd, rs) asm_rv32_emit_optimised_xor(state, rd, rs) #define ASM_CLR_REG(state, rd) +#define ASM_LOAD8_REG_REG_REG(state, rd, rs1, rs2) asm_rv32_emit_load_reg_reg_reg(state, rd, rs1, rs2, 0) +#define ASM_LOAD16_REG_REG_REG(state, rd, rs1, rs2) asm_rv32_emit_load_reg_reg_reg(state, rd, rs1, rs2, 1) +#define ASM_LOAD32_REG_REG_REG(state, rd, rs1, rs2) asm_rv32_emit_load_reg_reg_reg(state, rd, rs1, rs2, 2) +#define ASM_STORE8_REG_REG_REG(state, rd, rs1, rs2) asm_rv32_emit_store_reg_reg_reg(state, rd, rs1, rs2, 0) +#define ASM_STORE16_REG_REG_REG(state, rd, rs1, rs2) asm_rv32_emit_store_reg_reg_reg(state, rd, rs1, rs2, 1) +#define ASM_STORE32_REG_REG_REG(state, rd, rs1, rs2) asm_rv32_emit_store_reg_reg_reg(state, rd, rs1, rs2, 2) #endif diff --git a/py/asmthumb.c b/py/asmthumb.c index 420815e80269a..58cc7aea88085 100644 --- a/py/asmthumb.c +++ b/py/asmthumb.c @@ -37,7 +37,6 @@ #include "py/asmthumb.h" #include "py/misc.h" -#define UNSIGNED_FIT5(x) ((uint32_t)(x) < 32) #define UNSIGNED_FIT7(x) ((uint32_t)(x) < 128) #define UNSIGNED_FIT8(x) (((x) & 0xffffff00) == 0) #define UNSIGNED_FIT16(x) (((x) & 0xffff0000) == 0) @@ -52,12 +51,6 @@ #define OP_SUB_W_RRI_HI(reg_src) (0xf2a0 | (reg_src)) #define OP_SUB_W_RRI_LO(reg_dest, imm11) ((imm11 << 4 & 0x7000) | reg_dest << 8 | (imm11 & 0xff)) -#define OP_LDR_W_HI(reg_base) (0xf8d0 | (reg_base)) -#define OP_LDR_W_LO(reg_dest, imm12) ((reg_dest) << 12 | (imm12)) - -#define OP_LDRH_W_HI(reg_base) (0xf8b0 | (reg_base)) -#define OP_LDRH_W_LO(reg_dest, imm12) ((reg_dest) << 12 | (imm12)) - static inline byte *asm_thumb_get_cur_to_write_bytes(asm_thumb_t *as, int n) { return mp_asm_base_get_cur_to_write_bytes(&as->base, n); } @@ -274,9 +267,8 @@ bool asm_thumb_b_n_label(asm_thumb_t *as, uint label) { #define OP_BCC_N(cond, byte_offset) (0xd000 | ((cond) << 8) | (((byte_offset) >> 1) & 0x00ff)) -// all these bit-arithmetic operations need coverage testing! -#define OP_BCC_W_HI(cond, byte_offset) (0xf000 | ((cond) << 6) | (((byte_offset) >> 10) & 0x0400) | (((byte_offset) >> 14) & 0x003f)) -#define OP_BCC_W_LO(byte_offset) (0x8000 | ((byte_offset) & 0x2000) | (((byte_offset) >> 1) & 0x0fff)) +#define OP_BCC_W_HI(cond, byte_offset) (0xf000 | ((cond) << 6) | (((byte_offset) >> 10) & 0x0400) | (((byte_offset) >> 12) & 0x003f)) +#define OP_BCC_W_LO(byte_offset) (0x8000 | (((byte_offset) >> 5) & 0x2000) | (((byte_offset) >> 8) & 0x0800) | (((byte_offset) >> 1) & 0x07ff)) bool asm_thumb_bcc_nw_label(asm_thumb_t *as, int cond, uint label, bool wide) { mp_uint_t dest = get_label_dest(as, label); @@ -432,11 +424,6 @@ void asm_thumb_mov_reg_pcrel(asm_thumb_t *as, uint rlo_dest, uint label) { asm_thumb_add_reg_reg(as, rlo_dest, ASM_THUMB_REG_R15); // 2 bytes } -// ARMv7-M only -static inline void asm_thumb_ldr_reg_reg_i12(asm_thumb_t *as, uint reg_dest, uint reg_base, uint word_offset) { - asm_thumb_op32(as, OP_LDR_W_HI(reg_base), OP_LDR_W_LO(reg_dest, word_offset * 4)); -} - // emits code for: reg_dest = reg_base + offset << offset_shift static void asm_thumb_add_reg_reg_offset(asm_thumb_t *as, uint reg_dest, uint reg_base, uint offset, uint offset_shift) { if (reg_dest < ASM_THUMB_REG_R8 && reg_base < ASM_THUMB_REG_R8) { @@ -450,12 +437,12 @@ static void asm_thumb_add_reg_reg_offset(asm_thumb_t *as, uint reg_dest, uint re asm_thumb_lsl_rlo_rlo_i5(as, reg_dest, reg_dest, offset_shift); asm_thumb_add_rlo_rlo_rlo(as, reg_dest, reg_dest, reg_base); } else if (reg_dest != reg_base) { - asm_thumb_mov_rlo_i16(as, reg_dest, offset << offset_shift); - asm_thumb_add_rlo_rlo_rlo(as, reg_dest, reg_dest, reg_dest); + asm_thumb_mov_reg_i32_optimised(as, reg_dest, offset << offset_shift); + asm_thumb_add_rlo_rlo_rlo(as, reg_dest, reg_dest, reg_base); } else { uint reg_other = reg_dest ^ 7; asm_thumb_op16(as, OP_PUSH_RLIST((1 << reg_other))); - asm_thumb_mov_rlo_i16(as, reg_other, offset << offset_shift); + asm_thumb_mov_reg_i32_optimised(as, reg_other, offset << offset_shift); asm_thumb_add_rlo_rlo_rlo(as, reg_dest, reg_dest, reg_other); asm_thumb_op16(as, OP_POP_RLIST((1 << reg_other))); } @@ -464,30 +451,50 @@ static void asm_thumb_add_reg_reg_offset(asm_thumb_t *as, uint reg_dest, uint re } } -void asm_thumb_ldr_reg_reg_i12_optimised(asm_thumb_t *as, uint reg_dest, uint reg_base, uint word_offset) { - if (reg_dest < ASM_THUMB_REG_R8 && reg_base < ASM_THUMB_REG_R8 && UNSIGNED_FIT5(word_offset)) { - asm_thumb_ldr_rlo_rlo_i5(as, reg_dest, reg_base, word_offset); - } else if (asm_thumb_allow_armv7m(as)) { - asm_thumb_ldr_reg_reg_i12(as, reg_dest, reg_base, word_offset); +#define OP_LDR_STR_W_HI(operation_size, reg) ((0xf880 | (operation_size) << 5) | (reg)) +#define OP_LDR_STR_W_LO(reg, imm12) (((reg) << 12) | (imm12)) + +#define OP_LDR 0x01 +#define OP_STR 0x00 + +#define OP_LDR_W 0x10 +#define OP_STR_W 0x00 + +static const uint8_t OP_LDR_STR_TABLE[3] = { + 0x0E, 0x10, 0x0C +}; + +void asm_thumb_load_reg_reg_offset(asm_thumb_t *as, uint reg_dest, uint reg_base, uint offset, uint operation_size) { + assert(operation_size <= 2 && "Operation size out of range."); + + if (MP_FIT_UNSIGNED(5, offset) && (reg_dest < ASM_THUMB_REG_R8) && (reg_base < ASM_THUMB_REG_R8)) { + // Can use T1 encoding + asm_thumb_op16(as, ((OP_LDR_STR_TABLE[operation_size] | OP_LDR) << 11) | (offset << 6) | (reg_base << 3) | reg_dest); + } else if (asm_thumb_allow_armv7m(as) && MP_FIT_UNSIGNED(12, offset << operation_size)) { + // Can use T3 encoding + asm_thumb_op32(as, (OP_LDR_STR_W_HI(operation_size, reg_base) | OP_LDR_W), OP_LDR_STR_W_LO(reg_dest, (offset << operation_size))); } else { - asm_thumb_add_reg_reg_offset(as, reg_dest, reg_base, word_offset - 31, 2); - asm_thumb_ldr_rlo_rlo_i5(as, reg_dest, reg_dest, 31); + // Must use the generic sequence + asm_thumb_add_reg_reg_offset(as, reg_dest, reg_base, offset - 31, operation_size); + asm_thumb_op16(as, ((OP_LDR_STR_TABLE[operation_size] | OP_LDR) << 11) | (31 << 6) | (reg_dest << 3) | (reg_dest)); } } -// ARMv7-M only -static inline void asm_thumb_ldrh_reg_reg_i12(asm_thumb_t *as, uint reg_dest, uint reg_base, uint uint16_offset) { - asm_thumb_op32(as, OP_LDRH_W_HI(reg_base), OP_LDRH_W_LO(reg_dest, uint16_offset * 2)); -} +void asm_thumb_store_reg_reg_offset(asm_thumb_t *as, uint reg_src, uint reg_base, uint offset, uint operation_size) { + assert(operation_size <= 2 && "Operation size out of range."); -void asm_thumb_ldrh_reg_reg_i12_optimised(asm_thumb_t *as, uint reg_dest, uint reg_base, uint uint16_offset) { - if (reg_dest < ASM_THUMB_REG_R8 && reg_base < ASM_THUMB_REG_R8 && UNSIGNED_FIT5(uint16_offset)) { - asm_thumb_ldrh_rlo_rlo_i5(as, reg_dest, reg_base, uint16_offset); - } else if (asm_thumb_allow_armv7m(as)) { - asm_thumb_ldrh_reg_reg_i12(as, reg_dest, reg_base, uint16_offset); + if (MP_FIT_UNSIGNED(5, offset) && (reg_src < ASM_THUMB_REG_R8) && (reg_base < ASM_THUMB_REG_R8)) { + // Can use T1 encoding + asm_thumb_op16(as, ((OP_LDR_STR_TABLE[operation_size] | OP_STR) << 11) | (offset << 6) | (reg_base << 3) | reg_src); + } else if (asm_thumb_allow_armv7m(as) && MP_FIT_UNSIGNED(12, offset << operation_size)) { + // Can use T3 encoding + asm_thumb_op32(as, (OP_LDR_STR_W_HI(operation_size, reg_base) | OP_STR_W), OP_LDR_STR_W_LO(reg_src, (offset << operation_size))); } else { - asm_thumb_add_reg_reg_offset(as, reg_dest, reg_base, uint16_offset - 31, 1); - asm_thumb_ldrh_rlo_rlo_i5(as, reg_dest, reg_dest, 31); + // Must use the generic sequence + asm_thumb_op16(as, OP_PUSH_RLIST(1 << reg_base)); + asm_thumb_add_reg_reg_offset(as, reg_base, reg_base, offset - 31, operation_size); + asm_thumb_op16(as, ((OP_LDR_STR_TABLE[operation_size] | OP_STR) << 11) | (31 << 6) | (reg_base << 3) | reg_src); + asm_thumb_op16(as, OP_POP_RLIST(1 << reg_base)); } } @@ -495,6 +502,7 @@ void asm_thumb_ldrh_reg_reg_i12_optimised(asm_thumb_t *as, uint reg_dest, uint r #define OP_BW_HI(byte_offset) (0xf000 | (((byte_offset) >> 12) & 0x07ff)) #define OP_BW_LO(byte_offset) (0xb800 | (((byte_offset) >> 1) & 0x07ff)) +// In Thumb1 mode, this may clobber r1. void asm_thumb_b_label(asm_thumb_t *as, uint label) { mp_uint_t dest = get_label_dest(as, label); mp_int_t rel = dest - as->base.code_offset; @@ -514,19 +522,40 @@ void asm_thumb_b_label(asm_thumb_t *as, uint label) { if (asm_thumb_allow_armv7m(as)) { asm_thumb_op32(as, OP_BW_HI(rel), OP_BW_LO(rel)); } else { + // this code path has to be the same instruction size irrespective of the value of rel + bool need_align = as->base.code_offset & 2u; if (SIGNED_FIT12(rel)) { - // this code path has to be the same number of instructions irrespective of rel asm_thumb_op16(as, OP_B_N(rel)); - } else { asm_thumb_op16(as, ASM_THUMB_OP_NOP); - if (dest != (mp_uint_t)-1) { - // we have an actual branch > 12 bits; this is not handled yet - mp_raise_NotImplementedError(MP_ERROR_TEXT("native method too big")); + asm_thumb_op16(as, ASM_THUMB_OP_NOP); + asm_thumb_op16(as, ASM_THUMB_OP_NOP); + if (need_align) { + asm_thumb_op16(as, ASM_THUMB_OP_NOP); + } + } else { + // do a large jump using: + // (nop) + // ldr r1, [pc, _data] + // add pc, r1 + // _data: .word rel + // + // note: can't use r0 as a temporary because native code can have the return value + // in that register and use a large jump to get to the exit point of the function + + rel -= 2; // account for the "ldr r1, [pc, _data]" + if (need_align) { + asm_thumb_op16(as, ASM_THUMB_OP_NOP); + rel -= 2; // account for this nop } + asm_thumb_ldr_rlo_pcrel_i8(as, ASM_THUMB_REG_R1, 0); + asm_thumb_add_reg_reg(as, ASM_THUMB_REG_R15, ASM_THUMB_REG_R1); + asm_thumb_op16(as, rel & 0xffff); + asm_thumb_op16(as, rel >> 16); } } } +// In Thumb1 mode, this may clobber r1. void asm_thumb_bcc_label(asm_thumb_t *as, int cond, uint label) { mp_uint_t dest = get_label_dest(as, label); mp_int_t rel = dest - as->base.code_offset; @@ -547,8 +576,15 @@ void asm_thumb_bcc_label(asm_thumb_t *as, int cond, uint label) { asm_thumb_op32(as, OP_BCC_W_HI(cond, rel), OP_BCC_W_LO(rel)); } else { // reverse the sense of the branch to jump over a longer branch - asm_thumb_op16(as, OP_BCC_N(cond ^ 1, 0)); + size_t code_offset_start = as->base.code_offset; + byte *c = asm_thumb_get_cur_to_write_bytes(as, 2); asm_thumb_b_label(as, label); + size_t bytes_to_skip = as->base.code_offset - code_offset_start; + uint16_t op = OP_BCC_N(cond ^ 1, bytes_to_skip - 4); + if (c != NULL) { + c[0] = op; + c[1] = op >> 8; + } } } @@ -569,7 +605,7 @@ void asm_thumb_b_rel12(asm_thumb_t *as, int rel) { void asm_thumb_bl_ind(asm_thumb_t *as, uint fun_id, uint reg_temp) { // Load ptr to function from table, indexed by fun_id, then call it - asm_thumb_ldr_reg_reg_i12_optimised(as, reg_temp, ASM_THUMB_REG_FUN_TABLE, fun_id); + asm_thumb_load_reg_reg_offset(as, reg_temp, ASM_THUMB_REG_FUN_TABLE, fun_id, 2); asm_thumb_op16(as, OP_BLX(reg_temp)); } diff --git a/py/asmthumb.h b/py/asmthumb.h index 0584ed3227aad..cb786694f0b78 100644 --- a/py/asmthumb.h +++ b/py/asmthumb.h @@ -251,6 +251,50 @@ static inline void asm_thumb_bx_reg(asm_thumb_t *as, uint r_src) { asm_thumb_format_5(as, ASM_THUMB_FORMAT_5_BX, 0, r_src); } +// FORMAT 7: load/store with register offset +// FORMAT 8: load/store sign-extended byte/halfword + +#define ASM_THUMB_FORMAT_7_LDR (0x5800) +#define ASM_THUMB_FORMAT_7_STR (0x5000) +#define ASM_THUMB_FORMAT_7_WORD_TRANSFER (0x0000) +#define ASM_THUMB_FORMAT_7_BYTE_TRANSFER (0x0400) +#define ASM_THUMB_FORMAT_8_LDRH (0x5A00) +#define ASM_THUMB_FORMAT_8_STRH (0x5200) + +#define ASM_THUMB_FORMAT_7_8_ENCODE(op, rlo_dest, rlo_base, rlo_index) \ + ((op) | ((rlo_index) << 6) | ((rlo_base) << 3) | ((rlo_dest))) + +static inline void asm_thumb_format_7_8(asm_thumb_t *as, uint op, uint rlo_dest, uint rlo_base, uint rlo_index) { + assert(rlo_dest < ASM_THUMB_REG_R8); + assert(rlo_base < ASM_THUMB_REG_R8); + assert(rlo_index < ASM_THUMB_REG_R8); + asm_thumb_op16(as, ASM_THUMB_FORMAT_7_8_ENCODE(op, rlo_dest, rlo_base, rlo_index)); +} + +static inline void asm_thumb_ldrb_rlo_rlo_rlo(asm_thumb_t *as, uint rlo_dest, uint rlo_base, uint rlo_index) { + asm_thumb_format_7_8(as, ASM_THUMB_FORMAT_7_LDR | ASM_THUMB_FORMAT_7_BYTE_TRANSFER, rlo_dest, rlo_base, rlo_index); +} + +static inline void asm_thumb_ldrh_rlo_rlo_rlo(asm_thumb_t *as, uint rlo_dest, uint rlo_base, uint rlo_index) { + asm_thumb_format_7_8(as, ASM_THUMB_FORMAT_8_LDRH, rlo_dest, rlo_base, rlo_index); +} + +static inline void asm_thumb_ldr_rlo_rlo_rlo(asm_thumb_t *as, uint rlo_dest, uint rlo_base, uint rlo_index) { + asm_thumb_format_7_8(as, ASM_THUMB_FORMAT_7_LDR | ASM_THUMB_FORMAT_7_WORD_TRANSFER, rlo_dest, rlo_base, rlo_index); +} + +static inline void asm_thumb_strb_rlo_rlo_rlo(asm_thumb_t *as, uint rlo_src, uint rlo_base, uint rlo_index) { + asm_thumb_format_7_8(as, ASM_THUMB_FORMAT_7_STR | ASM_THUMB_FORMAT_7_BYTE_TRANSFER, rlo_src, rlo_base, rlo_index); +} + +static inline void asm_thumb_strh_rlo_rlo_rlo(asm_thumb_t *as, uint rlo_dest, uint rlo_base, uint rlo_index) { + asm_thumb_format_7_8(as, ASM_THUMB_FORMAT_8_STRH, rlo_dest, rlo_base, rlo_index); +} + +static inline void asm_thumb_str_rlo_rlo_rlo(asm_thumb_t *as, uint rlo_src, uint rlo_base, uint rlo_index) { + asm_thumb_format_7_8(as, ASM_THUMB_FORMAT_7_STR | ASM_THUMB_FORMAT_7_WORD_TRANSFER, rlo_src, rlo_base, rlo_index); +} + // FORMAT 9: load/store with immediate offset // For word transfers the offset must be aligned, and >>2 @@ -273,24 +317,6 @@ static inline void asm_thumb_format_9_10(asm_thumb_t *as, uint op, uint rlo_dest asm_thumb_op16(as, ASM_THUMB_FORMAT_9_10_ENCODE(op, rlo_dest, rlo_base, offset)); } -static inline void asm_thumb_str_rlo_rlo_i5(asm_thumb_t *as, uint rlo_src, uint rlo_base, uint word_offset) { - asm_thumb_format_9_10(as, ASM_THUMB_FORMAT_9_STR | ASM_THUMB_FORMAT_9_WORD_TRANSFER, rlo_src, rlo_base, word_offset); -} -static inline void asm_thumb_strb_rlo_rlo_i5(asm_thumb_t *as, uint rlo_src, uint rlo_base, uint byte_offset) { - asm_thumb_format_9_10(as, ASM_THUMB_FORMAT_9_STR | ASM_THUMB_FORMAT_9_BYTE_TRANSFER, rlo_src, rlo_base, byte_offset); -} -static inline void asm_thumb_strh_rlo_rlo_i5(asm_thumb_t *as, uint rlo_src, uint rlo_base, uint uint16_offset) { - asm_thumb_format_9_10(as, ASM_THUMB_FORMAT_10_STRH, rlo_src, rlo_base, uint16_offset); -} -static inline void asm_thumb_ldr_rlo_rlo_i5(asm_thumb_t *as, uint rlo_dest, uint rlo_base, uint word_offset) { - asm_thumb_format_9_10(as, ASM_THUMB_FORMAT_9_LDR | ASM_THUMB_FORMAT_9_WORD_TRANSFER, rlo_dest, rlo_base, word_offset); -} -static inline void asm_thumb_ldrb_rlo_rlo_i5(asm_thumb_t *as, uint rlo_dest, uint rlo_base, uint byte_offset) { - asm_thumb_format_9_10(as, ASM_THUMB_FORMAT_9_LDR | ASM_THUMB_FORMAT_9_BYTE_TRANSFER, rlo_dest, rlo_base, byte_offset); -} -static inline void asm_thumb_ldrh_rlo_rlo_i5(asm_thumb_t *as, uint rlo_dest, uint rlo_base, uint uint16_offset) { - asm_thumb_format_9_10(as, ASM_THUMB_FORMAT_10_LDRH, rlo_dest, rlo_base, uint16_offset); -} static inline void asm_thumb_lsl_rlo_rlo_i5(asm_thumb_t *as, uint rlo_dest, uint rlo_src, uint shift) { asm_thumb_format_1(as, ASM_THUMB_FORMAT_1_LSL, rlo_dest, rlo_src, shift); } @@ -338,8 +364,10 @@ void asm_thumb_mov_reg_local(asm_thumb_t *as, uint rlo_dest, int local_num); // void asm_thumb_mov_reg_local_addr(asm_thumb_t *as, uint rlo_dest, int local_num); // convenience void asm_thumb_mov_reg_pcrel(asm_thumb_t *as, uint rlo_dest, uint label); -void asm_thumb_ldr_reg_reg_i12_optimised(asm_thumb_t *as, uint reg_dest, uint reg_base, uint word_offset); // convenience -void asm_thumb_ldrh_reg_reg_i12_optimised(asm_thumb_t *as, uint reg_dest, uint reg_base, uint uint16_offset); // convenience +// Generate optimised load dest, [src, #offset] sequence +void asm_thumb_load_reg_reg_offset(asm_thumb_t *as, uint reg_dest, uint reg_base, uint offset, uint operation_size); +// Generate optimised store src, [dest, #offset] sequence +void asm_thumb_store_reg_reg_offset(asm_thumb_t *as, uint reg_src, uint reg_base, uint offset, uint operation_size); void asm_thumb_b_label(asm_thumb_t *as, uint label); // convenience: picks narrow or wide branch void asm_thumb_bcc_label(asm_thumb_t *as, int cc, uint label); // convenience: picks narrow or wide branch @@ -376,12 +404,12 @@ void asm_thumb_b_rel12(asm_thumb_t *as, int rel); #define REG_FUN_TABLE ASM_THUMB_REG_FUN_TABLE -#define ASM_T asm_thumb_t -#define ASM_END_PASS asm_thumb_end_pass -#define ASM_ENTRY asm_thumb_entry -#define ASM_EXIT asm_thumb_exit +#define ASM_T asm_thumb_t +#define ASM_END_PASS asm_thumb_end_pass +#define ASM_ENTRY(as, num_locals, name) asm_thumb_entry((as), (num_locals)) +#define ASM_EXIT asm_thumb_exit -#define ASM_JUMP asm_thumb_b_label +#define ASM_JUMP asm_thumb_b_label #define ASM_JUMP_IF_REG_ZERO(as, reg, label, bool_test) \ do { \ asm_thumb_cmp_rlo_i8(as, reg, 0); \ @@ -419,18 +447,44 @@ void asm_thumb_b_rel12(asm_thumb_t *as, int rel); #define ASM_SUB_REG_REG(as, reg_dest, reg_src) asm_thumb_sub_rlo_rlo_rlo((as), (reg_dest), (reg_dest), (reg_src)) #define ASM_MUL_REG_REG(as, reg_dest, reg_src) asm_thumb_format_4((as), ASM_THUMB_FORMAT_4_MUL, (reg_dest), (reg_src)) -#define ASM_LOAD_REG_REG(as, reg_dest, reg_base) asm_thumb_ldr_rlo_rlo_i5((as), (reg_dest), (reg_base), 0) -#define ASM_LOAD_REG_REG_OFFSET(as, reg_dest, reg_base, word_offset) asm_thumb_ldr_reg_reg_i12_optimised((as), (reg_dest), (reg_base), (word_offset)) -#define ASM_LOAD8_REG_REG(as, reg_dest, reg_base) asm_thumb_ldrb_rlo_rlo_i5((as), (reg_dest), (reg_base), 0) -#define ASM_LOAD16_REG_REG(as, reg_dest, reg_base) asm_thumb_ldrh_rlo_rlo_i5((as), (reg_dest), (reg_base), 0) -#define ASM_LOAD16_REG_REG_OFFSET(as, reg_dest, reg_base, uint16_offset) asm_thumb_ldrh_reg_reg_i12_optimised((as), (reg_dest), (reg_base), (uint16_offset)) -#define ASM_LOAD32_REG_REG(as, reg_dest, reg_base) asm_thumb_ldr_rlo_rlo_i5((as), (reg_dest), (reg_base), 0) - -#define ASM_STORE_REG_REG(as, reg_src, reg_base) asm_thumb_str_rlo_rlo_i5((as), (reg_src), (reg_base), 0) -#define ASM_STORE_REG_REG_OFFSET(as, reg_src, reg_base, word_offset) asm_thumb_str_rlo_rlo_i5((as), (reg_src), (reg_base), (word_offset)) -#define ASM_STORE8_REG_REG(as, reg_src, reg_base) asm_thumb_strb_rlo_rlo_i5((as), (reg_src), (reg_base), 0) -#define ASM_STORE16_REG_REG(as, reg_src, reg_base) asm_thumb_strh_rlo_rlo_i5((as), (reg_src), (reg_base), 0) -#define ASM_STORE32_REG_REG(as, reg_src, reg_base) asm_thumb_str_rlo_rlo_i5((as), (reg_src), (reg_base), 0) +#define ASM_LOAD_REG_REG_OFFSET(as, reg_dest, reg_base, word_offset) ASM_LOAD32_REG_REG_OFFSET((as), (reg_dest), (reg_base), (word_offset)) +#define ASM_LOAD8_REG_REG(as, reg_dest, reg_base) ASM_LOAD8_REG_REG_OFFSET((as), (reg_dest), (reg_base), 0) +#define ASM_LOAD8_REG_REG_OFFSET(as, reg_dest, reg_base, byte_offset) asm_thumb_load_reg_reg_offset((as), (reg_dest), (reg_base), (byte_offset), 0) +#define ASM_LOAD16_REG_REG(as, reg_dest, reg_base) ASM_LOAD16_REG_REG_OFFSET((as), (reg_dest), (reg_base), 0) +#define ASM_LOAD16_REG_REG_OFFSET(as, reg_dest, reg_base, halfword_offset) asm_thumb_load_reg_reg_offset((as), (reg_dest), (reg_base), (halfword_offset), 1) +#define ASM_LOAD32_REG_REG(as, reg_dest, reg_base) ASM_LOAD32_REG_REG_OFFSET((as), (reg_dest), (reg_base), 0) +#define ASM_LOAD32_REG_REG_OFFSET(as, reg_dest, reg_base, word_offset) asm_thumb_load_reg_reg_offset((as), (reg_dest), (reg_base), (word_offset), 2) + +#define ASM_STORE_REG_REG_OFFSET(as, reg_src, reg_base, word_offset) ASM_STORE32_REG_REG_OFFSET((as), (reg_src), (reg_base), (word_offset)) +#define ASM_STORE8_REG_REG(as, reg_src, reg_base) ASM_STORE8_REG_REG_OFFSET((as), (reg_src), (reg_base), 0) +#define ASM_STORE8_REG_REG_OFFSET(as, reg_src, reg_base, byte_offset) asm_thumb_store_reg_reg_offset((as), (reg_src), (reg_base), (byte_offset), 0) +#define ASM_STORE16_REG_REG(as, reg_src, reg_base) ASM_STORE16_REG_REG_OFFSET((as), (reg_src), (reg_base), 0) +#define ASM_STORE16_REG_REG_OFFSET(as, reg_src, reg_base, halfword_offset) asm_thumb_store_reg_reg_offset((as), (reg_src), (reg_base), (halfword_offset), 1) +#define ASM_STORE32_REG_REG(as, reg_src, reg_base) ASM_STORE32_REG_REG_OFFSET((as), (reg_src), (reg_base), 0) +#define ASM_STORE32_REG_REG_OFFSET(as, reg_src, reg_base, word_offset) asm_thumb_store_reg_reg_offset((as), (reg_src), (reg_base), (word_offset), 2) + +#define ASM_LOAD8_REG_REG_REG(as, reg_dest, reg_base, reg_index) asm_thumb_ldrb_rlo_rlo_rlo((as), (reg_dest), (reg_base), (reg_index)) +#define ASM_LOAD16_REG_REG_REG(as, reg_dest, reg_base, reg_index) \ + do { \ + asm_thumb_lsl_rlo_rlo_i5((as), (reg_index), (reg_index), 1); \ + asm_thumb_ldrh_rlo_rlo_rlo((as), (reg_dest), (reg_base), (reg_index)); \ + } while (0) +#define ASM_LOAD32_REG_REG_REG(as, reg_dest, reg_base, reg_index) \ + do { \ + asm_thumb_lsl_rlo_rlo_i5((as), (reg_index), (reg_index), 2); \ + asm_thumb_ldr_rlo_rlo_rlo((as), (reg_dest), (reg_base), (reg_index)); \ + } while (0) +#define ASM_STORE8_REG_REG_REG(as, reg_val, reg_base, reg_index) asm_thumb_strb_rlo_rlo_rlo((as), (reg_val), (reg_base), (reg_index)) +#define ASM_STORE16_REG_REG_REG(as, reg_val, reg_base, reg_index) \ + do { \ + asm_thumb_lsl_rlo_rlo_i5((as), (reg_index), (reg_index), 1); \ + asm_thumb_strh_rlo_rlo_rlo((as), (reg_val), (reg_base), (reg_index)); \ + } while (0) +#define ASM_STORE32_REG_REG_REG(as, reg_val, reg_base, reg_index) \ + do { \ + asm_thumb_lsl_rlo_rlo_i5((as), (reg_index), (reg_index), 2); \ + asm_thumb_str_rlo_rlo_rlo((as), (reg_val), (reg_base), (reg_index)); \ + } while (0) #endif // GENERIC_ASM_API diff --git a/py/asmx64.h b/py/asmx64.h index 03070b5f63d3b..1e8cb0c905f2b 100644 --- a/py/asmx64.h +++ b/py/asmx64.h @@ -155,12 +155,12 @@ void asm_x64_call_ind(asm_x64_t *as, size_t fun_id, int temp_r32); // Holds a pointer to mp_fun_table #define REG_FUN_TABLE ASM_X64_REG_FUN_TABLE -#define ASM_T asm_x64_t -#define ASM_END_PASS asm_x64_end_pass -#define ASM_ENTRY asm_x64_entry -#define ASM_EXIT asm_x64_exit +#define ASM_T asm_x64_t +#define ASM_END_PASS asm_x64_end_pass +#define ASM_ENTRY(as, num_locals, name) asm_x64_entry((as), (num_locals)) +#define ASM_EXIT asm_x64_exit -#define ASM_JUMP asm_x64_jmp_label +#define ASM_JUMP asm_x64_jmp_label #define ASM_JUMP_IF_REG_ZERO(as, reg, label, bool_test) \ do { \ if (bool_test) { \ @@ -206,18 +206,21 @@ void asm_x64_call_ind(asm_x64_t *as, size_t fun_id, int temp_r32); #define ASM_SUB_REG_REG(as, reg_dest, reg_src) asm_x64_sub_r64_r64((as), (reg_dest), (reg_src)) #define ASM_MUL_REG_REG(as, reg_dest, reg_src) asm_x64_mul_r64_r64((as), (reg_dest), (reg_src)) -#define ASM_LOAD_REG_REG(as, reg_dest, reg_base) asm_x64_mov_mem64_to_r64((as), (reg_base), 0, (reg_dest)) -#define ASM_LOAD_REG_REG_OFFSET(as, reg_dest, reg_base, word_offset) asm_x64_mov_mem64_to_r64((as), (reg_base), 8 * (word_offset), (reg_dest)) -#define ASM_LOAD8_REG_REG(as, reg_dest, reg_base) asm_x64_mov_mem8_to_r64zx((as), (reg_base), 0, (reg_dest)) -#define ASM_LOAD16_REG_REG(as, reg_dest, reg_base) asm_x64_mov_mem16_to_r64zx((as), (reg_base), 0, (reg_dest)) -#define ASM_LOAD16_REG_REG_OFFSET(as, reg_dest, reg_base, uint16_offset) asm_x64_mov_mem16_to_r64zx((as), (reg_base), 2 * (uint16_offset), (reg_dest)) -#define ASM_LOAD32_REG_REG(as, reg_dest, reg_base) asm_x64_mov_mem32_to_r64zx((as), (reg_base), 0, (reg_dest)) - -#define ASM_STORE_REG_REG(as, reg_src, reg_base) asm_x64_mov_r64_to_mem64((as), (reg_src), (reg_base), 0) -#define ASM_STORE_REG_REG_OFFSET(as, reg_src, reg_base, word_offset) asm_x64_mov_r64_to_mem64((as), (reg_src), (reg_base), 8 * (word_offset)) -#define ASM_STORE8_REG_REG(as, reg_src, reg_base) asm_x64_mov_r8_to_mem8((as), (reg_src), (reg_base), 0) -#define ASM_STORE16_REG_REG(as, reg_src, reg_base) asm_x64_mov_r16_to_mem16((as), (reg_src), (reg_base), 0) -#define ASM_STORE32_REG_REG(as, reg_src, reg_base) asm_x64_mov_r32_to_mem32((as), (reg_src), (reg_base), 0) +#define ASM_LOAD_REG_REG_OFFSET(as, reg_dest, reg_base, qword_offset) asm_x64_mov_mem64_to_r64((as), (reg_base), 8 * (qword_offset), (reg_dest)) +#define ASM_LOAD8_REG_REG(as, reg_dest, reg_base) ASM_LOAD8_REG_REG_OFFSET((as), (reg_dest), (reg_base), 0) +#define ASM_LOAD8_REG_REG_OFFSET(as, reg_dest, reg_base, byte_offset) asm_x64_mov_mem8_to_r64zx((as), (reg_base), (byte_offset), (reg_dest)) +#define ASM_LOAD16_REG_REG(as, reg_dest, reg_base) ASM_LOAD16_REG_REG_OFFSET((as), (reg_dest), (reg_base), 0) +#define ASM_LOAD16_REG_REG_OFFSET(as, reg_dest, reg_base, word_offset) asm_x64_mov_mem16_to_r64zx((as), (reg_base), 2 * (word_offset), (reg_dest)) +#define ASM_LOAD32_REG_REG(as, reg_dest, reg_base) ASM_LOAD32_REG_REG_OFFSET((as), (reg_dest), (reg_base), 0) +#define ASM_LOAD32_REG_REG_OFFSET(as, reg_dest, reg_base, dword_offset) asm_x64_mov_mem32_to_r64zx((as), (reg_base), 4 * (dword_offset), (reg_dest)) + +#define ASM_STORE_REG_REG_OFFSET(as, reg_src, reg_base, qword_offset) asm_x64_mov_r64_to_mem64((as), (reg_src), (reg_base), 8 * (qword_offset)) +#define ASM_STORE8_REG_REG(as, reg_src, reg_base) ASM_STORE8_REG_REG_OFFSET((as), (reg_src), (reg_base), 0) +#define ASM_STORE8_REG_REG_OFFSET(as, reg_src, reg_base, byte_offset) asm_x64_mov_r8_to_mem8((as), (reg_src), (reg_base), (byte_offset)) +#define ASM_STORE16_REG_REG(as, reg_src, reg_base) ASM_STORE16_REG_REG_OFFSET((as), (reg_src), (reg_base), 0) +#define ASM_STORE16_REG_REG_OFFSET(as, reg_src, reg_base, word_offset) asm_x64_mov_r16_to_mem16((as), (reg_src), (reg_base), 2 * (word_offset)) +#define ASM_STORE32_REG_REG(as, reg_src, reg_base) ASM_STORE32_REG_REG_OFFSET((as), (reg_src), (reg_base), 0) +#define ASM_STORE32_REG_REG_OFFSET(as, reg_src, reg_base, dword_offset) asm_x64_mov_r32_to_mem32((as), (reg_src), (reg_base), 4 * (dword_offset)) #endif // GENERIC_ASM_API diff --git a/py/asmx86.h b/py/asmx86.h index 7796d69762617..f5d37228a2f04 100644 --- a/py/asmx86.h +++ b/py/asmx86.h @@ -150,12 +150,12 @@ void asm_x86_call_ind(asm_x86_t *as, size_t fun_id, mp_uint_t n_args, int temp_r // Holds a pointer to mp_fun_table #define REG_FUN_TABLE ASM_X86_REG_FUN_TABLE -#define ASM_T asm_x86_t -#define ASM_END_PASS asm_x86_end_pass -#define ASM_ENTRY asm_x86_entry -#define ASM_EXIT asm_x86_exit +#define ASM_T asm_x86_t +#define ASM_END_PASS asm_x86_end_pass +#define ASM_ENTRY(as, num_locals, name) asm_x86_entry((as), (num_locals)) +#define ASM_EXIT asm_x86_exit -#define ASM_JUMP asm_x86_jmp_label +#define ASM_JUMP asm_x86_jmp_label #define ASM_JUMP_IF_REG_ZERO(as, reg, label, bool_test) \ do { \ if (bool_test) { \ @@ -201,18 +201,21 @@ void asm_x86_call_ind(asm_x86_t *as, size_t fun_id, mp_uint_t n_args, int temp_r #define ASM_SUB_REG_REG(as, reg_dest, reg_src) asm_x86_sub_r32_r32((as), (reg_dest), (reg_src)) #define ASM_MUL_REG_REG(as, reg_dest, reg_src) asm_x86_mul_r32_r32((as), (reg_dest), (reg_src)) -#define ASM_LOAD_REG_REG(as, reg_dest, reg_base) asm_x86_mov_mem32_to_r32((as), (reg_base), 0, (reg_dest)) -#define ASM_LOAD_REG_REG_OFFSET(as, reg_dest, reg_base, word_offset) asm_x86_mov_mem32_to_r32((as), (reg_base), 4 * (word_offset), (reg_dest)) -#define ASM_LOAD8_REG_REG(as, reg_dest, reg_base) asm_x86_mov_mem8_to_r32zx((as), (reg_base), 0, (reg_dest)) -#define ASM_LOAD16_REG_REG(as, reg_dest, reg_base) asm_x86_mov_mem16_to_r32zx((as), (reg_base), 0, (reg_dest)) -#define ASM_LOAD16_REG_REG_OFFSET(as, reg_dest, reg_base, uint16_offset) asm_x86_mov_mem16_to_r32zx((as), (reg_base), 2 * (uint16_offset), (reg_dest)) -#define ASM_LOAD32_REG_REG(as, reg_dest, reg_base) asm_x86_mov_mem32_to_r32((as), (reg_base), 0, (reg_dest)) - -#define ASM_STORE_REG_REG(as, reg_src, reg_base) asm_x86_mov_r32_to_mem32((as), (reg_src), (reg_base), 0) -#define ASM_STORE_REG_REG_OFFSET(as, reg_src, reg_base, word_offset) asm_x86_mov_r32_to_mem32((as), (reg_src), (reg_base), 4 * (word_offset)) -#define ASM_STORE8_REG_REG(as, reg_src, reg_base) asm_x86_mov_r8_to_mem8((as), (reg_src), (reg_base), 0) -#define ASM_STORE16_REG_REG(as, reg_src, reg_base) asm_x86_mov_r16_to_mem16((as), (reg_src), (reg_base), 0) -#define ASM_STORE32_REG_REG(as, reg_src, reg_base) asm_x86_mov_r32_to_mem32((as), (reg_src), (reg_base), 0) +#define ASM_LOAD_REG_REG_OFFSET(as, reg_dest, reg_base, dword_offset) ASM_LOAD32_REG_REG_OFFSET((as), (reg_dest), (reg_base), (dword_offset)) +#define ASM_LOAD8_REG_REG(as, reg_dest, reg_base) ASM_LOAD8_REG_REG_OFFSET((as), (reg_dest), (reg_base), 0) +#define ASM_LOAD8_REG_REG_OFFSET(as, reg_dest, reg_base, byte_offset) asm_x86_mov_mem8_to_r32zx((as), (reg_base), (byte_offset), (reg_dest)) +#define ASM_LOAD16_REG_REG(as, reg_dest, reg_base) ASM_LOAD16_REG_REG_OFFSET((as), (reg_dest), (reg_base), 0) +#define ASM_LOAD16_REG_REG_OFFSET(as, reg_dest, reg_base, word_offset) asm_x86_mov_mem16_to_r32zx((as), (reg_base), 2 * (word_offset), (reg_dest)) +#define ASM_LOAD32_REG_REG(as, reg_dest, reg_base) ASM_LOAD32_REG_REG_OFFSET((as), (reg_dest), (reg_base), 0) +#define ASM_LOAD32_REG_REG_OFFSET(as, reg_dest, reg_base, dword_offset) asm_x86_mov_mem32_to_r32((as), (reg_base), 4 * (dword_offset), (reg_dest)) + +#define ASM_STORE_REG_REG_OFFSET(as, reg_src, reg_base, dword_offset) ASM_STORE32_REG_REG_OFFSET((as), (reg_src), (reg_base), (dword_offset)) +#define ASM_STORE8_REG_REG(as, reg_src, reg_base) ASM_STORE8_REG_REG_OFFSET((as), (reg_src), (reg_base), 0) +#define ASM_STORE8_REG_REG_OFFSET(as, reg_src, reg_base, byte_offset) asm_x86_mov_r8_to_mem8((as), (reg_src), (reg_base), (byte_offset)) +#define ASM_STORE16_REG_REG(as, reg_src, reg_base) ASM_STORE16_REG_REG_OFFSET((as), (reg_src), (reg_base), 0) +#define ASM_STORE16_REG_REG_OFFSET(as, reg_src, reg_base, word_offset) asm_x86_mov_r16_to_mem16((as), (reg_src), (reg_base), 2 * (word_offset)) +#define ASM_STORE32_REG_REG(as, reg_src, reg_base) ASM_STORE32_REG_REG_OFFSET((as), (reg_src), (reg_base), 0) +#define ASM_STORE32_REG_REG_OFFSET(as, reg_src, reg_base, dword_offset) asm_x86_mov_r32_to_mem32((as), (reg_src), (reg_base), 4 * (dword_offset)) #endif // GENERIC_ASM_API diff --git a/py/asmxtensa.c b/py/asmxtensa.c index 0fbe351dcf3c7..bc3e717d9f3a0 100644 --- a/py/asmxtensa.c +++ b/py/asmxtensa.c @@ -34,9 +34,20 @@ #include "py/asmxtensa.h" +#if N_XTENSAWIN +#define REG_TEMP ASM_XTENSA_REG_TEMPORARY_WIN +#else +#define REG_TEMP ASM_XTENSA_REG_TEMPORARY +#endif + #define WORD_SIZE (4) +#define SIGNED_FIT6(x) ((((x) & 0xffffffe0) == 0) || (((x) & 0xffffffe0) == 0xffffffe0)) #define SIGNED_FIT8(x) ((((x) & 0xffffff80) == 0) || (((x) & 0xffffff80) == 0xffffff80)) #define SIGNED_FIT12(x) ((((x) & 0xfffff800) == 0) || (((x) & 0xfffff800) == 0xfffff800)) +#define SIGNED_FIT18(x) ((((x) & 0xfffe0000) == 0) || (((x) & 0xfffe0000) == 0xfffe0000)) + +#define ET_OUT_OF_RANGE MP_ERROR_TEXT("ERROR: xtensa %q out of range") +#define ET_NOT_ALIGNED MP_ERROR_TEXT("ERROR: %q %q not word-aligned") void asm_xtensa_end_pass(asm_xtensa_t *as) { as->num_const = as->cur_const; @@ -47,9 +58,9 @@ void asm_xtensa_end_pass(asm_xtensa_t *as) { if (as->base.pass == MP_ASM_PASS_EMIT) { uint8_t *d = as->base.code_base; printf("XTENSA ASM:"); - for (int i = 0; i < ((as->base.code_size + 15) & ~15); ++i) { + for (size_t i = 0; i < ((as->base.code_size + 15) & ~15); ++i) { if (i % 16 == 0) { - printf("\n%08x:", (uint32_t)&d[i]); + printf("\n%p:", &d[i]); } if (i % 2 == 0) { printf(" "); @@ -62,10 +73,12 @@ void asm_xtensa_end_pass(asm_xtensa_t *as) { } void asm_xtensa_entry(asm_xtensa_t *as, int num_locals) { - // jump over the constants - asm_xtensa_op_j(as, as->num_const * WORD_SIZE + 4 - 4); - mp_asm_base_get_cur_to_write_bytes(&as->base, 1); // padding/alignment byte - as->const_table = (uint32_t *)mp_asm_base_get_cur_to_write_bytes(&as->base, as->num_const * 4); + if (as->num_const > 0) { + // jump over the constants + asm_xtensa_op_j(as, as->num_const * WORD_SIZE + 4 - 4); + mp_asm_base_get_cur_to_write_bytes(&as->base, 1); // padding/alignment byte + as->const_table = (uint32_t *)mp_asm_base_get_cur_to_write_bytes(&as->base, as->num_const * 4); + } // adjust the stack-pointer to store a0, a12, a13, a14, a15 and locals, 16-byte aligned as->stack_adjust = (((ASM_XTENSA_NUM_REGS_SAVED + num_locals) * WORD_SIZE) + 15) & ~15; @@ -146,22 +159,60 @@ void asm_xtensa_j_label(asm_xtensa_t *as, uint label) { asm_xtensa_op_j(as, rel); } +static bool calculate_branch_displacement(asm_xtensa_t *as, uint label, ptrdiff_t *displacement) { + assert(displacement != NULL && "Displacement pointer is NULL"); + + uint32_t label_offset = get_label_dest(as, label); + *displacement = (ptrdiff_t)(label_offset - as->base.code_offset - 4); + return (label_offset != (uint32_t)-1) && (*displacement < 0); +} + void asm_xtensa_bccz_reg_label(asm_xtensa_t *as, uint cond, uint reg, uint label) { - uint32_t dest = get_label_dest(as, label); - int32_t rel = dest - as->base.code_offset - 4; - if (as->base.pass == MP_ASM_PASS_EMIT && !SIGNED_FIT12(rel)) { - printf("ERROR: xtensa bccz out of range\n"); + ptrdiff_t rel = 0; + bool can_emit_short_jump = calculate_branch_displacement(as, label, &rel); + + if (can_emit_short_jump && SIGNED_FIT12(rel)) { + // Backwards BCCZ opcodes with an offset that fits in 12 bits can + // be emitted without any change. + asm_xtensa_op_bccz(as, cond, reg, rel); + return; } - asm_xtensa_op_bccz(as, cond, reg, rel); + + // Range is effectively extended to 18 bits, as a more complex jump code + // sequence is emitted. + if (as->base.pass == MP_ASM_PASS_EMIT && !SIGNED_FIT18(rel - 6)) { + mp_raise_msg_varg(&mp_type_RuntimeError, ET_OUT_OF_RANGE, MP_QSTR_bccz); + } + + // ~BCCZ skip ; +0 <- Condition is flipped here (EQ -> NE, etc.) + // J addr ; +3 + // skip: ; +6 + asm_xtensa_op_bccz(as, cond ^ 1, reg, 6 - 4); + asm_xtensa_op_j(as, rel - 3); } void asm_xtensa_bcc_reg_reg_label(asm_xtensa_t *as, uint cond, uint reg1, uint reg2, uint label) { - uint32_t dest = get_label_dest(as, label); - int32_t rel = dest - as->base.code_offset - 4; - if (as->base.pass == MP_ASM_PASS_EMIT && !SIGNED_FIT8(rel)) { - printf("ERROR: xtensa bcc out of range\n"); + ptrdiff_t rel = 0; + bool can_emit_short_jump = calculate_branch_displacement(as, label, &rel); + + if (can_emit_short_jump && SIGNED_FIT8(rel)) { + // Backwards BCC opcodes with an offset that fits in 8 bits can + // be emitted without any change. + asm_xtensa_op_bcc(as, cond, reg1, reg2, rel); + return; + } + + // Range is effectively extended to 18 bits, as a more complex jump code + // sequence is emitted. + if (as->base.pass == MP_ASM_PASS_EMIT && !SIGNED_FIT18(rel - 6)) { + mp_raise_msg_varg(&mp_type_RuntimeError, ET_OUT_OF_RANGE, MP_QSTR_bcc); } - asm_xtensa_op_bcc(as, cond, reg1, reg2, rel); + + // ~BCC skip ; +0 <- Condition is flipped here (EQ -> NE, etc.) + // J addr ; +3 + // skip: ; +6 + asm_xtensa_op_bcc(as, cond ^ 8, reg1, reg2, 6 - 4); + asm_xtensa_op_j(as, rel - 3); } // convenience function; reg_dest must be different from reg_src[12] @@ -179,6 +230,8 @@ size_t asm_xtensa_mov_reg_i32(asm_xtensa_t *as, uint reg_dest, uint32_t i32) { // store the constant in the table if (as->const_table != NULL) { as->const_table[as->cur_const] = i32; + } else { + assert((as->base.pass != MP_ASM_PASS_EMIT) && "Constants table was not built."); } ++as->cur_const; return loc; @@ -240,17 +293,53 @@ void asm_xtensa_l32i_optimised(asm_xtensa_t *as, uint reg_dest, uint reg_base, u } else if (word_offset < 256) { asm_xtensa_op_l32i(as, reg_dest, reg_base, word_offset); } else { - mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("asm overflow")); + asm_xtensa_mov_reg_i32_optimised(as, reg_dest, word_offset * 4); + asm_xtensa_op_add_n(as, reg_dest, reg_base, reg_dest); + asm_xtensa_op_l32i_n(as, reg_dest, reg_dest, 0); } } -void asm_xtensa_s32i_optimised(asm_xtensa_t *as, uint reg_src, uint reg_base, uint word_offset) { - if (word_offset < 16) { - asm_xtensa_op_s32i_n(as, reg_src, reg_base, word_offset); - } else if (word_offset < 256) { - asm_xtensa_op_s32i(as, reg_src, reg_base, word_offset); +void asm_xtensa_load_reg_reg_offset(asm_xtensa_t *as, uint reg_dest, uint reg_base, uint offset, uint operation_size) { + assert(operation_size <= 2 && "Operation size value out of range."); + + if (operation_size == 2 && MP_FIT_UNSIGNED(4, offset)) { + asm_xtensa_op_l32i_n(as, reg_dest, reg_base, offset); + return; + } + + if (MP_FIT_UNSIGNED(8, offset)) { + asm_xtensa_op24(as, ASM_XTENSA_ENCODE_RRI8(2, operation_size, reg_base, reg_dest, offset)); + return; + } + + asm_xtensa_mov_reg_i32_optimised(as, reg_dest, offset << operation_size); + asm_xtensa_op_add_n(as, reg_dest, reg_base, reg_dest); + if (operation_size == 2) { + asm_xtensa_op_l32i_n(as, reg_dest, reg_dest, 0); } else { - mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("asm overflow")); + asm_xtensa_op24(as, ASM_XTENSA_ENCODE_RRI8(2, operation_size, reg_dest, reg_dest, 0)); + } +} + +void asm_xtensa_store_reg_reg_offset(asm_xtensa_t *as, uint reg_src, uint reg_base, uint offset, uint operation_size) { + assert(operation_size <= 2 && "Operation size value out of range."); + + if (operation_size == 2 && MP_FIT_UNSIGNED(4, offset)) { + asm_xtensa_op_s32i_n(as, reg_src, reg_base, offset); + return; + } + + if (MP_FIT_UNSIGNED(8, offset)) { + asm_xtensa_op24(as, ASM_XTENSA_ENCODE_RRI8(2, 0x04 | operation_size, reg_base, reg_src, offset)); + return; + } + + asm_xtensa_mov_reg_i32_optimised(as, REG_TEMP, offset << operation_size); + asm_xtensa_op_add_n(as, REG_TEMP, reg_base, REG_TEMP); + if (operation_size == 2) { + asm_xtensa_op_s32i_n(as, reg_src, REG_TEMP, 0); + } else { + asm_xtensa_op24(as, ASM_XTENSA_ENCODE_RRI8(2, 0x04 | operation_size, REG_TEMP, reg_src, 0)); } } @@ -264,4 +353,47 @@ void asm_xtensa_call_ind_win(asm_xtensa_t *as, uint idx) { asm_xtensa_op_callx8(as, ASM_XTENSA_REG_A8); } +void asm_xtensa_bit_branch(asm_xtensa_t *as, mp_uint_t reg, mp_uint_t bit, mp_uint_t label, mp_uint_t condition) { + uint32_t dest = get_label_dest(as, label); + int32_t rel = dest - as->base.code_offset - 4; + if (as->base.pass == MP_ASM_PASS_EMIT && !SIGNED_FIT8(rel)) { + mp_raise_msg_varg(&mp_type_RuntimeError, ET_OUT_OF_RANGE, MP_QSTR_bit_branch); + } + asm_xtensa_op24(as, ASM_XTENSA_ENCODE_RRI8(7, condition | ((bit >> 4) & 0x01), reg, bit & 0x0F, rel & 0xFF)); +} + +void asm_xtensa_call0(asm_xtensa_t *as, mp_uint_t label) { + uint32_t dest = get_label_dest(as, label); + int32_t rel = dest - as->base.code_offset - 3; + if (as->base.pass == MP_ASM_PASS_EMIT) { + if ((dest & 0x03) != 0) { + mp_raise_msg_varg(&mp_type_RuntimeError, ET_NOT_ALIGNED, MP_QSTR_call0, MP_QSTR_target); + } + if ((rel & 0x03) != 0) { + mp_raise_msg_varg(&mp_type_RuntimeError, ET_NOT_ALIGNED, MP_QSTR_call0, MP_QSTR_location); + } + if (!SIGNED_FIT18(rel)) { + mp_raise_msg_varg(&mp_type_RuntimeError, ET_OUT_OF_RANGE, MP_QSTR_call0); + } + } + asm_xtensa_op_call0(as, rel); +} + +void asm_xtensa_l32r(asm_xtensa_t *as, mp_uint_t reg, mp_uint_t label) { + uint32_t dest = get_label_dest(as, label); + int32_t rel = dest - as->base.code_offset; + if (as->base.pass == MP_ASM_PASS_EMIT) { + if ((dest & 0x03) != 0) { + mp_raise_msg_varg(&mp_type_RuntimeError, ET_NOT_ALIGNED, MP_QSTR_l32r, MP_QSTR_target); + } + if ((rel & 0x03) != 0) { + mp_raise_msg_varg(&mp_type_RuntimeError, ET_NOT_ALIGNED, MP_QSTR_l32r, MP_QSTR_location); + } + if (!SIGNED_FIT18(rel) || (rel >= 0)) { + mp_raise_msg_varg(&mp_type_RuntimeError, ET_OUT_OF_RANGE, MP_QSTR_l32r); + } + } + asm_xtensa_op_l32r(as, reg, as->base.code_offset, dest); +} + #endif // MICROPY_EMIT_XTENSA || MICROPY_EMIT_INLINE_XTENSA || MICROPY_EMIT_XTENSAWIN diff --git a/py/asmxtensa.h b/py/asmxtensa.h index a8c39206bd008..ed98c9d730516 100644 --- a/py/asmxtensa.h +++ b/py/asmxtensa.h @@ -64,9 +64,11 @@ #define ASM_XTENSA_REG_A14 (14) #define ASM_XTENSA_REG_A15 (15) -// for bccz +// for bccz and bcci #define ASM_XTENSA_CCZ_EQ (0) #define ASM_XTENSA_CCZ_NE (1) +#define ASM_XTENSA_CCZ_LT (2) +#define ASM_XTENSA_CCZ_GE (3) // for bcc and setcc #define ASM_XTENSA_CC_NONE (0) @@ -291,15 +293,24 @@ void asm_xtensa_mov_local_reg(asm_xtensa_t *as, int local_num, uint reg_src); void asm_xtensa_mov_reg_local(asm_xtensa_t *as, uint reg_dest, int local_num); void asm_xtensa_mov_reg_local_addr(asm_xtensa_t *as, uint reg_dest, int local_num); void asm_xtensa_mov_reg_pcrel(asm_xtensa_t *as, uint reg_dest, uint label); -void asm_xtensa_l32i_optimised(asm_xtensa_t *as, uint reg_dest, uint reg_base, uint word_offset); -void asm_xtensa_s32i_optimised(asm_xtensa_t *as, uint reg_src, uint reg_base, uint word_offset); +void asm_xtensa_load_reg_reg_offset(asm_xtensa_t *as, uint reg_dest, uint reg_base, uint offset, uint operation_size); +void asm_xtensa_store_reg_reg_offset(asm_xtensa_t *as, uint reg_src, uint reg_base, uint offset, uint operation_size); void asm_xtensa_call_ind(asm_xtensa_t *as, uint idx); void asm_xtensa_call_ind_win(asm_xtensa_t *as, uint idx); +void asm_xtensa_bit_branch(asm_xtensa_t *as, mp_uint_t reg, mp_uint_t bit, mp_uint_t label, mp_uint_t condition); +void asm_xtensa_immediate_branch(asm_xtensa_t *as, mp_uint_t reg, mp_uint_t immediate, mp_uint_t label, mp_uint_t cond); +void asm_xtensa_call0(asm_xtensa_t *as, mp_uint_t label); +void asm_xtensa_l32r(asm_xtensa_t *as, mp_uint_t reg, mp_uint_t label); // Holds a pointer to mp_fun_table #define ASM_XTENSA_REG_FUN_TABLE ASM_XTENSA_REG_A15 #define ASM_XTENSA_REG_FUN_TABLE_WIN ASM_XTENSA_REG_A7 +// Internal temporary register (currently aliased to REG_ARG_5 for xtensa, +// and to REG_TEMP2 for xtensawin). +#define ASM_XTENSA_REG_TEMPORARY ASM_XTENSA_REG_A6 +#define ASM_XTENSA_REG_TEMPORARY_WIN ASM_XTENSA_REG_A12 + // CIRCUITPY-CHANGE: prevent #if warning #if defined(GENERIC_ASM_API) && GENERIC_ASM_API @@ -330,9 +341,9 @@ void asm_xtensa_call_ind_win(asm_xtensa_t *as, uint idx); #define ASM_NUM_REGS_SAVED ASM_XTENSA_NUM_REGS_SAVED #define REG_FUN_TABLE ASM_XTENSA_REG_FUN_TABLE -#define ASM_ENTRY(as, nlocal) asm_xtensa_entry((as), (nlocal)) -#define ASM_EXIT(as) asm_xtensa_exit((as)) -#define ASM_CALL_IND(as, idx) asm_xtensa_call_ind((as), (idx)) +#define ASM_ENTRY(as, nlocal, name) asm_xtensa_entry((as), (nlocal)) +#define ASM_EXIT(as) asm_xtensa_exit((as)) +#define ASM_CALL_IND(as, idx) asm_xtensa_call_ind((as), (idx)) #else // Configuration for windowed calls with window size 8 @@ -360,9 +371,9 @@ void asm_xtensa_call_ind_win(asm_xtensa_t *as, uint idx); #define ASM_NUM_REGS_SAVED ASM_XTENSA_NUM_REGS_SAVED_WIN #define REG_FUN_TABLE ASM_XTENSA_REG_FUN_TABLE_WIN -#define ASM_ENTRY(as, nlocal) asm_xtensa_entry_win((as), (nlocal)) -#define ASM_EXIT(as) asm_xtensa_exit_win((as)) -#define ASM_CALL_IND(as, idx) asm_xtensa_call_ind_win((as), (idx)) +#define ASM_ENTRY(as, nlocal, name) asm_xtensa_entry_win((as), (nlocal)) +#define ASM_EXIT(as) asm_xtensa_exit_win((as)) +#define ASM_CALL_IND(as, idx) asm_xtensa_call_ind_win((as), (idx)) #endif @@ -408,16 +419,51 @@ void asm_xtensa_call_ind_win(asm_xtensa_t *as, uint idx); #define ASM_SUB_REG_REG(as, reg_dest, reg_src) asm_xtensa_op_sub((as), (reg_dest), (reg_dest), (reg_src)) #define ASM_MUL_REG_REG(as, reg_dest, reg_src) asm_xtensa_op_mull((as), (reg_dest), (reg_dest), (reg_src)) -#define ASM_LOAD_REG_REG_OFFSET(as, reg_dest, reg_base, word_offset) asm_xtensa_l32i_optimised((as), (reg_dest), (reg_base), (word_offset)) -#define ASM_LOAD8_REG_REG(as, reg_dest, reg_base) asm_xtensa_op_l8ui((as), (reg_dest), (reg_base), 0) -#define ASM_LOAD16_REG_REG(as, reg_dest, reg_base) asm_xtensa_op_l16ui((as), (reg_dest), (reg_base), 0) -#define ASM_LOAD16_REG_REG_OFFSET(as, reg_dest, reg_base, uint16_offset) asm_xtensa_op_l16ui((as), (reg_dest), (reg_base), (uint16_offset)) -#define ASM_LOAD32_REG_REG(as, reg_dest, reg_base) asm_xtensa_op_l32i_n((as), (reg_dest), (reg_base), 0) +#define ASM_LOAD_REG_REG_OFFSET(as, reg_dest, reg_base, word_offset) ASM_LOAD32_REG_REG_OFFSET((as), (reg_dest), (reg_base), (word_offset)) +#define ASM_LOAD8_REG_REG(as, reg_dest, reg_base) ASM_LOAD8_REG_REG_OFFSET((as), (reg_dest), (reg_base), 0) +#define ASM_LOAD8_REG_REG_OFFSET(as, reg_dest, reg_base, byte_offset) asm_xtensa_load_reg_reg_offset((as), (reg_dest), (reg_base), (byte_offset), 0) +#define ASM_LOAD8_REG_REG_REG(as, reg_dest, reg_base, reg_index) \ + do { \ + asm_xtensa_op_add_n((as), (reg_base), (reg_index), (reg_base)); \ + asm_xtensa_op_l8ui((as), (reg_dest), (reg_base), 0); \ + } while (0) +#define ASM_LOAD16_REG_REG(as, reg_dest, reg_base) ASM_LOAD16_REG_REG_OFFSET((as), (reg_dest), (reg_base), 0) +#define ASM_LOAD16_REG_REG_OFFSET(as, reg_dest, reg_base, halfword_offset) asm_xtensa_load_reg_reg_offset((as), (reg_dest), (reg_base), (halfword_offset), 1) +#define ASM_LOAD16_REG_REG_REG(as, reg_dest, reg_base, reg_index) \ + do { \ + asm_xtensa_op_addx2((as), (reg_base), (reg_index), (reg_base)); \ + asm_xtensa_op_l16ui((as), (reg_dest), (reg_base), 0); \ + } while (0) +#define ASM_LOAD32_REG_REG(as, reg_dest, reg_base) ASM_LOAD32_REG_REG_OFFSET((as), (reg_dest), (reg_base), 0) +#define ASM_LOAD32_REG_REG_OFFSET(as, reg_dest, reg_base, word_offset) asm_xtensa_load_reg_reg_offset((as), (reg_dest), (reg_base), (word_offset), 2) +#define ASM_LOAD32_REG_REG_REG(as, reg_dest, reg_base, reg_index) \ + do { \ + asm_xtensa_op_addx4((as), (reg_base), (reg_index), (reg_base)); \ + asm_xtensa_op_l32i_n((as), (reg_dest), (reg_base), 0); \ + } while (0) -#define ASM_STORE_REG_REG_OFFSET(as, reg_dest, reg_base, word_offset) asm_xtensa_s32i_optimised((as), (reg_dest), (reg_base), (word_offset)) -#define ASM_STORE8_REG_REG(as, reg_src, reg_base) asm_xtensa_op_s8i((as), (reg_src), (reg_base), 0) -#define ASM_STORE16_REG_REG(as, reg_src, reg_base) asm_xtensa_op_s16i((as), (reg_src), (reg_base), 0) -#define ASM_STORE32_REG_REG(as, reg_src, reg_base) asm_xtensa_op_s32i_n((as), (reg_src), (reg_base), 0) +#define ASM_STORE_REG_REG_OFFSET(as, reg_dest, reg_base, word_offset) ASM_STORE32_REG_REG_OFFSET((as), (reg_dest), (reg_base), (word_offset)) +#define ASM_STORE8_REG_REG(as, reg_src, reg_base) ASM_STORE8_REG_REG_OFFSET((as), (reg_src), (reg_base), 0) +#define ASM_STORE8_REG_REG_OFFSET(as, reg_src, reg_base, byte_offset) asm_xtensa_store_reg_reg_offset((as), (reg_src), (reg_base), (byte_offset), 0) +#define ASM_STORE8_REG_REG_REG(as, reg_val, reg_base, reg_index) \ + do { \ + asm_xtensa_op_add_n((as), (reg_base), (reg_index), (reg_base)); \ + asm_xtensa_op_s8i((as), (reg_val), (reg_base), 0); \ + } while (0) +#define ASM_STORE16_REG_REG(as, reg_src, reg_base) ASM_STORE16_REG_REG_OFFSET((as), (reg_src), (reg_base), 0) +#define ASM_STORE16_REG_REG_OFFSET(as, reg_src, reg_base, halfword_offset) asm_xtensa_store_reg_reg_offset((as), (reg_src), (reg_base), (halfword_offset), 1) +#define ASM_STORE16_REG_REG_REG(as, reg_val, reg_base, reg_index) \ + do { \ + asm_xtensa_op_addx2((as), (reg_base), (reg_index), (reg_base)); \ + asm_xtensa_op_s16i((as), (reg_val), (reg_base), 0); \ + } while (0) +#define ASM_STORE32_REG_REG(as, reg_src, reg_base) ASM_STORE32_REG_REG_OFFSET((as), (reg_src), (reg_base), 0) +#define ASM_STORE32_REG_REG_OFFSET(as, reg_dest, reg_base, word_offset) asm_xtensa_store_reg_reg_offset((as), (reg_dest), (reg_base), (word_offset), 2) +#define ASM_STORE32_REG_REG_REG(as, reg_val, reg_base, reg_index) \ + do { \ + asm_xtensa_op_addx4((as), (reg_base), (reg_index), (reg_base)); \ + asm_xtensa_op_s32i_n((as), (reg_val), (reg_base), 0); \ + } while (0) #endif // GENERIC_ASM_API diff --git a/py/bc.c b/py/bc.c index c2956030e38e1..4caa022e3d8b4 100644 --- a/py/bc.c +++ b/py/bc.c @@ -88,7 +88,7 @@ const byte *mp_decode_uint_skip(const byte *ptr) { return ptr; } -static NORETURN void fun_pos_args_mismatch(mp_obj_fun_bc_t *f, size_t expected, size_t given) { +static MP_NORETURN void fun_pos_args_mismatch(mp_obj_fun_bc_t *f, size_t expected, size_t given) { #if MICROPY_ERROR_REPORTING <= MICROPY_ERROR_REPORTING_TERSE // generic message, used also for other argument issues (void)f; diff --git a/py/bc.h b/py/bc.h index 7658f66414f17..3b7a6fe1c960b 100644 --- a/py/bc.h +++ b/py/bc.h @@ -226,6 +226,7 @@ typedef struct _mp_compiled_module_t { bool has_native; size_t n_qstr; size_t n_obj; + size_t arch_flags; #endif } mp_compiled_module_t; @@ -303,7 +304,7 @@ static inline void mp_module_context_alloc_tables(mp_module_context_t *context, size_t nq = (n_qstr * sizeof(qstr_short_t) + sizeof(mp_uint_t) - 1) / sizeof(mp_uint_t); size_t no = n_obj; // CIRCUITPY-CHANGE - mp_uint_t *mem = m_malloc_items(nq + no); + mp_uint_t *mem = (mp_uint_t *)m_malloc_items(nq + no); context->constants.qstr_table = (qstr_short_t *)mem; context->constants.obj_table = (mp_obj_t *)(mem + nq); #else @@ -316,25 +317,35 @@ static inline void mp_module_context_alloc_tables(mp_module_context_t *context, #endif } +typedef struct _mp_code_lineinfo_t { + size_t bc_increment; + size_t line_increment; +} mp_code_lineinfo_t; + +static inline mp_code_lineinfo_t mp_bytecode_decode_lineinfo(const byte **line_info) { + mp_code_lineinfo_t result; + size_t c = (*line_info)[0]; + if ((c & 0x80) == 0) { + // 0b0LLBBBBB encoding + result.bc_increment = c & 0x1f; + result.line_increment = c >> 5; + *line_info += 1; + } else { + // 0b1LLLBBBB 0bLLLLLLLL encoding (l's LSB in second byte) + result.bc_increment = c & 0xf; + result.line_increment = ((c << 4) & 0x700) | (*line_info)[1]; + *line_info += 2; + } + return result; +} + static inline size_t mp_bytecode_get_source_line(const byte *line_info, const byte *line_info_top, size_t bc_offset) { size_t source_line = 1; while (line_info < line_info_top) { - size_t c = *line_info; - size_t b, l; - if ((c & 0x80) == 0) { - // 0b0LLBBBBB encoding - b = c & 0x1f; - l = c >> 5; - line_info += 1; - } else { - // 0b1LLLBBBB 0bLLLLLLLL encoding (l's LSB in second byte) - b = c & 0xf; - l = ((c << 4) & 0x700) | line_info[1]; - line_info += 2; - } - if (bc_offset >= b) { - bc_offset -= b; - source_line += l; + mp_code_lineinfo_t decoded = mp_bytecode_decode_lineinfo(&line_info); + if (bc_offset >= decoded.bc_increment) { + bc_offset -= decoded.bc_increment; + source_line += decoded.line_increment; } else { // found source line corresponding to bytecode offset break; diff --git a/py/binary.c b/py/binary.c index 728f29815cd78..180a13beec7bf 100644 --- a/py/binary.c +++ b/py/binary.c @@ -71,8 +71,7 @@ size_t mp_binary_get_size(char struct_type, char val_type, size_t *palign) { case 'Q': size = 8; break; - // CIRCUITPY-CHANGE: non-standard typecodes can be turned off - #if MICROPY_NONSTANDARD_TYPECODES + #if MICROPY_PY_STRUCT_UNSAFE_TYPECODES case 'P': case 'O': case 'S': @@ -128,8 +127,7 @@ size_t mp_binary_get_size(char struct_type, char val_type, size_t *palign) { align = alignof(long long); size = sizeof(long long); break; - // CIRCUITPY-CHANGE: non-standard typecodes can be turned off - #if MICROPY_NONSTANDARD_TYPECODES + #if MICROPY_PY_STRUCT_UNSAFE_TYPECODES case 'P': case 'O': case 'S': @@ -208,7 +206,7 @@ static float mp_decode_half_float(uint16_t hf) { ++e; } - fpu.i = ((hf & 0x8000) << 16) | (e << 23) | (m << 13); + fpu.i = ((hf & 0x8000u) << 16) | (e << 23) | (m << 13); return fpu.f; } @@ -289,20 +287,17 @@ mp_obj_t mp_binary_get_val_array(char typecode, void *p, size_t index) { #if MICROPY_PY_BUILTINS_FLOAT case 'f': return mp_obj_new_float_from_f(((float *)p)[index]); - #if MICROPY_PY_DOUBLE_TYPECODE case 'd': return mp_obj_new_float_from_d(((double *)p)[index]); #endif - #endif - // CIRCUITPY-CHANGE: non-standard typecodes can be turned off - #if MICROPY_NONSTANDARD_TYPECODES - // Extension to CPython: array of objects + // Extension to CPython: array of objects + #if MICROPY_PY_STRUCT_UNSAFE_TYPECODES case 'O': return ((mp_obj_t *)p)[index]; // Extension to CPython: array of pointers case 'P': return mp_obj_new_int((mp_int_t)(uintptr_t)((void **)p)[index]); - #endif + #endif } return MP_OBJ_NEW_SMALL_INT(val); } @@ -352,14 +347,11 @@ mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte *p_base, byte * long long val = mp_binary_get_int(size, is_signed(val_type), (struct_type == '>'), p); - // CIRCUITPY-CHANGE: non-standard typecodes can be turned off - if (MICROPY_NONSTANDARD_TYPECODES && (val_type == 'O')) { + if (MICROPY_PY_STRUCT_UNSAFE_TYPECODES && val_type == 'O') { return (mp_obj_t)(mp_uint_t)val; - #if MICROPY_NONSTANDARD_TYPECODES - } else if (val_type == 'S') { + } else if (MICROPY_PY_STRUCT_UNSAFE_TYPECODES && val_type == 'S') { const char *s_val = (const char *)(uintptr_t)(mp_uint_t)val; return mp_obj_new_str_from_cstr(s_val); - #endif #if MICROPY_PY_BUILTINS_FLOAT } else if (val_type == 'e') { return mp_obj_new_float_from_f(mp_decode_half_float(val)); @@ -369,7 +361,6 @@ mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte *p_base, byte * float f; } fpu = {val}; return mp_obj_new_float_from_f(fpu.f); - #if MICROPY_PY_DOUBLE_TYPECODE } else if (val_type == 'd') { union { uint64_t i; @@ -377,7 +368,6 @@ mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte *p_base, byte * } fpu = {val}; return mp_obj_new_float_from_d(fpu.f); #endif - #endif } else if (is_signed(val_type)) { if ((long long)MP_SMALL_INT_MIN <= val && val <= (long long)MP_SMALL_INT_MAX) { return mp_obj_new_int((mp_int_t)val); @@ -430,8 +420,7 @@ void mp_binary_set_val(char struct_type, char val_type, mp_obj_t val_in, byte *p mp_uint_t val; switch (val_type) { - // CIRCUITPY-CHANGE: non-standard typecodes can be turned off - #if MICROPY_NONSTANDARD_TYPECODES + #if MICROPY_PY_STRUCT_UNSAFE_TYPECODES case 'O': val = (mp_uint_t)val_in; break; @@ -449,6 +438,7 @@ void mp_binary_set_val(char struct_type, char val_type, mp_obj_t val_in, byte *p val = fp_sp.i; break; } + // CIRCUITPY-CHANGE #if MICROPY_PY_DOUBLE_TYPECODE case 'd': { union { @@ -507,14 +497,11 @@ void mp_binary_set_val_array(char typecode, void *p, size_t index, mp_obj_t val_ case 'f': ((float *)p)[index] = mp_obj_get_float_to_f(val_in); break; - #if MICROPY_PY_DOUBLE_TYPECODE case 'd': ((double *)p)[index] = mp_obj_get_float_to_d(val_in); break; #endif - #endif - // CIRCUITPY-CHANGE: non-standard typecodes can be turned off - #if MICROPY_NONSTANDARD_TYPECODES + #if MICROPY_PY_STRUCT_UNSAFE_TYPECODES // Extension to CPython: array of objects case 'O': ((mp_obj_t *)p)[index] = val_in; @@ -582,18 +569,15 @@ void mp_binary_set_val_array_from_int(char typecode, void *p, size_t index, mp_i case 'f': ((float *)p)[index] = (float)val; break; - #if MICROPY_PY_DOUBLE_TYPECODE case 'd': ((double *)p)[index] = (double)val; break; #endif - #endif - // CIRCUITPY-CHANGE: non-standard typecodes can be turned off - #if MICROPY_NONSTANDARD_TYPECODES - // Extension to CPython: array of pointers + // Extension to CPython: array of pointers + #if MICROPY_PY_STRUCT_UNSAFE_TYPECODES case 'P': ((void **)p)[index] = (void *)(uintptr_t)val; break; - #endif + #endif } } diff --git a/py/builtin.h b/py/builtin.h index 6efe3e8facabd..388bc8470053d 100644 --- a/py/builtin.h +++ b/py/builtin.h @@ -138,6 +138,7 @@ extern const mp_obj_module_t mp_module_sys; extern const mp_obj_module_t mp_module_errno; extern const mp_obj_module_t mp_module_uctypes; extern const mp_obj_module_t mp_module_machine; +extern const mp_obj_module_t mp_module_math; extern const char MICROPY_PY_BUILTINS_HELP_TEXT[]; diff --git a/py/builtinhelp.c b/py/builtinhelp.c index d041d6915a63a..9ab2a12d21455 100644 --- a/py/builtinhelp.c +++ b/py/builtinhelp.c @@ -162,9 +162,8 @@ static void mp_help_print_obj(const mp_obj_t obj) { } if (map != NULL) { for (uint i = 0; i < map->alloc; i++) { - mp_obj_t key = map->table[i].key; - if (key != MP_OBJ_NULL) { - mp_help_print_info_about_object(key, map->table[i].value); + if (mp_map_slot_is_filled(map, i)) { + mp_help_print_info_about_object(map->table[i].key, map->table[i].value); } } } diff --git a/py/builtinimport.c b/py/builtinimport.c index e4e06d8e2cc8b..8fcac22ccb491 100644 --- a/py/builtinimport.c +++ b/py/builtinimport.c @@ -118,7 +118,7 @@ static mp_import_stat_t stat_module(vstr_t *path) { // path (i.e. "/mod_name(.py)"). static mp_import_stat_t stat_top_level(qstr mod_name, vstr_t *dest) { DEBUG_printf("stat_top_level: '%s'\n", qstr_str(mod_name)); - #if MICROPY_PY_SYS + #if MICROPY_PY_SYS && MICROPY_PY_SYS_PATH size_t path_num; mp_obj_t *path_items; mp_obj_get_array(mp_sys_path, &path_num, &path_items); @@ -155,7 +155,7 @@ static mp_import_stat_t stat_top_level(qstr mod_name, vstr_t *dest) { #if MICROPY_MODULE_FROZEN_STR || MICROPY_ENABLE_COMPILER static void do_load_from_lexer(mp_module_context_t *context, mp_lexer_t *lex) { - #if MICROPY_PY___FILE__ + #if MICROPY_MODULE___FILE__ qstr source_name = lex->source_name; mp_store_attr(MP_OBJ_FROM_PTR(&context->module), MP_QSTR___file__, MP_OBJ_NEW_QSTR(source_name)); #endif @@ -168,7 +168,7 @@ static void do_load_from_lexer(mp_module_context_t *context, mp_lexer_t *lex) { #if (MICROPY_HAS_FILE_READER && MICROPY_PERSISTENT_CODE_LOAD) || MICROPY_MODULE_FROZEN_MPY static void do_execute_proto_fun(const mp_module_context_t *context, mp_proto_fun_t proto_fun, qstr source_name) { - #if MICROPY_PY___FILE__ + #if MICROPY_MODULE___FILE__ mp_store_attr(MP_OBJ_FROM_PTR(&context->module), MP_QSTR___file__, MP_OBJ_NEW_QSTR(source_name)); #else (void)source_name; @@ -227,7 +227,7 @@ static void do_load(mp_module_context_t *module_obj, vstr_t *file) { if (frozen_type == MP_FROZEN_MPY) { const mp_frozen_module_t *frozen = modref; module_obj->constants = frozen->constants; - #if MICROPY_PY___FILE__ + #if MICROPY_MODULE___FILE__ qstr frozen_file_qstr = qstr_from_str(file_str + frozen_path_prefix_len); #else qstr frozen_file_qstr = MP_QSTRnull; @@ -270,7 +270,7 @@ static void do_load(mp_module_context_t *module_obj, vstr_t *file) { // Convert a relative (to the current module) import, going up "level" levels, // into an absolute import. -static void evaluate_relative_import(mp_int_t level, const char **module_name, size_t *module_name_len) { +static void evaluate_relative_import(mp_int_t level, const char **module_name, size_t *module_name_len, mp_obj_t globals) { // What we want to do here is to take the name of the current module, // remove trailing components, and concatenate the passed-in // module name. @@ -279,7 +279,7 @@ static void evaluate_relative_import(mp_int_t level, const char **module_name, s // module's position in the package hierarchy." // http://legacy.python.org/dev/peps/pep-0328/#relative-imports-and-name - mp_obj_t current_module_name_obj = mp_obj_dict_get(MP_OBJ_FROM_PTR(mp_globals_get()), MP_OBJ_NEW_QSTR(MP_QSTR___name__)); + mp_obj_t current_module_name_obj = mp_obj_dict_get(globals, MP_OBJ_NEW_QSTR(MP_QSTR___name__)); assert(current_module_name_obj != MP_OBJ_NULL); #if MICROPY_MODULE_OVERRIDE_MAIN_IMPORT && MICROPY_CPYTHON_COMPAT @@ -287,12 +287,12 @@ static void evaluate_relative_import(mp_int_t level, const char **module_name, s // This is a module loaded by -m command-line switch (e.g. unix port), // and so its __name__ has been set to "__main__". Get its real name // that we stored during import in the __main__ attribute. - current_module_name_obj = mp_obj_dict_get(MP_OBJ_FROM_PTR(mp_globals_get()), MP_OBJ_NEW_QSTR(MP_QSTR___main__)); + current_module_name_obj = mp_obj_dict_get(globals, MP_OBJ_NEW_QSTR(MP_QSTR___main__)); } #endif // If we have a __path__ in the globals dict, then we're a package. - bool is_pkg = mp_map_lookup(&mp_globals_get()->map, MP_OBJ_NEW_QSTR(MP_QSTR___path__), MP_MAP_LOOKUP); + bool is_pkg = mp_map_lookup(mp_obj_dict_get_map(globals), MP_OBJ_NEW_QSTR(MP_QSTR___path__), MP_MAP_LOOKUP); #if DEBUG_PRINT DEBUG_printf("Current module/package: "); @@ -370,7 +370,7 @@ static mp_obj_t process_import_at_level(qstr full_mod_name, qstr level_mod_name, // Immediately return if the module at this level is already loaded. mp_map_elem_t *elem; - #if MICROPY_PY_SYS + #if MICROPY_PY_SYS && MICROPY_PY_SYS_PATH // If sys.path is empty, the intention is to force using a built-in. This // means we should also ignore any loaded modules with the same name // which may have come from the filesystem. @@ -406,6 +406,7 @@ static mp_obj_t process_import_at_level(qstr full_mod_name, qstr level_mod_name, // all the locations in sys.path. stat = stat_top_level(level_mod_name, &path); + #if MICROPY_HAVE_REGISTERED_EXTENSIBLE_MODULES // If filesystem failed, now try and see if it matches an extensible // built-in module. if (stat == MP_IMPORT_STAT_NO_EXIST) { @@ -414,6 +415,7 @@ static mp_obj_t process_import_at_level(qstr full_mod_name, qstr level_mod_name, return module_obj; } } + #endif } else { DEBUG_printf("Searching for sub-module\n"); @@ -577,10 +579,19 @@ mp_obj_t mp_builtin___import___default(size_t n_args, const mp_obj_t *args) { const char *module_name = mp_obj_str_get_data(module_name_obj, &module_name_len); if (level != 0) { + // This is the dict with all global symbols. + mp_obj_t globals = MP_OBJ_FROM_PTR(mp_globals_get()); + if (n_args >= 2 && args[1] != mp_const_none) { + globals = args[1]; + if (!mp_obj_is_type(globals, &mp_type_dict)) { + mp_raise_TypeError(NULL); + } + } + // Turn "foo.bar" with level=3 into ".foo.bar". // Current module name is extracted from globals().__name__. - evaluate_relative_import(level, &module_name, &module_name_len); // module_name is now an absolute module path. + evaluate_relative_import(level, &module_name, &module_name_len, globals); } if (module_name_len == 0) { @@ -656,11 +667,13 @@ mp_obj_t mp_builtin___import___default(size_t n_args, const mp_obj_t *args) { if (module_obj != MP_OBJ_NULL) { return module_obj; } + #if MICROPY_HAVE_REGISTERED_EXTENSIBLE_MODULES // Now try as an extensible built-in (e.g. `time`). module_obj = mp_module_get_builtin(module_name_qstr, true); if (module_obj != MP_OBJ_NULL) { return module_obj; } + #endif // Couldn't find the module, so fail #if MICROPY_ERROR_REPORTING <= MICROPY_ERROR_REPORTING_TERSE diff --git a/py/circuitpy_mpconfig.h b/py/circuitpy_mpconfig.h index 3918296636afa..4b23784c5416c 100644 --- a/py/circuitpy_mpconfig.h +++ b/py/circuitpy_mpconfig.h @@ -17,6 +17,10 @@ // Always 1: defined in circuitpy_mpconfig.mk // #define CIRCUITPY (1) +#ifndef MP_SSIZE_MAX +#define MP_SSIZE_MAX (0x7fffffff) +#endif + // REPR_C encodes qstrs, 31-bit ints, and 30-bit floats in a single 32-bit word. #ifndef MICROPY_OBJ_REPR #define MICROPY_OBJ_REPR (MICROPY_OBJ_REPR_C) @@ -42,10 +46,14 @@ extern void common_hal_mcu_enable_interrupts(void); #define MICROPY_PY_BLUETOOTH (0) #define MICROPY_PY_LWIP_SLIP (0) #define MICROPY_PY_OS_DUPTERM (0) +#define MICROPY_PYEXEC_COMPILE_ONLY (0) #define MICROPY_ROM_TEXT_COMPRESSION (0) #define MICROPY_VFS_LFS1 (0) #define MICROPY_VFS_LFS2 (0) +// Always turn on exit code handling +#define MICROPY_PYEXEC_ENABLE_EXIT_CODE_HANDLING (1) + #ifndef MICROPY_GCREGS_SETJMP #define MICROPY_GCREGS_SETJMP (0) #endif @@ -88,7 +96,6 @@ extern void common_hal_mcu_enable_interrupts(void); #define MICROPY_MEM_STATS (0) #define MICROPY_MODULE_BUILTIN_INIT (1) #define MICROPY_MODULE_BUILTIN_SUBPACKAGES (1) -#define MICROPY_NONSTANDARD_TYPECODES (0) #define MICROPY_OPT_COMPUTED_GOTO (1) #define MICROPY_OPT_COMPUTED_GOTO_SAVE_SPACE (CIRCUITPY_COMPUTED_GOTO_SAVE_SPACE) #define MICROPY_OPT_LOAD_ATTR_FAST_PATH (CIRCUITPY_OPT_LOAD_ATTR_FAST_PATH) @@ -141,6 +148,7 @@ extern void common_hal_mcu_enable_interrupts(void); #define MICROPY_PY_RE (CIRCUITPY_RE) // Supplanted by shared-bindings/struct #define MICROPY_PY_STRUCT (0) +#define MICROPY_PY_STRUCT_UNSAFE_TYPECODES (0) #define MICROPY_PY_SYS (CIRCUITPY_SYS) #define MICROPY_PY_SYS_MAXSIZE (1) #define MICROPY_PY_SYS_STDFILES (1) @@ -193,21 +201,6 @@ extern void common_hal_mcu_enable_interrupts(void); // Track stack usage. Expose results via ustack module. #define MICROPY_MAX_STACK_USAGE (0) -#define UINT_FMT "%u" -#define INT_FMT "%d" -#ifdef __LP64__ -typedef long mp_int_t; // must be pointer size -typedef unsigned long mp_uint_t; // must be pointer size -#else -// These are definitions for machines where sizeof(int) == sizeof(void*), -// regardless of actual size. -typedef int mp_int_t; // must be pointer size -typedef unsigned int mp_uint_t; // must be pointer size -#endif -#if __GNUC__ >= 10 // on recent gcc versions we can check that this is so -_Static_assert(sizeof(mp_int_t) == sizeof(void *)); -_Static_assert(sizeof(mp_uint_t) == sizeof(void *)); -#endif typedef long mp_off_t; #define MP_PLAT_PRINT_STRN(str, len) mp_hal_stdout_tx_strn_cooked(str, len) @@ -303,12 +296,10 @@ typedef long mp_off_t; #ifdef LONGINT_IMPL_MPZ #define MICROPY_LONGINT_IMPL (MICROPY_LONGINT_IMPL_MPZ) -#define MP_SSIZE_MAX (0x7fffffff) #endif #ifdef LONGINT_IMPL_LONGLONG #define MICROPY_LONGINT_IMPL (MICROPY_LONGINT_IMPL_LONGLONG) -#define MP_SSIZE_MAX (0x7fffffff) #endif #ifndef MICROPY_PY_REVERSE_SPECIAL_METHODS diff --git a/py/compile.c b/py/compile.c index c7dd2fd476136..c8704131234f1 100644 --- a/py/compile.c +++ b/py/compile.c @@ -103,6 +103,7 @@ static const emit_method_table_t *emit_native_table[] = { &emit_native_xtensa_method_table, &emit_native_xtensawin_method_table, &emit_native_rv32_method_table, + NULL, &emit_native_debug_method_table, }; @@ -3295,7 +3296,9 @@ static void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind } // check structure of parse node - assert(MP_PARSE_NODE_IS_STRUCT(pns2->nodes[0])); + if (!MP_PARSE_NODE_IS_STRUCT(pns2->nodes[0])) { + goto not_an_instruction; + } if (!MP_PARSE_NODE_IS_NULL(pns2->nodes[1])) { goto not_an_instruction; } @@ -3587,6 +3590,13 @@ void mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool case MP_EMIT_OPT_NATIVE_PYTHON: case MP_EMIT_OPT_VIPER: if (emit_native == NULL) { + // The check looks like this to work around a false + // warning in GCC 13 (and possibly later), where it + // assumes that the check will always fail. + if ((uintptr_t)NATIVE_EMITTER_TABLE == (uintptr_t)NULL) { + comp->compile_error = mp_obj_new_exception_msg(&mp_type_NotImplementedError, MP_ERROR_TEXT("cannot emit native code for this architecture")); + goto emit_finished; + } emit_native = NATIVE_EMITTER(new)(&comp->emit_common, &comp->compile_error, &comp->next_label, max_num_labels); } comp->emit_method_table = NATIVE_EMITTER_TABLE; @@ -3619,6 +3629,10 @@ void mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool } } + #if MICROPY_EMIT_NATIVE +emit_finished: + #endif + if (comp->compile_error != MP_OBJ_NULL) { // if there is no line number for the error then use the line // number for the start of this scope diff --git a/py/cstack.h b/py/cstack.h index b12a18e13fcad..bcd919d31f932 100644 --- a/py/cstack.h +++ b/py/cstack.h @@ -35,7 +35,7 @@ void mp_cstack_init_with_sp_here(size_t stack_size); -inline static void mp_cstack_init_with_top(void *top, size_t stack_size) { +static inline void mp_cstack_init_with_top(void *top, size_t stack_size) { MP_STATE_THREAD(stack_top) = (char *)top; #if MICROPY_STACK_CHECK @@ -54,7 +54,7 @@ void mp_cstack_check(void); #else -inline static void mp_cstack_check(void) { +static inline void mp_cstack_check(void) { // No-op when stack checking is disabled } diff --git a/py/dynruntime.h b/py/dynruntime.h index e87cf6591c4b6..ab155a1387579 100644 --- a/py/dynruntime.h +++ b/py/dynruntime.h @@ -70,7 +70,7 @@ #define m_realloc(ptr, new_num_bytes) (m_realloc_dyn((ptr), (new_num_bytes))) #define m_realloc_maybe(ptr, new_num_bytes, allow_move) (m_realloc_maybe_dyn((ptr), (new_num_bytes), (allow_move))) -static NORETURN inline void m_malloc_fail_dyn(size_t num_bytes) { +static MP_NORETURN inline void m_malloc_fail_dyn(size_t num_bytes) { mp_fun_table.raise_msg( mp_fun_table.load_global(MP_QSTR_MemoryError), "memory allocation failed"); @@ -289,7 +289,7 @@ static inline void *mp_obj_malloc_helper_dyn(size_t num_bytes, const mp_obj_type #define nlr_raise(o) (mp_raise_dyn(o)) #define mp_raise_type_arg(type, arg) (mp_raise_dyn(mp_obj_new_exception_arg1_dyn((type), (arg)))) -// CIRCUITPY-CHANGE: use str +// CIRCUITPY-CHANGE: use mp_raise_msg_str #define mp_raise_msg(type, msg) (mp_fun_table.raise_msg_str((type), (msg))) #define mp_raise_OSError(er) (mp_raise_OSError_dyn(er)) #define mp_raise_NotImplementedError(msg) (mp_raise_msg(&mp_type_NotImplementedError, (msg))) @@ -301,14 +301,14 @@ static inline mp_obj_t mp_obj_new_exception_arg1_dyn(const mp_obj_type_t *exc_ty return mp_call_function_n_kw(MP_OBJ_FROM_PTR(exc_type), 1, 0, &args[0]); } -static NORETURN inline void mp_raise_dyn(mp_obj_t o) { +static MP_NORETURN inline void mp_raise_dyn(mp_obj_t o) { mp_fun_table.raise(o); for (;;) { } } // CIRCUITPY-CHANGE: new routine -static NORETURN inline void mp_raise_arg1(const mp_obj_type_t *exc_type, mp_obj_t arg) { +static MP_NORETURN inline void mp_raise_arg1(const mp_obj_type_t *exc_type, mp_obj_t arg) { mp_fun_table.raise(mp_obj_new_exception_arg1_dyn(exc_type, arg)); for (;;) { } diff --git a/py/dynruntime.mk b/py/dynruntime.mk index 807befb464a84..030728cfc9adf 100644 --- a/py/dynruntime.mk +++ b/py/dynruntime.mk @@ -63,14 +63,14 @@ else ifeq ($(ARCH),armv6m) # thumb CROSS = arm-none-eabi- CFLAGS_ARCH += -mthumb -mcpu=cortex-m0 -MICROPY_FLOAT_IMPL ?= none +MICROPY_FLOAT_IMPL ?= float else ifeq ($(ARCH),armv7m) # thumb CROSS = arm-none-eabi- CFLAGS_ARCH += -mthumb -mcpu=cortex-m3 -MICROPY_FLOAT_IMPL ?= none +MICROPY_FLOAT_IMPL ?= float else ifeq ($(ARCH),armv7emsp) @@ -124,6 +124,10 @@ else $(error architecture '$(ARCH)' not supported) endif +ifneq ($(findstring -musl,$(shell $(CROSS)gcc -dumpmachine)),) +USE_MUSL := 1 +endif + MICROPY_FLOAT_IMPL_UPPER = $(shell echo $(MICROPY_FLOAT_IMPL) | tr '[:lower:]' '[:upper:]') CFLAGS += $(CFLAGS_ARCH) -DMICROPY_FLOAT_IMPL=MICROPY_FLOAT_IMPL_$(MICROPY_FLOAT_IMPL_UPPER) @@ -147,6 +151,8 @@ ifeq ($(LINK_RUNTIME),1) # distribution. ifeq ($(USE_PICOLIBC),1) LIBM_NAME := libc.a +else ifeq ($(USE_MUSL),1) +LIBM_NAME := libc.a else LIBM_NAME := libm.a endif @@ -166,6 +172,9 @@ endif endif MPY_LD_FLAGS += $(addprefix -l, $(LIBGCC_PATH) $(LIBM_PATH)) endif +ifneq ($(MPY_EXTERN_SYM_FILE),) +MPY_LD_FLAGS += --externs "$(realpath $(MPY_EXTERN_SYM_FILE))" +endif CFLAGS += $(CFLAGS_EXTRA) diff --git a/py/emitcommon.c b/py/emitcommon.c index a9eb6e2021fc8..1369c544c54f5 100644 --- a/py/emitcommon.c +++ b/py/emitcommon.c @@ -25,6 +25,7 @@ */ #include +#include #include "py/emit.h" #include "py/nativeglue.h" @@ -72,7 +73,25 @@ static bool strictly_equal(mp_obj_t a, mp_obj_t b) { } return true; } else { - return mp_obj_equal(a, b); + if (!mp_obj_equal(a, b)) { + return false; + } + #if MICROPY_PY_BUILTINS_FLOAT && MICROPY_COMP_CONST_FLOAT + if (a_type == &mp_type_float) { + mp_float_t a_val = mp_obj_float_get(a); + // CIRCUITPY-CHANGE: ignore float equal warning + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wfloat-equal" + if (a_val == (mp_float_t)0.0) { + // Although 0.0 == -0.0, they are not strictly_equal and + // must be stored as two different constants in .mpy files + mp_float_t b_val = mp_obj_float_get(b); + return signbit(a_val) == signbit(b_val); + } + #pragma GCC diagnostic pop + } + #endif + return true; } } diff --git a/py/emitglue.c b/py/emitglue.c index 8cc14bdc54eb5..c6ad3e3bc3845 100644 --- a/py/emitglue.c +++ b/py/emitglue.c @@ -32,6 +32,7 @@ #include #include "py/emitglue.h" +#include "py/mphal.h" #include "py/runtime0.h" #include "py/bc.h" #include "py/objfun.h" @@ -133,6 +134,9 @@ void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, cons "mcr p15, 0, r0, c7, c7, 0\n" // invalidate I-cache and D-cache : : : "r0", "cc"); #endif + #elif (MICROPY_EMIT_RV32 || MICROPY_EMIT_INLINE_RV32) && defined(MP_HAL_CLEAN_DCACHE) + // Flush the D-cache. + MP_HAL_CLEAN_DCACHE(fun_data, fun_len); #endif rc->kind = kind; diff --git a/py/emitinlinerv32.c b/py/emitinlinerv32.c index a539242b84d95..e81b152087de3 100644 --- a/py/emitinlinerv32.c +++ b/py/emitinlinerv32.c @@ -196,7 +196,6 @@ typedef enum { CALL_R, // Opcode Register CALL_RL, // Opcode Register, Label CALL_N, // Opcode - CALL_I, // Opcode Immediate CALL_RII, // Opcode Register, Register, Immediate CALL_RIR, // Opcode Register, Immediate(Register) CALL_COUNT @@ -210,7 +209,6 @@ typedef enum { #define U (1 << 2) // Unsigned immediate #define Z (1 << 3) // Non-zero -typedef void (*call_l_t)(asm_rv32_t *state, mp_uint_t label_index); typedef void (*call_ri_t)(asm_rv32_t *state, mp_uint_t rd, mp_int_t immediate); typedef void (*call_rri_t)(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs1, mp_int_t immediate); typedef void (*call_rii_t)(asm_rv32_t *state, mp_uint_t rd, mp_uint_t immediate1, mp_int_t immediate2); @@ -225,7 +223,7 @@ typedef struct _opcode_t { uint16_t argument1_mask : 4; uint16_t argument2_mask : 4; uint16_t argument3_mask : 4; - uint16_t arguments_count : 2; + uint16_t parse_nodes : 2; // 2 bits available here uint32_t calling_convention : 4; uint32_t argument1_kind : 4; @@ -234,7 +232,8 @@ typedef struct _opcode_t { uint32_t argument2_shift : 4; uint32_t argument3_kind : 4; uint32_t argument3_shift : 4; - // 4 bits available here + uint32_t required_extensions : 1; + // 3 bits available here void *emitter; } opcode_t; @@ -297,88 +296,91 @@ static const uint32_t OPCODE_MASKS[] = { }; static const opcode_t OPCODES[] = { - { MP_QSTR_add, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_add }, - { MP_QSTR_addi, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, asm_rv32_opcode_addi }, - { MP_QSTR_and_, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_and }, - { MP_QSTR_andi, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, asm_rv32_opcode_andi }, - { MP_QSTR_auipc, MASK_FFFFFFFF, MASK_FFFFF000, MASK_NOT_USED, 2, CALL_RI, R, 0, I, 12, N, 0, asm_rv32_opcode_auipc }, - { MP_QSTR_beq, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00001FFE, 3, CALL_RRL, R, 0, R, 0, L, 0, asm_rv32_opcode_beq }, - { MP_QSTR_bge, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00001FFE, 3, CALL_RRL, R, 0, R, 0, L, 0, asm_rv32_opcode_bge }, - { MP_QSTR_bgeu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00001FFE, 3, CALL_RRL, R, 0, R, 0, L, 0, asm_rv32_opcode_bgeu }, - { MP_QSTR_blt, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00001FFE, 3, CALL_RRL, R, 0, R, 0, L, 0, asm_rv32_opcode_blt }, - { MP_QSTR_bltu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00001FFE, 3, CALL_RRL, R, 0, R, 0, L, 0, asm_rv32_opcode_bltu }, - { MP_QSTR_bne, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00001FFE, 3, CALL_RRL, R, 0, R, 0, L, 0, asm_rv32_opcode_bne }, - { MP_QSTR_csrrc, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, IU, 0, asm_rv32_opcode_csrrc }, - { MP_QSTR_csrrs, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, IU, 0, asm_rv32_opcode_csrrs }, - { MP_QSTR_csrrw, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, IU, 0, asm_rv32_opcode_csrrw }, - { MP_QSTR_csrrci, MASK_FFFFFFFF, MASK_00000FFF, MASK_0000001F, 3, CALL_RII, R, 0, IU, 0, IU, 0, asm_rv32_opcode_csrrci }, - { MP_QSTR_csrrsi, MASK_FFFFFFFF, MASK_00000FFF, MASK_0000001F, 3, CALL_RII, R, 0, IU, 0, IU, 0, asm_rv32_opcode_csrrsi }, - { MP_QSTR_csrrwi, MASK_FFFFFFFF, MASK_00000FFF, MASK_0000001F, 3, CALL_RII, R, 0, IU, 0, IU, 0, asm_rv32_opcode_csrrwi }, - { MP_QSTR_c_add, MASK_FFFFFFFE, MASK_FFFFFFFE, MASK_NOT_USED, 2, CALL_RR, R, 0, R, 0, N, 0, asm_rv32_opcode_cadd }, - { MP_QSTR_c_addi, MASK_FFFFFFFE, MASK_0000003F, MASK_NOT_USED, 2, CALL_RI, R, 0, IZ, 0, N, 0, asm_rv32_opcode_caddi }, - { MP_QSTR_c_addi4spn, MASK_0000FF00, MASK_000003FC, MASK_NOT_USED, 2, CALL_RI, R, 0, IUZ, 0, N, 0, asm_rv32_opcode_caddi4spn }, - { MP_QSTR_c_and, MASK_0000FF00, MASK_0000FF00, MASK_NOT_USED, 2, CALL_RR, RC, 0, RC, 0, N, 0, asm_rv32_opcode_cand }, - { MP_QSTR_c_andi, MASK_0000FF00, MASK_0000003F, MASK_NOT_USED, 2, CALL_RI, RC, 0, I, 0, N, 0, asm_rv32_opcode_candi }, - { MP_QSTR_c_beqz, MASK_0000FF00, MASK_000001FE, MASK_NOT_USED, 2, CALL_RL, RC, 0, L, 0, N, 0, asm_rv32_opcode_cbeqz }, - { MP_QSTR_c_bnez, MASK_0000FF00, MASK_000001FE, MASK_NOT_USED, 2, CALL_RL, RC, 0, L, 0, N, 0, asm_rv32_opcode_cbnez }, - { MP_QSTR_c_ebreak, MASK_NOT_USED, MASK_NOT_USED, MASK_NOT_USED, 0, CALL_N, N, 0, N, 0, N, 0, asm_rv32_opcode_cebreak }, - { MP_QSTR_c_j, MASK_00000FFE, MASK_NOT_USED, MASK_NOT_USED, 1, CALL_L, L, 0, N, 0, N, 0, asm_rv32_opcode_cj }, - { MP_QSTR_c_jal, MASK_00000FFE, MASK_NOT_USED, MASK_NOT_USED, 1, CALL_L, L, 0, N, 0, N, 0, asm_rv32_opcode_cjal }, - { MP_QSTR_c_jalr, MASK_FFFFFFFE, MASK_NOT_USED, MASK_NOT_USED, 1, CALL_R, R, 0, N, 0, N, 0, asm_rv32_opcode_cjalr }, - { MP_QSTR_c_jr, MASK_FFFFFFFE, MASK_NOT_USED, MASK_NOT_USED, 1, CALL_R, R, 0, N, 0, N, 0, asm_rv32_opcode_cjr }, - { MP_QSTR_c_li, MASK_FFFFFFFE, MASK_0000003F, MASK_NOT_USED, 2, CALL_RI, R, 0, I, 0, N, 0, asm_rv32_opcode_cli }, - { MP_QSTR_c_lui, MASK_FFFFFFFA, MASK_0001F800, MASK_NOT_USED, 2, CALL_RI, R, 0, IUZ, 12, N, 0, asm_rv32_opcode_clui }, - { MP_QSTR_c_lw, MASK_0000FF00, MASK_0000007C, MASK_0000FF00, 3, CALL_RIR, RC, 0, I, 0, RC, 0, asm_rv32_opcode_clw }, - { MP_QSTR_c_lwsp, MASK_FFFFFFFE, MASK_000000FC, MASK_NOT_USED, 2, CALL_RI, R, 0, I, 0, N, 0, asm_rv32_opcode_clwsp }, - { MP_QSTR_c_mv, MASK_FFFFFFFE, MASK_FFFFFFFE, MASK_NOT_USED, 2, CALL_RR, R, 0, R, 0, N, 0, asm_rv32_opcode_cmv }, - { MP_QSTR_c_nop, MASK_NOT_USED, MASK_NOT_USED, MASK_NOT_USED, 0, CALL_N, N, 0, N, 0, N, 0, asm_rv32_opcode_cnop }, - { MP_QSTR_c_or, MASK_0000FF00, MASK_0000FF00, MASK_NOT_USED, 2, CALL_RR, RC, 0, RC, 0, N, 0, asm_rv32_opcode_cor }, - { MP_QSTR_c_slli, MASK_FFFFFFFE, MASK_0000001F, MASK_NOT_USED, 2, CALL_RI, R, 0, IU, 0, N, 0, asm_rv32_opcode_cslli }, - { MP_QSTR_c_srai, MASK_0000FF00, MASK_0000001F, MASK_NOT_USED, 2, CALL_RI, RC, 0, IU, 0, N, 0, asm_rv32_opcode_csrai }, - { MP_QSTR_c_srli, MASK_0000FF00, MASK_0000001F, MASK_NOT_USED, 2, CALL_RI, RC, 0, IU, 0, N, 0, asm_rv32_opcode_csrli }, - { MP_QSTR_c_sub, MASK_0000FF00, MASK_0000FF00, MASK_NOT_USED, 2, CALL_RR, RC, 0, RC, 0, N, 0, asm_rv32_opcode_csub }, - { MP_QSTR_c_sw, MASK_0000FF00, MASK_0000007C, MASK_0000FF00, 3, CALL_RIR, RC, 0, I, 0, RC, 0, asm_rv32_opcode_csw }, - { MP_QSTR_c_swsp, MASK_FFFFFFFF, MASK_000000FC, MASK_NOT_USED, 2, CALL_RI, R, 0, I, 0, N, 0, asm_rv32_opcode_cswsp }, - { MP_QSTR_c_xor, MASK_0000FF00, MASK_0000FF00, MASK_NOT_USED, 2, CALL_RR, RC, 0, RC, 0, N, 0, asm_rv32_opcode_cxor }, - { MP_QSTR_div, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_div }, - { MP_QSTR_divu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_divu }, - { MP_QSTR_ebreak, MASK_NOT_USED, MASK_NOT_USED, MASK_NOT_USED, 0, CALL_N, N, 0, N, 0, N, 0, asm_rv32_opcode_ebreak }, - { MP_QSTR_ecall, MASK_NOT_USED, MASK_NOT_USED, MASK_NOT_USED, 0, CALL_N, N, 0, N, 0, N, 0, asm_rv32_opcode_ecall }, - { MP_QSTR_jal, MASK_FFFFFFFF, MASK_001FFFFE, MASK_NOT_USED, 2, CALL_RL, R, 0, L, 0, N, 0, asm_rv32_opcode_jal }, - { MP_QSTR_jalr, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, asm_rv32_opcode_jalr }, - { MP_QSTR_la, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_NOT_USED, 2, CALL_RL, R, 0, L, 0, N, 0, opcode_la }, - { MP_QSTR_lb, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 3, CALL_RIR, R, 0, I, 0, R, 0, asm_rv32_opcode_lb }, - { MP_QSTR_lbu, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 3, CALL_RIR, R, 0, I, 0, R, 0, asm_rv32_opcode_lbu }, - { MP_QSTR_lh, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 3, CALL_RIR, R, 0, I, 0, R, 0, asm_rv32_opcode_lh }, - { MP_QSTR_lhu, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 3, CALL_RIR, R, 0, I, 0, R, 0, asm_rv32_opcode_lhu }, - { MP_QSTR_li, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_NOT_USED, 2, CALL_RI, R, 0, I, 0, N, 0, opcode_li }, - { MP_QSTR_lui, MASK_FFFFFFFF, MASK_FFFFF000, MASK_NOT_USED, 2, CALL_RI, R, 0, I, 12, N, 0, asm_rv32_opcode_lui }, - { MP_QSTR_lw, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 3, CALL_RIR, R, 0, I, 0, R, 0, asm_rv32_opcode_lw }, - { MP_QSTR_mv, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_NOT_USED, 2, CALL_RR, R, 0, R, 0, N, 0, asm_rv32_opcode_cmv }, - { MP_QSTR_mul, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_mul }, - { MP_QSTR_mulh, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_mulh }, - { MP_QSTR_mulhsu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_mulhsu }, - { MP_QSTR_mulhu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_mulhu }, - { MP_QSTR_or_, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_or }, - { MP_QSTR_ori, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, asm_rv32_opcode_ori }, - { MP_QSTR_rem, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_rem }, - { MP_QSTR_remu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_remu }, - { MP_QSTR_sb, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 3, CALL_RIR, R, 0, I, 0, R, 0, asm_rv32_opcode_sb }, - { MP_QSTR_sh, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 3, CALL_RIR, R, 0, I, 0, R, 0, asm_rv32_opcode_sh }, - { MP_QSTR_sll, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_sll }, - { MP_QSTR_slli, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_0000001F, 3, CALL_RRI, R, 0, R, 0, IU, 0, asm_rv32_opcode_slli }, - { MP_QSTR_slt, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_slt }, - { MP_QSTR_slti, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, asm_rv32_opcode_slti }, - { MP_QSTR_sltiu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, asm_rv32_opcode_sltiu }, - { MP_QSTR_sltu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_sltu }, - { MP_QSTR_sra, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_sra }, - { MP_QSTR_srai, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_0000001F, 3, CALL_RRI, R, 0, R, 0, IU, 0, asm_rv32_opcode_srai }, - { MP_QSTR_srl, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_srl }, - { MP_QSTR_srli, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_0000001F, 3, CALL_RRI, R, 0, R, 0, IU, 0, asm_rv32_opcode_srli }, - { MP_QSTR_sub, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_sub }, - { MP_QSTR_sw, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 3, CALL_RIR, R, 0, I, 0, R, 0, asm_rv32_opcode_sw }, - { MP_QSTR_xor, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_xor }, - { MP_QSTR_xori, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, asm_rv32_opcode_xori }, + { MP_QSTR_add, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, RV32_EXT_NONE, asm_rv32_opcode_add }, + { MP_QSTR_addi, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, RV32_EXT_NONE, asm_rv32_opcode_addi }, + { MP_QSTR_and_, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, RV32_EXT_NONE, asm_rv32_opcode_and }, + { MP_QSTR_andi, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, RV32_EXT_NONE, asm_rv32_opcode_andi }, + { MP_QSTR_auipc, MASK_FFFFFFFF, MASK_FFFFF000, MASK_NOT_USED, 2, CALL_RI, R, 0, I, 12, N, 0, RV32_EXT_NONE, asm_rv32_opcode_auipc }, + { MP_QSTR_beq, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00001FFE, 3, CALL_RRL, R, 0, R, 0, L, 0, RV32_EXT_NONE, asm_rv32_opcode_beq }, + { MP_QSTR_bge, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00001FFE, 3, CALL_RRL, R, 0, R, 0, L, 0, RV32_EXT_NONE, asm_rv32_opcode_bge }, + { MP_QSTR_bgeu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00001FFE, 3, CALL_RRL, R, 0, R, 0, L, 0, RV32_EXT_NONE, asm_rv32_opcode_bgeu }, + { MP_QSTR_blt, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00001FFE, 3, CALL_RRL, R, 0, R, 0, L, 0, RV32_EXT_NONE, asm_rv32_opcode_blt }, + { MP_QSTR_bltu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00001FFE, 3, CALL_RRL, R, 0, R, 0, L, 0, RV32_EXT_NONE, asm_rv32_opcode_bltu }, + { MP_QSTR_bne, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00001FFE, 3, CALL_RRL, R, 0, R, 0, L, 0, RV32_EXT_NONE, asm_rv32_opcode_bne }, + { MP_QSTR_csrrc, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, IU, 0, RV32_EXT_NONE, asm_rv32_opcode_csrrc }, + { MP_QSTR_csrrs, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, IU, 0, RV32_EXT_NONE, asm_rv32_opcode_csrrs }, + { MP_QSTR_csrrw, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, IU, 0, RV32_EXT_NONE, asm_rv32_opcode_csrrw }, + { MP_QSTR_csrrci, MASK_FFFFFFFF, MASK_00000FFF, MASK_0000001F, 3, CALL_RII, R, 0, IU, 0, IU, 0, RV32_EXT_NONE, asm_rv32_opcode_csrrci }, + { MP_QSTR_csrrsi, MASK_FFFFFFFF, MASK_00000FFF, MASK_0000001F, 3, CALL_RII, R, 0, IU, 0, IU, 0, RV32_EXT_NONE, asm_rv32_opcode_csrrsi }, + { MP_QSTR_csrrwi, MASK_FFFFFFFF, MASK_00000FFF, MASK_0000001F, 3, CALL_RII, R, 0, IU, 0, IU, 0, RV32_EXT_NONE, asm_rv32_opcode_csrrwi }, + { MP_QSTR_c_add, MASK_FFFFFFFE, MASK_FFFFFFFE, MASK_NOT_USED, 2, CALL_RR, R, 0, R, 0, N, 0, RV32_EXT_NONE, asm_rv32_opcode_cadd }, + { MP_QSTR_c_addi, MASK_FFFFFFFE, MASK_0000003F, MASK_NOT_USED, 2, CALL_RI, R, 0, IZ, 0, N, 0, RV32_EXT_NONE, asm_rv32_opcode_caddi }, + { MP_QSTR_c_addi4spn, MASK_0000FF00, MASK_000003FC, MASK_NOT_USED, 2, CALL_RI, R, 0, IUZ, 0, N, 0, RV32_EXT_NONE, asm_rv32_opcode_caddi4spn }, + { MP_QSTR_c_and, MASK_0000FF00, MASK_0000FF00, MASK_NOT_USED, 2, CALL_RR, RC, 0, RC, 0, N, 0, RV32_EXT_NONE, asm_rv32_opcode_cand }, + { MP_QSTR_c_andi, MASK_0000FF00, MASK_0000003F, MASK_NOT_USED, 2, CALL_RI, RC, 0, I, 0, N, 0, RV32_EXT_NONE, asm_rv32_opcode_candi }, + { MP_QSTR_c_beqz, MASK_0000FF00, MASK_000001FE, MASK_NOT_USED, 2, CALL_RL, RC, 0, L, 0, N, 0, RV32_EXT_NONE, asm_rv32_opcode_cbeqz }, + { MP_QSTR_c_bnez, MASK_0000FF00, MASK_000001FE, MASK_NOT_USED, 2, CALL_RL, RC, 0, L, 0, N, 0, RV32_EXT_NONE, asm_rv32_opcode_cbnez }, + { MP_QSTR_c_ebreak, MASK_NOT_USED, MASK_NOT_USED, MASK_NOT_USED, 0, CALL_N, N, 0, N, 0, N, 0, RV32_EXT_NONE, asm_rv32_opcode_cebreak }, + { MP_QSTR_c_j, MASK_00000FFE, MASK_NOT_USED, MASK_NOT_USED, 1, CALL_L, L, 0, N, 0, N, 0, RV32_EXT_NONE, asm_rv32_opcode_cj }, + { MP_QSTR_c_jal, MASK_00000FFE, MASK_NOT_USED, MASK_NOT_USED, 1, CALL_L, L, 0, N, 0, N, 0, RV32_EXT_NONE, asm_rv32_opcode_cjal }, + { MP_QSTR_c_jalr, MASK_FFFFFFFE, MASK_NOT_USED, MASK_NOT_USED, 1, CALL_R, R, 0, N, 0, N, 0, RV32_EXT_NONE, asm_rv32_opcode_cjalr }, + { MP_QSTR_c_jr, MASK_FFFFFFFE, MASK_NOT_USED, MASK_NOT_USED, 1, CALL_R, R, 0, N, 0, N, 0, RV32_EXT_NONE, asm_rv32_opcode_cjr }, + { MP_QSTR_c_li, MASK_FFFFFFFE, MASK_0000003F, MASK_NOT_USED, 2, CALL_RI, R, 0, I, 0, N, 0, RV32_EXT_NONE, asm_rv32_opcode_cli }, + { MP_QSTR_c_lui, MASK_FFFFFFFA, MASK_0001F800, MASK_NOT_USED, 2, CALL_RI, R, 0, IUZ, 12, N, 0, RV32_EXT_NONE, asm_rv32_opcode_clui }, + { MP_QSTR_c_lw, MASK_0000FF00, MASK_0000007C, MASK_0000FF00, 2, CALL_RIR, RC, 0, I, 0, RC, 0, RV32_EXT_NONE, asm_rv32_opcode_clw }, + { MP_QSTR_c_lwsp, MASK_FFFFFFFE, MASK_000000FC, MASK_NOT_USED, 2, CALL_RI, R, 0, I, 0, N, 0, RV32_EXT_NONE, asm_rv32_opcode_clwsp }, + { MP_QSTR_c_mv, MASK_FFFFFFFE, MASK_FFFFFFFE, MASK_NOT_USED, 2, CALL_RR, R, 0, R, 0, N, 0, RV32_EXT_NONE, asm_rv32_opcode_cmv }, + { MP_QSTR_c_nop, MASK_NOT_USED, MASK_NOT_USED, MASK_NOT_USED, 0, CALL_N, N, 0, N, 0, N, 0, RV32_EXT_NONE, asm_rv32_opcode_cnop }, + { MP_QSTR_c_or, MASK_0000FF00, MASK_0000FF00, MASK_NOT_USED, 2, CALL_RR, RC, 0, RC, 0, N, 0, RV32_EXT_NONE, asm_rv32_opcode_cor }, + { MP_QSTR_c_slli, MASK_FFFFFFFE, MASK_0000001F, MASK_NOT_USED, 2, CALL_RI, R, 0, IU, 0, N, 0, RV32_EXT_NONE, asm_rv32_opcode_cslli }, + { MP_QSTR_c_srai, MASK_0000FF00, MASK_0000001F, MASK_NOT_USED, 2, CALL_RI, RC, 0, IU, 0, N, 0, RV32_EXT_NONE, asm_rv32_opcode_csrai }, + { MP_QSTR_c_srli, MASK_0000FF00, MASK_0000001F, MASK_NOT_USED, 2, CALL_RI, RC, 0, IU, 0, N, 0, RV32_EXT_NONE, asm_rv32_opcode_csrli }, + { MP_QSTR_c_sub, MASK_0000FF00, MASK_0000FF00, MASK_NOT_USED, 2, CALL_RR, RC, 0, RC, 0, N, 0, RV32_EXT_NONE, asm_rv32_opcode_csub }, + { MP_QSTR_c_sw, MASK_0000FF00, MASK_0000007C, MASK_0000FF00, 2, CALL_RIR, RC, 0, I, 0, RC, 0, RV32_EXT_NONE, asm_rv32_opcode_csw }, + { MP_QSTR_c_swsp, MASK_FFFFFFFF, MASK_000000FC, MASK_NOT_USED, 2, CALL_RI, R, 0, I, 0, N, 0, RV32_EXT_NONE, asm_rv32_opcode_cswsp }, + { MP_QSTR_c_xor, MASK_0000FF00, MASK_0000FF00, MASK_NOT_USED, 2, CALL_RR, RC, 0, RC, 0, N, 0, RV32_EXT_NONE, asm_rv32_opcode_cxor }, + { MP_QSTR_div, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, RV32_EXT_NONE, asm_rv32_opcode_div }, + { MP_QSTR_divu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, RV32_EXT_NONE, asm_rv32_opcode_divu }, + { MP_QSTR_ebreak, MASK_NOT_USED, MASK_NOT_USED, MASK_NOT_USED, 0, CALL_N, N, 0, N, 0, N, 0, RV32_EXT_NONE, asm_rv32_opcode_ebreak }, + { MP_QSTR_ecall, MASK_NOT_USED, MASK_NOT_USED, MASK_NOT_USED, 0, CALL_N, N, 0, N, 0, N, 0, RV32_EXT_NONE, asm_rv32_opcode_ecall }, + { MP_QSTR_jal, MASK_FFFFFFFF, MASK_001FFFFE, MASK_NOT_USED, 2, CALL_RL, R, 0, L, 0, N, 0, RV32_EXT_NONE, asm_rv32_opcode_jal }, + { MP_QSTR_jalr, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, RV32_EXT_NONE, asm_rv32_opcode_jalr }, + { MP_QSTR_la, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_NOT_USED, 2, CALL_RL, R, 0, L, 0, N, 0, RV32_EXT_NONE, opcode_la }, + { MP_QSTR_lb, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 2, CALL_RIR, R, 0, I, 0, R, 0, RV32_EXT_NONE, asm_rv32_opcode_lb }, + { MP_QSTR_lbu, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 2, CALL_RIR, R, 0, I, 0, R, 0, RV32_EXT_NONE, asm_rv32_opcode_lbu }, + { MP_QSTR_lh, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 2, CALL_RIR, R, 0, I, 0, R, 0, RV32_EXT_NONE, asm_rv32_opcode_lh }, + { MP_QSTR_lhu, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 2, CALL_RIR, R, 0, I, 0, R, 0, RV32_EXT_NONE, asm_rv32_opcode_lhu }, + { MP_QSTR_li, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_NOT_USED, 2, CALL_RI, R, 0, I, 0, N, 0, RV32_EXT_NONE, opcode_li }, + { MP_QSTR_lui, MASK_FFFFFFFF, MASK_FFFFF000, MASK_NOT_USED, 2, CALL_RI, R, 0, I, 12, N, 0, RV32_EXT_NONE, asm_rv32_opcode_lui }, + { MP_QSTR_lw, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 2, CALL_RIR, R, 0, I, 0, R, 0, RV32_EXT_NONE, asm_rv32_opcode_lw }, + { MP_QSTR_mv, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_NOT_USED, 2, CALL_RR, R, 0, R, 0, N, 0, RV32_EXT_NONE, asm_rv32_opcode_cmv }, + { MP_QSTR_mul, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, RV32_EXT_NONE, asm_rv32_opcode_mul }, + { MP_QSTR_mulh, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, RV32_EXT_NONE, asm_rv32_opcode_mulh }, + { MP_QSTR_mulhsu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, RV32_EXT_NONE, asm_rv32_opcode_mulhsu }, + { MP_QSTR_mulhu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, RV32_EXT_NONE, asm_rv32_opcode_mulhu }, + { MP_QSTR_or_, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, RV32_EXT_NONE, asm_rv32_opcode_or }, + { MP_QSTR_ori, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, RV32_EXT_NONE, asm_rv32_opcode_ori }, + { MP_QSTR_rem, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, RV32_EXT_NONE, asm_rv32_opcode_rem }, + { MP_QSTR_remu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, RV32_EXT_NONE, asm_rv32_opcode_remu }, + { MP_QSTR_sb, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 2, CALL_RIR, R, 0, I, 0, R, 0, RV32_EXT_NONE, asm_rv32_opcode_sb }, + { MP_QSTR_sh, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 2, CALL_RIR, R, 0, I, 0, R, 0, RV32_EXT_NONE, asm_rv32_opcode_sh }, + { MP_QSTR_sh1add, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, RV32_EXT_ZBA, asm_rv32_opcode_sh1add }, + { MP_QSTR_sh2add, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, RV32_EXT_ZBA, asm_rv32_opcode_sh2add }, + { MP_QSTR_sh3add, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, RV32_EXT_ZBA, asm_rv32_opcode_sh3add }, + { MP_QSTR_sll, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, RV32_EXT_NONE, asm_rv32_opcode_sll }, + { MP_QSTR_slli, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_0000001F, 3, CALL_RRI, R, 0, R, 0, IU, 0, RV32_EXT_NONE, asm_rv32_opcode_slli }, + { MP_QSTR_slt, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, RV32_EXT_NONE, asm_rv32_opcode_slt }, + { MP_QSTR_slti, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, RV32_EXT_NONE, asm_rv32_opcode_slti }, + { MP_QSTR_sltiu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, RV32_EXT_NONE, asm_rv32_opcode_sltiu }, + { MP_QSTR_sltu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, RV32_EXT_NONE, asm_rv32_opcode_sltu }, + { MP_QSTR_sra, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, RV32_EXT_NONE, asm_rv32_opcode_sra }, + { MP_QSTR_srai, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_0000001F, 3, CALL_RRI, R, 0, R, 0, IU, 0, RV32_EXT_NONE, asm_rv32_opcode_srai }, + { MP_QSTR_srl, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, RV32_EXT_NONE, asm_rv32_opcode_srl }, + { MP_QSTR_srli, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_0000001F, 3, CALL_RRI, R, 0, R, 0, IU, 0, RV32_EXT_NONE, asm_rv32_opcode_srli }, + { MP_QSTR_sub, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, RV32_EXT_NONE, asm_rv32_opcode_sub }, + { MP_QSTR_sw, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 2, CALL_RIR, R, 0, I, 0, R, 0, RV32_EXT_NONE, asm_rv32_opcode_sw }, + { MP_QSTR_xor, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, RV32_EXT_NONE, asm_rv32_opcode_xor }, + { MP_QSTR_xori, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, RV32_EXT_NONE, asm_rv32_opcode_xori }, }; #undef RC @@ -388,9 +390,9 @@ static const opcode_t OPCODES[] = { // These two checks assume the bitmasks are contiguous. -static bool is_in_signed_mask(mp_uint_t mask, mp_uint_t value) { - mp_uint_t leading_zeroes = mp_clz(mask); - if (leading_zeroes == 0 || leading_zeroes > 32) { +static bool is_in_signed_mask(uint32_t mask, mp_uint_t value) { + uint32_t leading_zeroes = mp_clz(mask); + if (leading_zeroes == 0) { return true; } mp_uint_t positive_mask = ~(mask & ~(1U << (31 - leading_zeroes))); @@ -435,9 +437,9 @@ static bool validate_integer(mp_uint_t value, mp_uint_t mask, mp_uint_t flags) { #define ET_WRONG_ARGUMENTS_COUNT MP_ERROR_TEXT("opcode '%q': expecting %d arguments") #define ET_OUT_OF_RANGE MP_ERROR_TEXT("opcode '%q' argument %d: out of range") -static bool validate_argument(emit_inline_asm_t *emit, qstr opcode_qstr, - const opcode_t *opcode, mp_parse_node_t node, mp_uint_t node_index) { +static bool serialise_argument(emit_inline_asm_t *emit, const opcode_t *opcode, mp_parse_node_t node, mp_uint_t node_index, mp_uint_t *serialised) { assert((node_index < 3) && "Invalid argument node number."); + assert(serialised && "Serialised value pointer is NULL."); uint32_t kind = 0; uint32_t shift = 0; @@ -466,17 +468,19 @@ static bool validate_argument(emit_inline_asm_t *emit, qstr opcode_qstr, break; } + mp_uint_t serialised_value = 0; + switch (kind & 0x03) { case N: assert(mask == OPCODE_MASKS[MASK_NOT_USED] && "Invalid mask index for missing operand."); - return true; + break; case R: { mp_uint_t register_index; if (!parse_register_node(node, ®ister_index, false)) { emit_inline_rv32_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, - ET_WRONG_ARGUMENT_KIND, opcode_qstr, node_index + 1, MP_QSTR_register)); + ET_WRONG_ARGUMENT_KIND, opcode->qstring, node_index + 1, MP_QSTR_register)); return false; } @@ -484,11 +488,11 @@ static bool validate_argument(emit_inline_asm_t *emit, qstr opcode_qstr, emit_inline_rv32_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, MP_ERROR_TEXT("opcode '%q' argument %d: unknown register"), - opcode_qstr, node_index + 1)); + opcode->qstring, node_index + 1)); return false; } - return true; + serialised_value = (kind & C) ? RV32_MAP_IN_C_REGISTER_WINDOW(register_index) : register_index; } break; @@ -497,7 +501,7 @@ static bool validate_argument(emit_inline_asm_t *emit, qstr opcode_qstr, if (!mp_parse_node_get_int_maybe(node, &object)) { emit_inline_rv32_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, - ET_WRONG_ARGUMENT_KIND, opcode_qstr, node_index + 1, MP_QSTR_integer)); + ET_WRONG_ARGUMENT_KIND, opcode->qstring, node_index + 1, MP_QSTR_integer)); return false; } @@ -516,7 +520,7 @@ static bool validate_argument(emit_inline_asm_t *emit, qstr opcode_qstr, goto zero_immediate; } - return true; + serialised_value = immediate; } break; @@ -524,7 +528,7 @@ static bool validate_argument(emit_inline_asm_t *emit, qstr opcode_qstr, if (!MP_PARSE_NODE_IS_ID(node)) { emit_inline_rv32_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, - ET_WRONG_ARGUMENT_KIND, opcode_qstr, node_index + 1, MP_QSTR_label)); + ET_WRONG_ARGUMENT_KIND, opcode->qstring, node_index + 1, MP_QSTR_label)); return false; } @@ -534,7 +538,7 @@ static bool validate_argument(emit_inline_asm_t *emit, qstr opcode_qstr, emit_inline_rv32_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, MP_ERROR_TEXT("opcode '%q' argument %d: undefined label '%q'"), - opcode_qstr, node_index + 1, qstring)); + opcode->qstring, node_index + 1, qstring)); return false; } @@ -548,43 +552,45 @@ static bool validate_argument(emit_inline_asm_t *emit, qstr opcode_qstr, goto out_of_range; } } - return true; + + serialised_value = displacement; } break; default: assert(!"Unknown argument kind"); + MP_UNREACHABLE; break; } - return false; + *serialised = serialised_value; + return true; out_of_range: emit_inline_rv32_error_exc(emit, - mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, ET_OUT_OF_RANGE, opcode_qstr, node_index + 1)); + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, ET_OUT_OF_RANGE, opcode->qstring, node_index + 1)); return false; zero_immediate: emit_inline_rv32_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, MP_ERROR_TEXT("opcode '%q' argument %d: must not be zero"), - opcode_qstr, node_index + 1)); + opcode->qstring, node_index + 1)); return false; } -static bool parse_register_offset_node(emit_inline_asm_t *emit, qstr opcode_qstr, const opcode_t *opcode_data, mp_parse_node_t node, mp_uint_t node_index, mp_parse_node_t *register_node, mp_parse_node_t *offset_node, bool *negative) { - assert(register_node != NULL && "Register node pointer is NULL."); - assert(offset_node != NULL && "Offset node pointer is NULL."); - assert(negative != NULL && "Negative pointer is NULL."); +static bool serialise_register_offset_node(emit_inline_asm_t *emit, const opcode_t *opcode_data, mp_parse_node_t node, mp_uint_t node_index, mp_uint_t *offset, mp_uint_t *base) { + assert(offset && "Attempting to store the offset value into NULL."); + assert(base && "Attempting to store the base register into NULL."); if (!MP_PARSE_NODE_IS_STRUCT_KIND(node, PN_atom_expr_normal) && !MP_PARSE_NODE_IS_STRUCT_KIND(node, PN_factor_2)) { goto invalid_structure; } mp_parse_node_struct_t *node_struct = (mp_parse_node_struct_t *)node; - *negative = false; + bool negative = false; if (MP_PARSE_NODE_IS_STRUCT_KIND(node, PN_factor_2)) { if (MP_PARSE_NODE_IS_TOKEN_KIND(node_struct->nodes[0], MP_TOKEN_OP_MINUS)) { - *negative = true; + negative = true; } else { if (!MP_PARSE_NODE_IS_TOKEN_KIND(node_struct->nodes[0], MP_TOKEN_OP_PLUS)) { goto invalid_structure; @@ -596,186 +602,97 @@ static bool parse_register_offset_node(emit_inline_asm_t *emit, qstr opcode_qstr node_struct = (mp_parse_node_struct_t *)node_struct->nodes[1]; } - if (*negative) { + if (negative) { // If the value is negative, RULE_atom_expr_normal's first token will be the // offset stripped of its negative marker; range check will then fail if the // default method is used, so a custom check is used instead. mp_obj_t object; if (!mp_parse_node_get_int_maybe(node_struct->nodes[0], &object)) { emit_inline_rv32_error_exc(emit, - mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, ET_WRONG_ARGUMENT_KIND, opcode_qstr, 2, MP_QSTR_integer)); + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, ET_WRONG_ARGUMENT_KIND, opcode_data->qstring, 2, MP_QSTR_integer)); return false; } mp_uint_t value = mp_obj_get_int_truncated(object); value = (~value + 1) & (mp_uint_t)-1; if (!validate_integer(value << opcode_data->argument2_shift, OPCODE_MASKS[opcode_data->argument2_mask], opcode_data->argument2_kind)) { emit_inline_rv32_error_exc(emit, - mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, ET_OUT_OF_RANGE, opcode_qstr, 2)); + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, ET_OUT_OF_RANGE, opcode_data->qstring, 2)); return false; } + *offset = value; } else { - if (!validate_argument(emit, opcode_qstr, opcode_data, node_struct->nodes[0], 1)) { + if (!serialise_argument(emit, opcode_data, node_struct->nodes[0], 1, offset)) { return false; } } - *offset_node = node_struct->nodes[0]; node_struct = (mp_parse_node_struct_t *)node_struct->nodes[1]; - if (!validate_argument(emit, opcode_qstr, opcode_data, node_struct->nodes[0], 2)) { + if (!serialise_argument(emit, opcode_data, node_struct->nodes[0], 2, base)) { return false; } - *register_node = node_struct->nodes[0]; return true; invalid_structure: emit_inline_rv32_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, - ET_WRONG_ARGUMENT_KIND, opcode_qstr, node_index + 1, MP_QSTR_offset)); + ET_WRONG_ARGUMENT_KIND, opcode_data->qstring, node_index + 1, MP_QSTR_offset)); return false; } -static void handle_opcode(emit_inline_asm_t *emit, qstr opcode, const opcode_t *opcode_data, mp_parse_node_t *arguments) { - mp_uint_t rd = 0; - mp_uint_t rs1 = 0; - mp_uint_t rs2 = 0; - +static void handle_opcode(emit_inline_asm_t *emit, const opcode_t *opcode_data, mp_uint_t *arguments) { switch (opcode_data->calling_convention) { - case CALL_RRR: { - parse_register_node(arguments[0], &rd, opcode_data->argument1_kind & C); - parse_register_node(arguments[1], &rs1, opcode_data->argument2_kind & C); - parse_register_node(arguments[2], &rs2, opcode_data->argument3_kind & C); - ((call_rrr_t)opcode_data->emitter)(&emit->as, rd, rs1, rs2); + case CALL_RRR: + ((call_rrr_t)opcode_data->emitter)(&emit->as, arguments[0], arguments[1], arguments[2]); break; - } - case CALL_RR: { - parse_register_node(arguments[0], &rd, opcode_data->argument1_kind & C); - parse_register_node(arguments[1], &rs1, opcode_data->argument2_kind & C); - ((call_rr_t)opcode_data->emitter)(&emit->as, rd, rs1); + case CALL_RR: + ((call_rr_t)opcode_data->emitter)(&emit->as, arguments[0], arguments[1]); break; - } - case CALL_RRI: { - parse_register_node(arguments[0], &rd, opcode_data->argument1_kind & C); - parse_register_node(arguments[1], &rs1, opcode_data->argument2_kind & C); - mp_obj_t object; - mp_parse_node_get_int_maybe(arguments[2], &object); - mp_uint_t immediate = mp_obj_get_int_truncated(object) << opcode_data->argument3_shift; - ((call_rri_t)opcode_data->emitter)(&emit->as, rd, rs1, immediate); + case CALL_RRI: + ((call_rri_t)opcode_data->emitter)(&emit->as, arguments[0], arguments[1], arguments[2]); break; - } - case CALL_RI: { - parse_register_node(arguments[0], &rd, opcode_data->argument1_kind & C); - mp_obj_t object; - mp_parse_node_get_int_maybe(arguments[1], &object); - mp_uint_t immediate = mp_obj_get_int_truncated(object) << opcode_data->argument2_shift; - ((call_ri_t)opcode_data->emitter)(&emit->as, rd, immediate); + case CALL_RI: + ((call_ri_t)opcode_data->emitter)(&emit->as, arguments[0], arguments[1]); break; - } - case CALL_R: { - parse_register_node(arguments[0], &rd, opcode_data->argument1_kind & C); - ((call_r_t)opcode_data->emitter)(&emit->as, rd); + case CALL_R: + ((call_r_t)opcode_data->emitter)(&emit->as, arguments[0]); break; - } - case CALL_RRL: { - parse_register_node(arguments[0], &rd, opcode_data->argument1_kind & C); - parse_register_node(arguments[1], &rs1, opcode_data->argument2_kind & C); - qstr qstring; - mp_uint_t label_index = lookup_label(emit, arguments[2], &qstring); - ptrdiff_t displacement = label_code_offset(emit, label_index); - ((call_rri_t)opcode_data->emitter)(&emit->as, rd, rs1, displacement); + case CALL_RRL: + ((call_rri_t)opcode_data->emitter)(&emit->as, arguments[0], arguments[1], (ptrdiff_t)arguments[2]); break; - } - case CALL_RL: { - parse_register_node(arguments[0], &rd, opcode_data->argument1_kind & C); - qstr qstring; - mp_uint_t label_index = lookup_label(emit, arguments[1], &qstring); - ptrdiff_t displacement = label_code_offset(emit, label_index); - ((call_ri_t)opcode_data->emitter)(&emit->as, rd, displacement); + case CALL_RL: + ((call_ri_t)opcode_data->emitter)(&emit->as, arguments[0], (ptrdiff_t)arguments[1]); break; - } - case CALL_L: { - qstr qstring; - mp_uint_t label_index = lookup_label(emit, arguments[0], &qstring); - ptrdiff_t displacement = label_code_offset(emit, label_index); - ((call_i_t)opcode_data->emitter)(&emit->as, displacement); + case CALL_L: + ((call_i_t)opcode_data->emitter)(&emit->as, (ptrdiff_t)arguments[0]); break; - } case CALL_N: ((call_n_t)opcode_data->emitter)(&emit->as); break; - case CALL_I: { - mp_obj_t object; - mp_parse_node_get_int_maybe(arguments[0], &object); - mp_uint_t immediate = mp_obj_get_int_truncated(object) << opcode_data->argument1_shift; - ((call_i_t)opcode_data->emitter)(&emit->as, immediate); + case CALL_RII: + ((call_rii_t)opcode_data->emitter)(&emit->as, arguments[0], arguments[1], arguments[2]); break; - } - - case CALL_RII: { - parse_register_node(arguments[0], &rd, opcode_data->argument1_kind & C); - mp_obj_t object; - mp_parse_node_get_int_maybe(arguments[1], &object); - mp_uint_t immediate1 = mp_obj_get_int_truncated(object) << opcode_data->argument2_shift; - mp_parse_node_get_int_maybe(arguments[2], &object); - mp_uint_t immediate2 = mp_obj_get_int_truncated(object) << opcode_data->argument3_shift; - ((call_rii_t)opcode_data->emitter)(&emit->as, rd, immediate1, immediate2); - break; - } case CALL_RIR: - assert(!"Should not get here."); + // The last two arguments indices are swapped on purpose. + ((call_rri_t)opcode_data->emitter)(&emit->as, arguments[0], arguments[2], arguments[1]); break; default: assert(!"Unhandled call convention."); + MP_UNREACHABLE; break; } } -static bool handle_load_store_opcode_with_offset(emit_inline_asm_t *emit, qstr opcode, const opcode_t *opcode_data, mp_parse_node_t *argument_nodes) { - mp_parse_node_t nodes[3] = {0}; - if (!validate_argument(emit, opcode, opcode_data, argument_nodes[0], 0)) { - return false; - } - nodes[0] = argument_nodes[0]; - bool negative = false; - if (!parse_register_offset_node(emit, opcode, opcode_data, argument_nodes[1], 1, &nodes[1], &nodes[2], &negative)) { - return false; - } - - mp_uint_t rd = 0; - mp_uint_t rs1 = 0; - if (!parse_register_node(nodes[0], &rd, opcode_data->argument1_kind & C)) { - return false; - } - if (!parse_register_node(nodes[1], &rs1, opcode_data->argument3_kind & C)) { - return false; - } - - mp_obj_t object; - mp_parse_node_get_int_maybe(nodes[2], &object); - mp_uint_t immediate = mp_obj_get_int_truncated(object) << opcode_data->argument2_shift; - if (negative) { - immediate = (~immediate + 1) & (mp_uint_t)-1; - } - if (!is_in_signed_mask(OPCODE_MASKS[opcode_data->argument2_mask], immediate)) { - emit_inline_rv32_error_exc(emit, - mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, ET_OUT_OF_RANGE, opcode, 2)); - return false; - } - - ((call_rri_t)opcode_data->emitter)(&emit->as, rd, rs1, immediate); - return true; -} - static void emit_inline_rv32_opcode(emit_inline_asm_t *emit, qstr opcode, mp_uint_t arguments_count, mp_parse_node_t *argument_nodes) { const opcode_t *opcode_data = NULL; for (mp_uint_t index = 0; index < MP_ARRAY_SIZE(OPCODES); index++) { @@ -785,10 +702,9 @@ static void emit_inline_rv32_opcode(emit_inline_asm_t *emit, qstr opcode, mp_uin } } - if (!opcode_data) { - emit_inline_rv32_error_exc(emit, - mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, - MP_ERROR_TEXT("unknown RV32 instruction '%q'"), opcode)); + if (!opcode_data || (asm_rv32_allowed_extensions() & opcode_data->required_extensions) != opcode_data->required_extensions) { + emit_inline_rv32_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, + MP_ERROR_TEXT("invalid RV32 instruction '%q'"), opcode)); return; } @@ -796,36 +712,36 @@ static void emit_inline_rv32_opcode(emit_inline_asm_t *emit, qstr opcode, mp_uin assert((opcode_data->argument2_mask < MP_ARRAY_SIZE(OPCODE_MASKS)) && "Argument #2 opcode mask index out of bounds."); assert((opcode_data->argument3_mask < MP_ARRAY_SIZE(OPCODE_MASKS)) && "Argument #3 opcode mask index out of bounds."); assert((opcode_data->calling_convention < CALL_COUNT) && "Calling convention index out of bounds."); - if (opcode_data->calling_convention != CALL_RIR) { - if (opcode_data->arguments_count != arguments_count) { - emit_inline_rv32_error_exc(emit, - mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, - ET_WRONG_ARGUMENTS_COUNT, opcode, opcode_data->arguments_count)); - return; - } - if (opcode_data->arguments_count >= 1 && !validate_argument(emit, opcode, opcode_data, argument_nodes[0], 0)) { + mp_uint_t serialised_arguments[3] = { 0 }; + if (arguments_count != opcode_data->parse_nodes) { + emit_inline_rv32_error_exc(emit, + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, + ET_WRONG_ARGUMENTS_COUNT, opcode, opcode_data->parse_nodes)); + return; + } + + if (opcode_data->parse_nodes >= 1 && !serialise_argument(emit, opcode_data, argument_nodes[0], 0, &serialised_arguments[0])) { + return; + } + if (opcode_data->calling_convention == CALL_RIR) { + // "register, offset(base)" calls require some preprocessing to + // split the offset and base nodes - not to mention that if the offset + // is negative, the parser won't see the offset as a single node but as + // a sequence of the minus sign token followed by the number itself. + + if (!serialise_register_offset_node(emit, opcode_data, argument_nodes[1], 1, &serialised_arguments[1], &serialised_arguments[2])) { return; } - if (opcode_data->arguments_count >= 2 && !validate_argument(emit, opcode, opcode_data, argument_nodes[1], 1)) { + } else { + if (opcode_data->parse_nodes >= 2 && !serialise_argument(emit, opcode_data, argument_nodes[1], 1, &serialised_arguments[1])) { return; } - if (opcode_data->arguments_count >= 3 && !validate_argument(emit, opcode, opcode_data, argument_nodes[2], 2)) { + if (opcode_data->parse_nodes >= 3 && !serialise_argument(emit, opcode_data, argument_nodes[2], 2, &serialised_arguments[2])) { return; } - handle_opcode(emit, opcode, opcode_data, argument_nodes); - return; - } - - assert((opcode_data->argument2_kind & U) == 0 && "Offset must not be unsigned."); - assert((opcode_data->argument2_kind & Z) == 0 && "Offset can be zero."); - - if (arguments_count != 2) { - emit_inline_rv32_error_exc(emit, - mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, ET_WRONG_ARGUMENTS_COUNT, opcode, 2)); - return; } - handle_load_store_opcode_with_offset(emit, opcode, opcode_data, argument_nodes); + handle_opcode(emit, opcode_data, serialised_arguments); } #undef N diff --git a/py/emitinlinextensa.c b/py/emitinlinextensa.c index fed259cfc6b20..d0eb3d566fce3 100644 --- a/py/emitinlinextensa.c +++ b/py/emitinlinextensa.c @@ -173,7 +173,7 @@ static int get_arg_label(emit_inline_asm_t *emit, const char *op, mp_parse_node_ #define RRI8_B (2) typedef struct _opcode_table_3arg_t { - uint16_t name; // actually a qstr, which should fit in 16 bits + qstr_short_t name; uint8_t type; uint8_t a0 : 4; uint8_t a1 : 4; @@ -187,6 +187,13 @@ static const opcode_table_3arg_t opcode_table_3arg[] = { {MP_QSTR_add, RRR, 0, 8}, {MP_QSTR_sub, RRR, 0, 12}, {MP_QSTR_mull, RRR, 2, 8}, + {MP_QSTR_addx2, RRR, 0, 9}, + {MP_QSTR_addx4, RRR, 0, 10}, + {MP_QSTR_addx8, RRR, 0, 11}, + {MP_QSTR_subx2, RRR, 0, 13}, + {MP_QSTR_subx4, RRR, 0, 14}, + {MP_QSTR_subx8, RRR, 0, 15}, + {MP_QSTR_src, RRR, 1, 8}, // load/store/addi opcodes: reg, reg, imm // upper nibble of type encodes the range of the immediate arg @@ -208,21 +215,62 @@ static const opcode_table_3arg_t opcode_table_3arg[] = { {MP_QSTR_bge, RRI8_B, ASM_XTENSA_CC_GE, 0}, {MP_QSTR_bgeu, RRI8_B, ASM_XTENSA_CC_GEU, 0}, {MP_QSTR_blt, RRI8_B, ASM_XTENSA_CC_LT, 0}, + {MP_QSTR_bltu, RRI8_B, ASM_XTENSA_CC_LTU, 0}, {MP_QSTR_bnall, RRI8_B, ASM_XTENSA_CC_NALL, 0}, {MP_QSTR_bne, RRI8_B, ASM_XTENSA_CC_NE, 0}, {MP_QSTR_bnone, RRI8_B, ASM_XTENSA_CC_NONE, 0}, }; +// The index of the first four qstrs matches the CCZ condition value to be +// embedded into the opcode. +static const qstr_short_t BCCZ_OPCODES[] = { + MP_QSTR_beqz, MP_QSTR_bnez, MP_QSTR_bltz, MP_QSTR_bgez, + MP_QSTR_beqz_n, MP_QSTR_bnez_n +}; + +#if MICROPY_EMIT_INLINE_XTENSA_UNCOMMON_OPCODES +typedef struct _single_opcode_t { + qstr_short_t name; + uint16_t value; +} single_opcode_t; + +static const single_opcode_t NOARGS_OPCODES[] = { + {MP_QSTR_dsync, 0x2030}, + {MP_QSTR_esync, 0x2020}, + {MP_QSTR_extw, 0x20D0}, + {MP_QSTR_ill, 0x0000}, + {MP_QSTR_isync, 0x2000}, + {MP_QSTR_memw, 0x20C0}, + {MP_QSTR_rsync, 0x2010}, +}; +#endif + static void emit_inline_xtensa_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_args, mp_parse_node_t *pn_args) { size_t op_len; const char *op_str = (const char *)qstr_data(op, &op_len); if (n_args == 0) { - if (op == MP_QSTR_ret_n) { + if (op == MP_QSTR_ret_n || op == MP_QSTR_ret) { asm_xtensa_op_ret_n(&emit->as); - } else { - goto unknown_op; + return; + } else if (op == MP_QSTR_nop) { + asm_xtensa_op24(&emit->as, 0x20F0); + return; + } else if (op == MP_QSTR_nop_n) { + asm_xtensa_op16(&emit->as, 0xF03D); + return; } + #if MICROPY_EMIT_INLINE_XTENSA_UNCOMMON_OPCODES + for (size_t index = 0; index < MP_ARRAY_SIZE(NOARGS_OPCODES); index++) { + const single_opcode_t *opcode = &NOARGS_OPCODES[index]; + if (op == opcode->name) { + asm_xtensa_op24(&emit->as, opcode->value); + return; + } + } + #endif + + goto unknown_op; } else if (n_args == 1) { if (op == MP_QSTR_callx0) { @@ -234,19 +282,45 @@ static void emit_inline_xtensa_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_ } else if (op == MP_QSTR_jx) { uint r0 = get_arg_reg(emit, op_str, pn_args[0]); asm_xtensa_op_jx(&emit->as, r0); + } else if (op == MP_QSTR_ssl) { + mp_uint_t r0 = get_arg_reg(emit, op_str, pn_args[0]); + asm_xtensa_op_ssl(&emit->as, r0); + } else if (op == MP_QSTR_ssr) { + mp_uint_t r0 = get_arg_reg(emit, op_str, pn_args[0]); + asm_xtensa_op_ssr(&emit->as, r0); + } else if (op == MP_QSTR_ssai) { + mp_uint_t sa = get_arg_i(emit, op_str, pn_args[0], 0, 31); + asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRR(0, 0, 4, 4, sa & 0x0F, (sa >> 4) & 0x01)); + } else if (op == MP_QSTR_ssa8b) { + mp_uint_t r0 = get_arg_reg(emit, op_str, pn_args[0]); + asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRR(0, 0, 4, 3, r0, 0)); + } else if (op == MP_QSTR_ssa8l) { + mp_uint_t r0 = get_arg_reg(emit, op_str, pn_args[0]); + asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRR(0, 0, 4, 2, r0, 0)); + } else if (op == MP_QSTR_call0) { + mp_uint_t label = get_arg_label(emit, op_str, pn_args[0]); + asm_xtensa_call0(&emit->as, label); + #if MICROPY_EMIT_INLINE_XTENSA_UNCOMMON_OPCODES + } else if (op == MP_QSTR_fsync) { + mp_uint_t imm3 = get_arg_i(emit, op_str, pn_args[0], 0, 7); + asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRR(0, 0, 0, 2, 8 | imm3, 0)); + } else if (op == MP_QSTR_ill_n) { + asm_xtensa_op16(&emit->as, 0xF06D); + #endif } else { goto unknown_op; } } else if (n_args == 2) { uint r0 = get_arg_reg(emit, op_str, pn_args[0]); - if (op == MP_QSTR_beqz) { - int label = get_arg_label(emit, op_str, pn_args[1]); - asm_xtensa_bccz_reg_label(&emit->as, ASM_XTENSA_CCZ_EQ, r0, label); - } else if (op == MP_QSTR_bnez) { - int label = get_arg_label(emit, op_str, pn_args[1]); - asm_xtensa_bccz_reg_label(&emit->as, ASM_XTENSA_CCZ_NE, r0, label); - } else if (op == MP_QSTR_mov || op == MP_QSTR_mov_n) { + for (size_t index = 0; index < MP_ARRAY_SIZE(BCCZ_OPCODES); index++) { + if (op == BCCZ_OPCODES[index]) { + mp_uint_t label = get_arg_label(emit, op_str, pn_args[1]); + asm_xtensa_bccz_reg_label(&emit->as, index & 0x03, r0, label); + return; + } + } + if (op == MP_QSTR_mov || op == MP_QSTR_mov_n) { // we emit mov.n for both "mov" and "mov_n" opcodes uint r1 = get_arg_reg(emit, op_str, pn_args[1]); asm_xtensa_op_mov_n(&emit->as, r0, r1); @@ -254,7 +328,53 @@ static void emit_inline_xtensa_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_ // for convenience we emit l32r if the integer doesn't fit in movi uint32_t imm = get_arg_i(emit, op_str, pn_args[1], 0, 0); asm_xtensa_mov_reg_i32(&emit->as, r0, imm); - } else { + } else if (op == MP_QSTR_abs_) { + mp_uint_t r1 = get_arg_reg(emit, op_str, pn_args[1]); + asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRR(0, 0, 6, r0, 1, r1)); + } else if (op == MP_QSTR_neg) { + mp_uint_t r1 = get_arg_reg(emit, op_str, pn_args[1]); + asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRR(0, 0, 6, r0, 0, r1)); + } else if (op == MP_QSTR_sll) { + mp_uint_t r1 = get_arg_reg(emit, op_str, pn_args[1]); + asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRR(0, 1, 10, r0, r1, 0)); + } else if (op == MP_QSTR_sra) { + mp_uint_t r1 = get_arg_reg(emit, op_str, pn_args[1]); + asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRR(0, 1, 11, r0, 0, r1)); + } else if (op == MP_QSTR_srl) { + mp_uint_t r1 = get_arg_reg(emit, op_str, pn_args[1]); + asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRR(0, 1, 9, r0, 0, r1)); + } else if (op == MP_QSTR_nsa) { + mp_uint_t r1 = get_arg_reg(emit, op_str, pn_args[1]); + asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRR(0, 0, 4, 14, r1, r0)); + } else if (op == MP_QSTR_nsau) { + mp_uint_t r1 = get_arg_reg(emit, op_str, pn_args[1]); + asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRR(0, 0, 4, 15, r1, r0)); + } else if (op == MP_QSTR_l32r) { + mp_uint_t label = get_arg_label(emit, op_str, pn_args[1]); + asm_xtensa_l32r(&emit->as, r0, label); + } else if (op == MP_QSTR_movi_n) { + mp_int_t imm = get_arg_i(emit, op_str, pn_args[1], -32, 95); + asm_xtensa_op_movi_n(&emit->as, r0, imm); + } else + #if MICROPY_EMIT_INLINE_XTENSA_UNCOMMON_OPCODES + if (op == MP_QSTR_rsr) { + mp_uint_t sr = get_arg_i(emit, op_str, pn_args[1], 0, 255); + asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RSR(0, 3, 0, sr, r0)); + } else if (op == MP_QSTR_rur) { + mp_uint_t imm8 = get_arg_i(emit, op_str, pn_args[1], 0, 255); + asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRR(0, 3, 14, r0, (imm8 >> 4) & 0x0F, imm8 & 0x0F)); + } else if (op == MP_QSTR_wsr) { + mp_uint_t sr = get_arg_i(emit, op_str, pn_args[1], 0, 255); + asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RSR(0, 3, 1, sr, r0)); + } else if (op == MP_QSTR_wur) { + mp_uint_t sr = get_arg_i(emit, op_str, pn_args[1], 0, 255); + asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RSR(0, 3, 15, sr, r0)); + } else if (op == MP_QSTR_xsr) { + mp_uint_t sr = get_arg_i(emit, op_str, pn_args[1], 0, 255); + asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RSR(0, 1, 6, sr, r0)); + } else + #endif + { goto unknown_op; } @@ -288,7 +408,72 @@ static void emit_inline_xtensa_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_ return; } } - goto unknown_op; + + if (op == MP_QSTR_add_n) { + mp_uint_t r0 = get_arg_reg(emit, op_str, pn_args[0]); + mp_uint_t r1 = get_arg_reg(emit, op_str, pn_args[1]); + mp_uint_t r2 = get_arg_reg(emit, op_str, pn_args[2]); + asm_xtensa_op16(&emit->as, ASM_XTENSA_ENCODE_RRRN(10, r0, r1, r2)); + } else if (op == MP_QSTR_addi_n) { + mp_uint_t r0 = get_arg_reg(emit, op_str, pn_args[0]); + mp_uint_t r1 = get_arg_reg(emit, op_str, pn_args[1]); + mp_int_t imm4 = get_arg_i(emit, op_str, pn_args[2], -1, 15); + asm_xtensa_op16(&emit->as, ASM_XTENSA_ENCODE_RRRN(11, r0, r1, (imm4 != 0 ? imm4 : -1))); + } else if (op == MP_QSTR_addmi) { + mp_uint_t r0 = get_arg_reg(emit, op_str, pn_args[0]); + mp_uint_t r1 = get_arg_reg(emit, op_str, pn_args[1]); + mp_int_t imm8 = get_arg_i(emit, op_str, pn_args[2], -128 * 256, 127 * 256); + if ((imm8 & 0xFF) != 0) { + emit_inline_xtensa_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, MP_ERROR_TEXT("%d is not a multiple of %d"), imm8, 256)); + } else { + asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRI8(2, 13, r1, r0, imm8 >> 8)); + } + } else if (op == MP_QSTR_bbci) { + mp_uint_t r0 = get_arg_reg(emit, op_str, pn_args[0]); + mp_uint_t bit = get_arg_i(emit, op_str, pn_args[1], 0, 31); + mp_int_t label = get_arg_label(emit, op_str, pn_args[2]); + asm_xtensa_bit_branch(&emit->as, r0, bit, label, 6); + } else if (op == MP_QSTR_bbsi) { + mp_uint_t r0 = get_arg_reg(emit, op_str, pn_args[0]); + mp_uint_t bit = get_arg_i(emit, op_str, pn_args[1], 0, 31); + mp_uint_t label = get_arg_label(emit, op_str, pn_args[2]); + asm_xtensa_bit_branch(&emit->as, r0, bit, label, 14); + } else if (op == MP_QSTR_slli) { + mp_uint_t r0 = get_arg_reg(emit, op_str, pn_args[0]); + mp_uint_t r1 = get_arg_reg(emit, op_str, pn_args[1]); + mp_uint_t bits = 32 - get_arg_i(emit, op_str, pn_args[2], 1, 31); + asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRR(0, 1, 0 | ((bits >> 4) & 0x01), r0, r1, bits & 0x0F)); + } else if (op == MP_QSTR_srai) { + mp_uint_t r0 = get_arg_reg(emit, op_str, pn_args[0]); + mp_uint_t r1 = get_arg_reg(emit, op_str, pn_args[1]); + mp_uint_t bits = get_arg_i(emit, op_str, pn_args[2], 0, 31); + asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRR(0, 1, 2 | ((bits >> 4) & 0x01), r0, bits & 0x0F, r1)); + } else if (op == MP_QSTR_srli) { + mp_uint_t r0 = get_arg_reg(emit, op_str, pn_args[0]); + mp_uint_t r1 = get_arg_reg(emit, op_str, pn_args[1]); + mp_uint_t bits = get_arg_i(emit, op_str, pn_args[2], 0, 15); + asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRR(0, 1, 4, r0, bits, r1)); + } else if (op == MP_QSTR_l32i_n) { + mp_uint_t r0 = get_arg_reg(emit, op_str, pn_args[0]); + mp_uint_t r1 = get_arg_reg(emit, op_str, pn_args[1]); + mp_uint_t imm = get_arg_i(emit, op_str, pn_args[2], 0, 60); + if ((imm & 0x03) != 0) { + emit_inline_xtensa_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, MP_ERROR_TEXT("%d is not a multiple of %d"), imm, 4)); + } else { + asm_xtensa_op_l32i_n(&emit->as, r0, r1, imm >> 2); + } + } else if (op == MP_QSTR_s32i_n) { + mp_uint_t r0 = get_arg_reg(emit, op_str, pn_args[0]); + mp_uint_t r1 = get_arg_reg(emit, op_str, pn_args[1]); + mp_uint_t imm = get_arg_i(emit, op_str, pn_args[2], 0, 60); + if ((imm & 0x03) != 0) { + emit_inline_xtensa_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, MP_ERROR_TEXT("%d is not a multiple of %d"), imm, 4)); + } else { + asm_xtensa_op_s32i_n(&emit->as, r0, r1, imm >> 2); + } + } else { + goto unknown_op; + } } else { goto unknown_op; diff --git a/py/emitnative.c b/py/emitnative.c index a888418e5dfed..6f12200f7e171 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -222,12 +222,6 @@ static const uint8_t reg_local_table[MAX_REGS_FOR_LOCAL_VARS] = {REG_LOCAL_1, RE *emit->error_slot = mp_obj_new_exception_msg_varg(&mp_type_ViperTypeError, __VA_ARGS__); \ } while (0) -#if N_RV32 -#define FIT_SIGNED(value, bits) \ - ((((value) & ~((1U << ((bits) - 1)) - 1)) == 0) || \ - (((value) & ~((1U << ((bits) - 1)) - 1)) == ~((1U << ((bits) - 1)) - 1))) -#endif - typedef enum { STACK_VALUE, STACK_REG, @@ -335,12 +329,27 @@ struct _emit_t { #define ASM_CLR_REG(state, rd) ASM_XOR_REG_REG(state, rd, rd) #endif +#if N_RV32 +#define ASM_MOV_LOCAL_MP_OBJ_NULL(as, local_num, reg_temp) \ + ASM_MOV_LOCAL_REG(as, local_num, REG_ZERO) +#else +#define ASM_MOV_LOCAL_MP_OBJ_NULL(as, local_num, reg_temp) \ + ASM_MOV_REG_IMM(as, reg_temp, (mp_uint_t)MP_OBJ_NULL); \ + ASM_MOV_LOCAL_REG(as, local_num, reg_temp) +#endif + static void emit_load_reg_with_object(emit_t *emit, int reg, mp_obj_t obj); static void emit_native_global_exc_entry(emit_t *emit); static void emit_native_global_exc_exit(emit_t *emit); static void emit_native_load_const_obj(emit_t *emit, mp_obj_t obj); emit_t *EXPORT_FUN(new)(mp_emit_common_t * emit_common, mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels) { + // Generated code performing exception handling assumes that MP_OBJ_NULL + // equals to 0 to simplify some checks, leveraging dedicated opcodes for + // comparisons against 0. If this assumption does not hold true anymore + // then generated code won't work correctly. + MP_STATIC_ASSERT(MP_OBJ_NULL == 0); + emit_t *emit = m_new0(emit_t, 1); emit->emit_common = emit_common; emit->error_slot = error_slot; @@ -467,6 +476,30 @@ static void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop emit->stack_info[i].vtype = VTYPE_UNBOUND; } + char *qualified_name = NULL; + + #if N_DEBUG + scope_t *current_scope = scope; + vstr_t *qualified_name_vstr = vstr_new(qstr_len(current_scope->simple_name)); + size_t fragment_length = 0; + const byte *fragment_pointer; + for (;;) { + fragment_pointer = qstr_data(current_scope->simple_name, &fragment_length); + vstr_hint_size(qualified_name_vstr, fragment_length); + memmove(qualified_name_vstr->buf + fragment_length, qualified_name_vstr->buf, qualified_name_vstr->len); + memcpy(qualified_name_vstr->buf, fragment_pointer, fragment_length); + qualified_name_vstr->len += fragment_length; + if (current_scope->parent == NULL || current_scope->parent->simple_name == MP_QSTR__lt_module_gt_) { + break; + } + vstr_ins_char(qualified_name_vstr, 0, '.'); + current_scope = current_scope->parent; + } + qualified_name = vstr_null_terminated_str(qualified_name_vstr); + #else + (void)qualified_name; + #endif + mp_asm_base_start_pass(&emit->as->base, pass == MP_PASS_EMIT ? MP_ASM_PASS_EMIT : MP_ASM_PASS_COMPUTE); // generate code for entry to function @@ -513,7 +546,7 @@ static void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop } // Entry to function - ASM_ENTRY(emit->as, emit->stack_start + emit->n_state - num_locals_in_regs); + ASM_ENTRY(emit->as, emit->stack_start + emit->n_state - num_locals_in_regs, qualified_name); #if N_X86 asm_x86_mov_arg_to_r32(emit->as, 0, REG_PARENT_ARG_1); @@ -584,7 +617,7 @@ static void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop if (emit->scope->scope_flags & MP_SCOPE_FLAG_GENERATOR) { mp_asm_base_data(&emit->as->base, ASM_WORD_SIZE, (uintptr_t)emit->start_offset); - ASM_ENTRY(emit->as, emit->code_state_start); + ASM_ENTRY(emit->as, emit->code_state_start, qualified_name); // Reset the state size for the state pointed to by REG_GENERATOR_STATE emit->code_state_start = 0; @@ -616,7 +649,7 @@ static void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop emit->stack_start = emit->code_state_start + SIZEOF_CODE_STATE; // Allocate space on C-stack for code_state structure, which includes state - ASM_ENTRY(emit->as, emit->stack_start + emit->n_state); + ASM_ENTRY(emit->as, emit->stack_start + emit->n_state, qualified_name); // Prepare incoming arguments for call to mp_setup_code_state @@ -682,6 +715,10 @@ static void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop } } } + + #if N_DEBUG + vstr_free(qualified_name_vstr); + #endif } static inline void emit_native_write_code_info_byte(emit_t *emit, byte val) { @@ -1311,8 +1348,7 @@ static void emit_native_global_exc_entry(emit_t *emit) { // Check LOCAL_IDX_THROW_VAL for any injected value ASM_MOV_REG_LOCAL(emit->as, REG_ARG_1, LOCAL_IDX_THROW_VAL(emit)); - ASM_MOV_REG_IMM(emit->as, REG_ARG_2, (mp_uint_t)MP_OBJ_NULL); - ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_THROW_VAL(emit), REG_ARG_2); + ASM_MOV_LOCAL_MP_OBJ_NULL(emit->as, LOCAL_IDX_THROW_VAL(emit), REG_ARG_2); emit_call(emit, MP_F_NATIVE_RAISE); } } @@ -1579,87 +1615,50 @@ static void emit_native_load_subscr(emit_t *emit) { switch (vtype_base) { case VTYPE_PTR8: { // pointer to 8-bit memory - // TODO optimise to use thumb ldrb r1, [r2, r3] + #ifdef ASM_LOAD8_REG_REG_OFFSET + ASM_LOAD8_REG_REG_OFFSET(emit->as, REG_RET, reg_base, index_value); + #else if (index_value != 0) { // index is non-zero - #if N_THUMB - if (index_value > 0 && index_value < 32) { - asm_thumb_ldrb_rlo_rlo_i5(emit->as, REG_RET, reg_base, index_value); - break; - } - #elif N_RV32 - if (FIT_SIGNED(index_value, 12)) { - asm_rv32_opcode_lbu(emit->as, REG_RET, reg_base, index_value); - break; - } - #elif N_XTENSA || N_XTENSAWIN - if (index_value > 0 && index_value < 256) { - asm_xtensa_op_l8ui(emit->as, REG_RET, reg_base, index_value); - break; - } - #endif need_reg_single(emit, reg_index, 0); ASM_MOV_REG_IMM(emit->as, reg_index, index_value); ASM_ADD_REG_REG(emit->as, reg_index, reg_base); // add index to base reg_base = reg_index; } ASM_LOAD8_REG_REG(emit->as, REG_RET, reg_base); // load from (base+index) + #endif break; } case VTYPE_PTR16: { // pointer to 16-bit memory + #ifdef ASM_LOAD16_REG_REG_OFFSET + ASM_LOAD16_REG_REG_OFFSET(emit->as, REG_RET, reg_base, index_value); + #else if (index_value != 0) { // index is a non-zero immediate - #if N_THUMB - if (index_value > 0 && index_value < 32) { - asm_thumb_ldrh_rlo_rlo_i5(emit->as, REG_RET, reg_base, index_value); - break; - } - #elif N_RV32 - if (FIT_SIGNED(index_value, 11)) { - asm_rv32_opcode_lhu(emit->as, REG_RET, reg_base, index_value << 1); - break; - } - #elif N_XTENSA || N_XTENSAWIN - if (index_value > 0 && index_value < 256) { - asm_xtensa_op_l16ui(emit->as, REG_RET, reg_base, index_value); - break; - } - #endif need_reg_single(emit, reg_index, 0); ASM_MOV_REG_IMM(emit->as, reg_index, index_value << 1); ASM_ADD_REG_REG(emit->as, reg_index, reg_base); // add 2*index to base reg_base = reg_index; } ASM_LOAD16_REG_REG(emit->as, REG_RET, reg_base); // load from (base+2*index) + #endif break; } case VTYPE_PTR32: { // pointer to 32-bit memory + #ifdef ASM_LOAD32_REG_REG_OFFSET + ASM_LOAD32_REG_REG_OFFSET(emit->as, REG_RET, reg_base, index_value); + #else if (index_value != 0) { // index is a non-zero immediate - #if N_THUMB - if (index_value > 0 && index_value < 32) { - asm_thumb_ldr_rlo_rlo_i5(emit->as, REG_RET, reg_base, index_value); - break; - } - #elif N_RV32 - if (FIT_SIGNED(index_value, 10)) { - asm_rv32_opcode_lw(emit->as, REG_RET, reg_base, index_value << 2); - break; - } - #elif N_XTENSA || N_XTENSAWIN - if (index_value > 0 && index_value < 256) { - asm_xtensa_l32i_optimised(emit->as, REG_RET, reg_base, index_value); - break; - } - #endif need_reg_single(emit, reg_index, 0); ASM_MOV_REG_IMM(emit->as, reg_index, index_value << 2); ASM_ADD_REG_REG(emit->as, reg_index, reg_base); // add 4*index to base reg_base = reg_index; } ASM_LOAD32_REG_REG(emit->as, REG_RET, reg_base); // load from (base+4*index) + #endif break; } default: @@ -1680,40 +1679,36 @@ static void emit_native_load_subscr(emit_t *emit) { switch (vtype_base) { case VTYPE_PTR8: { // pointer to 8-bit memory - // TODO optimise to use thumb ldrb r1, [r2, r3] + #ifdef ASM_LOAD8_REG_REG_REG + ASM_LOAD8_REG_REG_REG(emit->as, REG_RET, REG_ARG_1, reg_index); + #else ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base ASM_LOAD8_REG_REG(emit->as, REG_RET, REG_ARG_1); // store value to (base+index) + #endif break; } case VTYPE_PTR16: { // pointer to 16-bit memory - #if N_XTENSA || N_XTENSAWIN - asm_xtensa_op_addx2(emit->as, REG_ARG_1, reg_index, REG_ARG_1); - asm_xtensa_op_l16ui(emit->as, REG_RET, REG_ARG_1, 0); - break; - #endif + #ifdef ASM_LOAD16_REG_REG_REG + ASM_LOAD16_REG_REG_REG(emit->as, REG_RET, REG_ARG_1, reg_index); + #else ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base ASM_LOAD16_REG_REG(emit->as, REG_RET, REG_ARG_1); // load from (base+2*index) + #endif break; } case VTYPE_PTR32: { // pointer to word-size memory - #if N_RV32 - asm_rv32_opcode_slli(emit->as, REG_TEMP2, reg_index, 2); - asm_rv32_opcode_cadd(emit->as, REG_ARG_1, REG_TEMP2); - asm_rv32_opcode_lw(emit->as, REG_RET, REG_ARG_1, 0); - break; - #elif N_XTENSA || N_XTENSAWIN - asm_xtensa_op_addx4(emit->as, REG_ARG_1, reg_index, REG_ARG_1); - asm_xtensa_op_l32i_n(emit->as, REG_RET, REG_ARG_1, 0); - break; - #endif + #ifdef ASM_LOAD32_REG_REG_REG + ASM_LOAD32_REG_REG_REG(emit->as, REG_RET, REG_ARG_1, reg_index); + #else ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base ASM_LOAD32_REG_REG(emit->as, REG_RET, REG_ARG_1); // load from (base+4*index) + #endif break; } default: @@ -1856,92 +1851,47 @@ static void emit_native_store_subscr(emit_t *emit) { switch (vtype_base) { case VTYPE_PTR8: { // pointer to 8-bit memory - // TODO optimise to use thumb strb r1, [r2, r3] + #ifdef ASM_STORE8_REG_REG_OFFSET + ASM_STORE8_REG_REG_OFFSET(emit->as, reg_value, reg_base, index_value); + #else if (index_value != 0) { // index is non-zero - #if N_THUMB - if (index_value > 0 && index_value < 32) { - asm_thumb_strb_rlo_rlo_i5(emit->as, reg_value, reg_base, index_value); - break; - } - #elif N_RV32 - if (FIT_SIGNED(index_value, 12)) { - asm_rv32_opcode_sb(emit->as, reg_value, reg_base, index_value); - break; - } - #elif N_XTENSA || N_XTENSAWIN - if (index_value > 0 && index_value < 256) { - asm_xtensa_op_s8i(emit->as, REG_RET, reg_base, index_value); - break; - } - #endif ASM_MOV_REG_IMM(emit->as, reg_index, index_value); - #if N_ARM - asm_arm_strb_reg_reg_reg(emit->as, reg_value, reg_base, reg_index); - return; - #endif ASM_ADD_REG_REG(emit->as, reg_index, reg_base); // add index to base reg_base = reg_index; } ASM_STORE8_REG_REG(emit->as, reg_value, reg_base); // store value to (base+index) + #endif break; } case VTYPE_PTR16: { // pointer to 16-bit memory + #ifdef ASM_STORE16_REG_REG_OFFSET + ASM_STORE16_REG_REG_OFFSET(emit->as, reg_value, reg_base, index_value); + #else if (index_value != 0) { // index is a non-zero immediate - #if N_THUMB - if (index_value > 0 && index_value < 32) { - asm_thumb_strh_rlo_rlo_i5(emit->as, reg_value, reg_base, index_value); - break; - } - #elif N_RV32 - if (FIT_SIGNED(index_value, 11)) { - asm_rv32_opcode_sh(emit->as, reg_value, reg_base, index_value << 1); - break; - } - #elif N_XTENSA || N_XTENSAWIN - if (index_value > 0 && index_value < 256) { - asm_xtensa_op_s16i(emit->as, REG_RET, reg_base, index_value); - break; - } - #endif ASM_MOV_REG_IMM(emit->as, reg_index, index_value << 1); ASM_ADD_REG_REG(emit->as, reg_index, reg_base); // add 2*index to base reg_base = reg_index; } ASM_STORE16_REG_REG(emit->as, reg_value, reg_base); // store value to (base+2*index) + #endif break; } case VTYPE_PTR32: { // pointer to 32-bit memory + #ifdef ASM_STORE32_REG_REG_OFFSET + ASM_STORE32_REG_REG_OFFSET(emit->as, reg_value, reg_base, index_value); + #else if (index_value != 0) { // index is a non-zero immediate - #if N_THUMB - if (index_value > 0 && index_value < 32) { - asm_thumb_str_rlo_rlo_i5(emit->as, reg_value, reg_base, index_value); - break; - } - #elif N_RV32 - if (FIT_SIGNED(index_value, 10)) { - asm_rv32_opcode_sw(emit->as, reg_value, reg_base, index_value << 2); - break; - } - #elif N_XTENSA || N_XTENSAWIN - if (index_value > 0 && index_value < 256) { - asm_xtensa_s32i_optimised(emit->as, REG_RET, reg_base, index_value); - break; - } - #elif N_ARM - ASM_MOV_REG_IMM(emit->as, reg_index, index_value); - asm_arm_str_reg_reg_reg(emit->as, reg_value, reg_base, reg_index); - return; - #endif ASM_MOV_REG_IMM(emit->as, reg_index, index_value << 2); ASM_ADD_REG_REG(emit->as, reg_index, reg_base); // add 4*index to base reg_base = reg_index; } ASM_STORE32_REG_REG(emit->as, reg_value, reg_base); // store value to (base+4*index) + #endif break; } default: @@ -1972,50 +1922,36 @@ static void emit_native_store_subscr(emit_t *emit) { switch (vtype_base) { case VTYPE_PTR8: { // pointer to 8-bit memory - // TODO optimise to use thumb strb r1, [r2, r3] - #if N_ARM - asm_arm_strb_reg_reg_reg(emit->as, reg_value, REG_ARG_1, reg_index); - break; - #endif + #ifdef ASM_STORE8_REG_REG_REG + ASM_STORE8_REG_REG_REG(emit->as, reg_value, REG_ARG_1, reg_index); + #else ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base ASM_STORE8_REG_REG(emit->as, reg_value, REG_ARG_1); // store value to (base+index) + #endif break; } case VTYPE_PTR16: { // pointer to 16-bit memory - #if N_ARM - asm_arm_strh_reg_reg_reg(emit->as, reg_value, REG_ARG_1, reg_index); - break; - #elif N_XTENSA || N_XTENSAWIN - asm_xtensa_op_addx2(emit->as, REG_ARG_1, reg_index, REG_ARG_1); - asm_xtensa_op_s16i(emit->as, reg_value, REG_ARG_1, 0); - break; - #endif + #ifdef ASM_STORE16_REG_REG_REG + ASM_STORE16_REG_REG_REG(emit->as, reg_value, REG_ARG_1, reg_index); + #else ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base ASM_STORE16_REG_REG(emit->as, reg_value, REG_ARG_1); // store value to (base+2*index) + #endif break; } case VTYPE_PTR32: { // pointer to 32-bit memory - #if N_ARM - asm_arm_str_reg_reg_reg(emit->as, reg_value, REG_ARG_1, reg_index); - break; - #elif N_RV32 - asm_rv32_opcode_slli(emit->as, REG_TEMP2, reg_index, 2); - asm_rv32_opcode_cadd(emit->as, REG_ARG_1, REG_TEMP2); - asm_rv32_opcode_sw(emit->as, reg_value, REG_ARG_1, 0); - break; - #elif N_XTENSA || N_XTENSAWIN - asm_xtensa_op_addx4(emit->as, REG_ARG_1, reg_index, REG_ARG_1); - asm_xtensa_op_s32i_n(emit->as, reg_value, REG_ARG_1, 0); - break; - #endif + #ifdef ASM_STORE32_REG_REG_REG + ASM_STORE32_REG_REG_REG(emit->as, reg_value, REG_ARG_1, reg_index); + #else ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base ASM_STORE32_REG_REG(emit->as, reg_value, REG_ARG_1); // store value to (base+4*index) + #endif break; } default: @@ -2328,8 +2264,7 @@ static void emit_native_with_cleanup(emit_t *emit, mp_uint_t label) { // Replace exception with MP_OBJ_NULL. emit_native_label_assign(emit, *emit->label_slot); - ASM_MOV_REG_IMM(emit->as, REG_TEMP0, (mp_uint_t)MP_OBJ_NULL); - ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_VAL(emit), REG_TEMP0); + ASM_MOV_LOCAL_MP_OBJ_NULL(emit->as, LOCAL_IDX_EXC_VAL(emit), REG_TEMP0); // end of with cleanup nlr_catch block emit_native_label_assign(emit, *emit->label_slot + 1); @@ -2436,8 +2371,7 @@ static void emit_native_for_iter_end(emit_t *emit) { static void emit_native_pop_except_jump(emit_t *emit, mp_uint_t label, bool within_exc_handler) { if (within_exc_handler) { // Cancel any active exception so subsequent handlers don't see it - ASM_MOV_REG_IMM(emit->as, REG_TEMP0, (mp_uint_t)MP_OBJ_NULL); - ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_VAL(emit), REG_TEMP0); + ASM_MOV_LOCAL_MP_OBJ_NULL(emit->as, LOCAL_IDX_EXC_VAL(emit), REG_TEMP0); } else { emit_native_leave_exc_stack(emit, false); } @@ -3122,8 +3056,7 @@ static void emit_native_yield(emit_t *emit, int kind) { if (kind == MP_EMIT_YIELD_VALUE) { // Check LOCAL_IDX_THROW_VAL for any injected value ASM_MOV_REG_LOCAL(emit->as, REG_ARG_1, LOCAL_IDX_THROW_VAL(emit)); - ASM_MOV_REG_IMM(emit->as, REG_ARG_2, (mp_uint_t)MP_OBJ_NULL); - ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_THROW_VAL(emit), REG_ARG_2); + ASM_MOV_LOCAL_MP_OBJ_NULL(emit->as, LOCAL_IDX_THROW_VAL(emit), REG_ARG_2); emit_call(emit, MP_F_NATIVE_RAISE); } else { // Label loop entry diff --git a/py/emitndebug.c b/py/emitndebug.c index bd896a75c8ddf..e49c5cdbffa7b 100644 --- a/py/emitndebug.c +++ b/py/emitndebug.c @@ -108,8 +108,8 @@ static void asm_debug_end_pass(asm_debug_t *as) { (void)as; } -static void asm_debug_entry(asm_debug_t *as, int num_locals) { - asm_debug_printf(as, "ENTRY(num_locals=%d)\n", num_locals); +static void asm_debug_entry(asm_debug_t *as, int num_locals, char *name) { + asm_debug_printf(as, "ENTRY(%s, num_locals=%d)\n", name != NULL ? name : "?", num_locals); } static void asm_debug_exit(asm_debug_t *as) { @@ -195,8 +195,8 @@ static void asm_debug_setcc_reg_reg_reg(asm_debug_t *as, int op, int reg1, int r #define ASM_T asm_debug_t #define ASM_END_PASS asm_debug_end_pass -#define ASM_ENTRY(as, num_locals) \ - asm_debug_entry(as, num_locals) +#define ASM_ENTRY(as, num_locals, name) \ + asm_debug_entry(as, num_locals, name) #define ASM_EXIT(as) \ asm_debug_exit(as) @@ -251,8 +251,6 @@ static void asm_debug_setcc_reg_reg_reg(asm_debug_t *as, int op, int reg1, int r #define ASM_MUL_REG_REG(as, reg_dest, reg_src) \ asm_debug_reg_reg(as, "mul", reg_dest, reg_src) -#define ASM_LOAD_REG_REG(as, reg_dest, reg_base) \ - asm_debug_reg_reg(as, "load", reg_dest, reg_base) #define ASM_LOAD_REG_REG_OFFSET(as, reg_dest, reg_base, word_offset) \ asm_debug_reg_reg_offset(as, "load", reg_dest, reg_base, word_offset) #define ASM_LOAD8_REG_REG(as, reg_dest, reg_base) \ @@ -264,8 +262,6 @@ static void asm_debug_setcc_reg_reg_reg(asm_debug_t *as, int op, int reg1, int r #define ASM_LOAD32_REG_REG(as, reg_dest, reg_base) \ asm_debug_reg_reg(as, "load32", reg_dest, reg_base) -#define ASM_STORE_REG_REG(as, reg_src, reg_base) \ - asm_debug_reg_reg(as, "store", reg_src, reg_base) #define ASM_STORE_REG_REG_OFFSET(as, reg_src, reg_base, word_offset) \ asm_debug_reg_reg_offset(as, "store", reg_src, reg_base, word_offset) #define ASM_STORE8_REG_REG(as, reg_src, reg_base) \ diff --git a/py/formatfloat.c b/py/formatfloat.c index b4348122ff428..1ea34f84bf7fb 100644 --- a/py/formatfloat.c +++ b/py/formatfloat.c @@ -33,394 +33,537 @@ #include #include #include "py/formatfloat.h" +#include "py/parsenum.h" /*********************************************************************** Routine for converting a arbitrary floating point number into a string. - The code in this function was inspired from Fred Bayer's pdouble.c. - Since pdouble.c was released as Public Domain, I'm releasing this - code as public domain as well. + The code in this function was inspired from Dave Hylands's previous + version, which was itself inspired from Fred Bayer's pdouble.c. The original code can be found in https://github.com/dhylands/format-float - Dave Hylands - ***********************************************************************/ -#if MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_FLOAT -// 1 sign bit, 8 exponent bits, and 23 mantissa bits. -// exponent values 0 and 255 are reserved, exponent can be 1 to 254. -// exponent is stored with a bias of 127. -// The min and max floats are on the order of 1x10^37 and 1x10^-37 - -#define FPTYPE float -#define FPCONST(x) x##F -#define FPROUND_TO_ONE 0.9999995F -#define FPDECEXP 32 -#define FPMIN_BUF_SIZE 6 // +9e+99 +// Float formatting debug code is intended for use in ports/unix only, +// as it uses the libc float printing function as a reference. +#define DEBUG_FLOAT_FORMATTING 0 + +#if DEBUG_FLOAT_FORMATTING +#define DEBUG_PRINTF(...) fprintf(stderr, __VA_ARGS__) +#else +#define DEBUG_PRINTF(...) +#endif + +#if MICROPY_FLOAT_FORMAT_IMPL == MICROPY_FLOAT_FORMAT_IMPL_EXACT || MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_DOUBLE +#define MP_FFUINT_FMT "%lu" +#else +#define MP_FFUINT_FMT "%u" +#endif + +static inline int fp_expval(mp_float_t x) { + mp_float_union_t fb = { x }; + return (int)fb.p.exp - MP_FLOAT_EXP_OFFSET; +} -#define FLT_SIGN_MASK 0x80000000 +#if MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_DOUBLE -static inline int fp_signbit(float x) { - mp_float_union_t fb = {x}; - return fb.i & FLT_SIGN_MASK; +static inline int fp_isless1(mp_float_t x) { + return x < 1.0; } -#define fp_isnan(x) isnan(x) -#define fp_isinf(x) isinf(x) -static inline int fp_iszero(float x) { - mp_float_union_t fb = {x}; - return fb.i == 0; + +static inline int fp_iszero(mp_float_t x) { + return x == 0.0; } -static inline int fp_isless1(float x) { - mp_float_union_t fb = {x}; + +#if MICROPY_FLOAT_FORMAT_IMPL != MICROPY_FLOAT_FORMAT_IMPL_APPROX +static inline int fp_equal(mp_float_t x, mp_float_t y) { + return x == y; +} +#else +static inline mp_float_t fp_diff(mp_float_t x, mp_float_t y) { + return x - y; +} +#endif + +#elif MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_FLOAT + +// The functions below are roughly equivalent to the ones above, +// but they are optimized to reduce code footprint by skipping +// handling for special values such as nan, inf, +/-0.0 +// for ports where FP support is done in software. +// +// They also take into account lost bits of REPR_C as needed. + +static inline int fp_isless1(mp_float_t x) { + mp_float_union_t fb = { x }; return fb.i < 0x3f800000; } -#elif MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_DOUBLE +static inline int fp_iszero(mp_float_t x) { + mp_float_union_t x_check = { x }; + return !x_check.i; // this is valid for REPR_C as well +} -// CIRCUITPY-CHANGE: prevent warnings -#pragma GCC diagnostic ignored "-Wfloat-equal" -#define FPTYPE double -#define FPCONST(x) x -#define FPROUND_TO_ONE 0.999999999995 -#define FPDECEXP 256 -#define FPMIN_BUF_SIZE 7 // +9e+199 -#define fp_signbit(x) signbit(x) -#define fp_isnan(x) isnan(x) -#define fp_isinf(x) isinf(x) -#define fp_iszero(x) (x == 0) -#define fp_isless1(x) (x < 1.0) +#if MICROPY_FLOAT_FORMAT_IMPL != MICROPY_FLOAT_FORMAT_IMPL_APPROX +static inline int fp_equal(mp_float_t x, mp_float_t y) { + mp_float_union_t x_check = { x }; + mp_float_union_t y_check = { y }; + #if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_C + return (x_check.i & ~3) == (y_check.i & ~3); + #else + return x_check.i == y_check.i; + #endif +} +#else +static inline mp_float_t fp_diff(mp_float_t x, mp_float_t y) { + #if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_C + mp_float_union_t x_check = { x }; + mp_float_union_t y_check = { y }; + x_check.i &= ~3; + y_check.i &= ~3; + return x_check.f - y_check.f; + #else + return x - y; + #endif +} +#endif -#endif // MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_FLOAT/DOUBLE +#endif -static inline int fp_expval(FPTYPE x) { - mp_float_union_t fb = {x}; - return (int)((fb.i >> MP_FLOAT_FRAC_BITS) & (~(0xFFFFFFFF << MP_FLOAT_EXP_BITS))) - MP_FLOAT_EXP_OFFSET; +#if MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_FLOAT +#define FPMIN_BUF_SIZE 6 // +9e+99 +#define MAX_MANTISSA_DIGITS (9) +#define SAFE_MANTISSA_DIGITS (6) +#elif MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_DOUBLE +#define FPMIN_BUF_SIZE 7 // +9e+199 +#define MAX_MANTISSA_DIGITS (19) +#define SAFE_MANTISSA_DIGITS (16) +#endif + +// Internal formatting flags +#define FMT_MODE_E 0x01 // render using scientific notation (%e) +#define FMT_MODE_G 0x02 // render using general format (%g) +#define FMT_MODE_F 0x04 // render using using expanded fixed-point format (%f) +#define FMT_E_CASE 0x20 // don't change this value (used for case conversion!) + +static char *mp_prepend_zeros(char *s, int cnt) { + *s++ = '0'; + *s++ = '.'; + while (cnt > 0) { + *s++ = '0'; + cnt--; + } + return s; } -int mp_format_float(FPTYPE f, char *buf, size_t buf_size, char fmt, int prec, char sign) { +// Helper to convert a decimal mantissa (provided as an mp_large_float_uint_t) to string +static int mp_format_mantissa(mp_large_float_uint_t mantissa, mp_large_float_uint_t mantissa_cap, char *buf, char *s, + int num_digits, int max_exp_zeros, int trailing_zeros, int dec, int e, int fmt_flags) { - char *s = buf; + DEBUG_PRINTF("mantissa=" MP_FFUINT_FMT " exp=%d (cap=" MP_FFUINT_FMT "):\n", mantissa, e, mantissa_cap); - if (buf_size <= FPMIN_BUF_SIZE) { - // FPMIN_BUF_SIZE is the minimum size needed to store any FP number. - // If the buffer does not have enough room for this (plus null terminator) - // then don't try to format the float. + if (mantissa) { + // If rounding/searching created an extra digit or removed too many, fix mantissa first + if (mantissa >= mantissa_cap) { + if (fmt_flags & FMT_MODE_F) { + assert(e >= 0); + num_digits++; + dec++; + } else { + mantissa /= 10; + e++; + } + } + } - if (buf_size >= 2) { - *s++ = '?'; + // When 'g' format is used, replace small exponents by explicit zeros + if ((fmt_flags & FMT_MODE_G) && e != 0) { + if (e >= 0) { + // If 0 < e < max_exp_zeros, expand positive exponent into trailing zeros + if (e < max_exp_zeros) { + dec += e; + if (dec >= num_digits) { + trailing_zeros = dec - (num_digits - 1); + } + e = 0; + } + } else { + // If -4 <= e < 0, expand negative exponent without losing significant digits + if (e >= -4) { + int cnt = 0; + while (e < 0 && !(mantissa % 10)) { + mantissa /= 10; + cnt++; + e++; + } + num_digits -= cnt; + s = mp_prepend_zeros(s, cnt - e - 1); + dec = 255; + e = 0; + } } - if (buf_size >= 1) { - *s = '\0'; + } + + // Convert the integer mantissa to string + for (int digit = num_digits - 1; digit >= 0; digit--) { + int digit_ofs = (digit > dec ? digit + 1 : digit); + s[digit_ofs] = '0' + (int)(mantissa % 10); + mantissa /= 10; + } + int dot = (dec >= 255); + if (dec + 1 < num_digits) { + dot = 1; + s++; + s[dec] = '.'; + } + s += num_digits; + #if DEBUG_FLOAT_FORMATTING + *s = 0; + DEBUG_PRINTF(" = %s exp=%d num_digits=%d zeros=%d dec=%d\n", buf, e, num_digits, trailing_zeros, dec); + #endif + + // Append or remove trailing zeros, as required by format + if (trailing_zeros) { + dec -= num_digits - 1; + while (trailing_zeros--) { + if (!dec--) { + *s++ = '.'; + dot = 1; + } + *s++ = '0'; } - return buf_size >= 2; } - if (fp_signbit(f) && !fp_isnan(f)) { - *s++ = '-'; - f = -f; - } else { - if (sign) { - *s++ = sign; + if (fmt_flags & FMT_MODE_G) { + // 'g' format requires to remove trailing zeros after decimal point + if (dot) { + while (s[-1] == '0') { + s--; + } + if (s[-1] == '.') { + s--; + } + } + } + + // Append the exponent if needed + if (((e != 0) || (fmt_flags & FMT_MODE_E)) && !(fmt_flags & FMT_MODE_F)) { + *s++ = 'E' | (fmt_flags & FMT_E_CASE); + if (e >= 0) { + *s++ = '+'; + } else { + *s++ = '-'; + e = -e; } + if (e >= 100) { + *s++ = '0' + (e / 100); + } + *s++ = '0' + ((e / 10) % 10); + *s++ = '0' + (e % 10); } + *s = '\0'; + DEBUG_PRINTF(" ===> %s\n", buf); - // buf_remaining contains bytes available for digits and exponent. - // It is buf_size minus room for the sign and null byte. - int buf_remaining = buf_size - 1 - (s - buf); + return s - buf; +} +// minimal value expected for buf_size, to avoid checking everywhere for overflow +#define MIN_BUF_SIZE (MAX_MANTISSA_DIGITS + 10) + +int mp_format_float(mp_float_t f_entry, char *buf_entry, size_t buf_size, char fmt, int prec, char sign) { + assert(buf_size >= MIN_BUF_SIZE); + + // Handle sign + mp_float_t f = f_entry; + char *buf = buf_entry; + if (signbit(f_entry) && !isnan(f_entry)) { + f = -f; + sign = '-'; + } + if (sign) { + *buf++ = sign; + buf_size--; + } + + // Handle inf/nan + char uc = fmt & 0x20; { - char uc = fmt & 0x20; - if (fp_isinf(f)) { + char *s = buf; + if (isinf(f)) { *s++ = 'I' ^ uc; *s++ = 'N' ^ uc; *s++ = 'F' ^ uc; goto ret; - } else if (fp_isnan(f)) { + } else if (isnan(f)) { *s++ = 'N' ^ uc; *s++ = 'A' ^ uc; *s++ = 'N' ^ uc; ret: *s = '\0'; - return s - buf; + return s - buf_entry; } } + // Decode format character + int fmt_flags = (unsigned char)uc; // setup FMT_E_CASE, clear all other bits + char lofmt = (char)(fmt | 0x20); // fmt in lowercase + if (lofmt == 'f') { + fmt_flags |= FMT_MODE_F; + } else if (lofmt == 'g') { + fmt_flags |= FMT_MODE_G; + } else { + fmt_flags |= FMT_MODE_E; + } + + // When precision is unspecified, default to 6 if (prec < 0) { prec = 6; } - char e_char = 'E' | (fmt & 0x20); // e_char will match case of fmt - fmt |= 0x20; // Force fmt to be lowercase - char org_fmt = fmt; - if (fmt == 'g' && prec == 0) { - prec = 1; + // Use high precision for `repr`, but switch to exponent mode + // after 16 digits in any case to match CPython behaviour + int max_exp_zeros = (prec < (int)buf_size - 3 ? prec : (int)buf_size - 3); + if (prec == MP_FLOAT_REPR_PREC) { + prec = MAX_MANTISSA_DIGITS; + max_exp_zeros = 16; } - int e; - int dec = 0; - char e_sign = '\0'; - int num_digits = 0; - int signed_e = 0; - // Approximate power of 10 exponent from binary exponent. - // abs(e_guess) is lower bound on abs(power of 10 exponent). - int e_guess = (int)(fp_expval(f) * FPCONST(0.3010299956639812)); // 1/log2(10). - if (fp_iszero(f)) { - e = 0; - if (fmt == 'f') { - // Truncate precision to prevent buffer overflow - if (prec + 2 > buf_remaining) { - prec = buf_remaining - 2; - } - num_digits = prec + 1; - } else { - // Truncate precision to prevent buffer overflow - if (prec + 6 > buf_remaining) { - prec = buf_remaining - 6; - } - if (fmt == 'e') { - e_sign = '+'; - } + // Precompute the exact decimal exponent of f, such that + // abs(e) is lower bound on abs(power of 10 exponent). + int e = 0; + if (!fp_iszero(f)) { + // Approximate power of 10 exponent from binary exponent. + e = (int)(fp_expval(f) * MICROPY_FLOAT_CONST(0.3010299956639812)); // 1/log2(10). + int positive_exp = !fp_isless1(f); + mp_float_t u_base = (mp_float_t)mp_decimal_exp((mp_large_float_t)1.0, e + positive_exp); + while ((f >= u_base) == positive_exp) { + e += (positive_exp ? 1 : -1); + u_base = (mp_float_t)mp_decimal_exp((mp_large_float_t)1.0, e + positive_exp); } - } else if (fp_isless1(f)) { - FPTYPE f_entry = f; // Save f in case we go to 'f' format. - // Build negative exponent - e = -e_guess; - FPTYPE u_base = MICROPY_FLOAT_C_FUN(pow)(10, -e); - while (u_base > f) { - ++e; - u_base = MICROPY_FLOAT_C_FUN(pow)(10, -e); - } - // Normalize out the inferred unit. Use divide because - // pow(10, e) * pow(10, -e) is slightly < 1 for some e in float32 - // (e.g. print("%.12f" % ((1e13) * (1e-13)))) - f /= u_base; - - // If the user specified 'g' format, and e is <= 4, then we'll switch - // to the fixed format ('f') - - if (fmt == 'f' || (fmt == 'g' && e <= 4)) { - fmt = 'f'; - dec = 0; + } - if (org_fmt == 'g') { - prec += (e - 1); - } + // For 'e' format, prec is # digits after the decimal + // For 'f' format, prec is # digits after the decimal + // For 'g' format, prec is the max number of significant digits + // + // For 'e' & 'g' format, there will be a single digit before the decimal + // For 'f' format, zeros must be expanded instead of using an exponent. + // Make sure there is enough room in the buffer for them, or switch to format 'g'. + if ((fmt_flags & FMT_MODE_F) && e > 0) { + int req_size = e + prec + 2; + if (req_size > (int)buf_size) { + fmt_flags ^= FMT_MODE_F; + fmt_flags |= FMT_MODE_G; + prec++; + } + } - // truncate precision to prevent buffer overflow - if (prec + 2 > buf_remaining) { - prec = buf_remaining - 2; + // To work independently of the format, we precompute: + // - the max number of significant digits to produce + // - the number of leading zeros to prepend (mode f only) + // - the number of trailing zeros to append + int max_digits = prec; + int lead_zeros = 0; + int trail_zeros = 0; + if (fmt_flags & FMT_MODE_F) { + if (max_digits > (int)buf_size - 3) { + // cannot satisfy requested number of decimals given buf_size, sorry + max_digits = (int)buf_size - 3; + } + if (e < 0) { + if (max_digits > 2 && e < -2) { + // Insert explicit leading zeros + lead_zeros = (-e < max_digits ? -e : max_digits) - 2; + max_digits -= lead_zeros; + } else { + max_digits++; } - - num_digits = prec; - signed_e = 0; - f = f_entry; - ++num_digits; } else { - // For e & g formats, we'll be printing the exponent, so set the - // sign. - e_sign = '-'; - dec = 0; - - if (prec > (buf_remaining - FPMIN_BUF_SIZE)) { - prec = buf_remaining - FPMIN_BUF_SIZE; - if (fmt == 'g') { - prec++; - } - } - signed_e = -e; + max_digits += e + 1; } } else { - // Build positive exponent. - // We don't modify f at this point to avoid inaccuracies from - // scaling it. Instead, we find the product of powers of 10 - // that is not greater than it, and use that to start the - // mantissa. - e = e_guess; - FPTYPE next_u = MICROPY_FLOAT_C_FUN(pow)(10, e + 1); - while (f >= next_u) { - ++e; - next_u = MICROPY_FLOAT_C_FUN(pow)(10, e + 1); + if (!(fmt_flags & FMT_MODE_G) || max_digits == 0) { + max_digits++; } + } + if (max_digits > MAX_MANTISSA_DIGITS) { + // use trailing zeros to avoid overflowing the mantissa + trail_zeros = max_digits - MAX_MANTISSA_DIGITS; + max_digits = MAX_MANTISSA_DIGITS; + } + int overhead = (fmt_flags & FMT_MODE_F ? 3 : FPMIN_BUF_SIZE + 1); + if (trail_zeros > (int)buf_size - max_digits - overhead) { + // cannot satisfy requested number of decimals given buf_size, sorry + trail_zeros = (int)buf_size - max_digits - overhead; + } - // If the user specified fixed format (fmt == 'f') and e makes the - // number too big to fit into the available buffer, then we'll - // switch to the 'e' format. - - if (fmt == 'f') { - if (e >= buf_remaining) { - fmt = 'e'; - } else if ((e + prec + 2) > buf_remaining) { - prec = buf_remaining - e - 2; - if (prec < 0) { - // This means no decimal point, so we can add one back - // for the decimal. - prec++; - } - } - } - if (fmt == 'e' && prec > (buf_remaining - FPMIN_BUF_SIZE)) { - prec = buf_remaining - FPMIN_BUF_SIZE; - } - if (fmt == 'g') { - // Truncate precision to prevent buffer overflow - if (prec + (FPMIN_BUF_SIZE - 1) > buf_remaining) { - prec = buf_remaining - (FPMIN_BUF_SIZE - 1); - } - } - // If the user specified 'g' format, and e is < prec, then we'll switch - // to the fixed format. + // When the caller asks for more precision than available for sure, + // Look for a shorter (rounded) representation first, and only dig + // into more digits if there is no short representation. + int num_digits = (SAFE_MANTISSA_DIGITS < max_digits ? SAFE_MANTISSA_DIGITS : max_digits); +try_again: + ; - if (fmt == 'g' && e < prec) { - fmt = 'f'; - prec -= (e + 1); - } - if (fmt == 'f') { - dec = e; - num_digits = prec + e + 1; + char *s = buf; + int extra_zeros = trail_zeros + (max_digits - num_digits); + int decexp; + int dec = 0; + + if (fp_iszero(f)) { + // no need for scaling 0.0 + decexp = 0; + } else if (fmt_flags & FMT_MODE_F) { + decexp = num_digits - 1; + if (e < 0) { + // Negative exponent: we keep a single leading zero in the mantissa, + // as using more would waste precious digits needed for accuracy. + if (lead_zeros > 0) { + // We are using leading zeros + s = mp_prepend_zeros(s, lead_zeros); + decexp += lead_zeros + 1; + dec = 255; // no decimal dot + } else { + // Small negative exponent, work directly on the mantissa + dec = 0; + } } else { - e_sign = '+'; + // Positive exponent: we will add trailing zeros separately + decexp -= e; + dec = e; } - signed_e = e; + } else { + decexp = num_digits - e - 1; } - if (prec < 0) { - // This can happen when the prec is trimmed to prevent buffer overflow - prec = 0; + DEBUG_PRINTF("input=%.19g e=%d fmt=%c max_d=%d num_d=%d decexp=%d dec=%d l0=%d r0=%d\n", + (double)f, e, lofmt, max_digits, num_digits, decexp, dec, lead_zeros, extra_zeros); + + // At this point, + // - buf points to beginning of output buffer for the unsigned representation + // - num_digits == the number of mantissa digits to add + // - (dec + 1) == the number of digits to print before adding a decimal point + // - decexp == the power of 10 exponent to apply to f to get the decimal mantissa + // - e == the power of 10 exponent to append ('e' or 'g' format) + mp_large_float_uint_t mantissa_cap = 10; + for (int n = 1; n < num_digits; n++) { + mantissa_cap *= 10; } - // At this point e contains the absolute value of the power of 10 exponent. - // (dec + 1) == the number of dgits before the decimal. - - // For e, prec is # digits after the decimal - // For f, prec is # digits after the decimal - // For g, prec is the max number of significant digits - // - // For e & g there will be a single digit before the decimal - // for f there will be e digits before the decimal - - if (fmt == 'e') { - num_digits = prec + 1; - } else if (fmt == 'g') { - if (prec == 0) { - prec = 1; + // Build the decimal mantissa into a large uint + mp_large_float_uint_t mantissa = 1; + if (sizeof(mp_large_float_t) == sizeof(mp_float_t) && num_digits > SAFE_MANTISSA_DIGITS && decexp > 1) { + // if we don't have large floats, use integer multiply to produce the last digits + if (num_digits > SAFE_MANTISSA_DIGITS + 1 && decexp > 2) { + mantissa = 100; + decexp -= 2; + } else { + mantissa = 10; + decexp -= 1; } - num_digits = prec; } - - int d = 0; - for (int digit_index = signed_e; num_digits >= 0; --digit_index) { - FPTYPE u_base = FPCONST(1.0); - if (digit_index > 0) { - // Generate 10^digit_index for positive digit_index. - u_base = MICROPY_FLOAT_C_FUN(pow)(10, digit_index); - } - for (d = 0; d < 9; ++d) { - if (f < u_base) { - break; - } - f -= u_base; - } - // We calculate one more digit than we display, to use in rounding - // below. So only emit the digit if it's one that we display. - if (num_digits > 0) { - // Emit this number (the leading digit). - *s++ = '0' + d; - if (dec == 0 && prec > 0) { - *s++ = '.'; - } - } - --dec; - --num_digits; - if (digit_index <= 0) { - // Once we get below 1.0, we scale up f instead of calculating - // negative powers of 10 in u_base. This provides better - // renditions of exact decimals like 1/16 etc. - f *= FPCONST(10.0); + mp_large_float_t mantissa_f = mp_decimal_exp((mp_large_float_t)f, decexp); + mantissa *= (mp_large_float_uint_t)(mantissa_f + (mp_large_float_t)0.5); + DEBUG_PRINTF("input=%.19g fmt=%c num_digits=%d dec=%d mantissa=" MP_FFUINT_FMT " r0=%d\n", (double)f, lofmt, num_digits, dec, mantissa, extra_zeros); + + // Finally convert the decimal mantissa to a floating-point string, according to formatting rules + int reprlen = mp_format_mantissa(mantissa, mantissa_cap, buf, s, num_digits, max_exp_zeros, extra_zeros, dec, e, fmt_flags); + assert(reprlen + 1 <= (int)buf_size); + + #if MICROPY_FLOAT_FORMAT_IMPL != MICROPY_FLOAT_FORMAT_IMPL_APPROX + + if (num_digits < max_digits) { + // The initial precision might not be sufficient for an exact representation + // for all numbers. If the result is not exact, restart using next precision. + // parse the resulting number and compare against the original + mp_float_t check; + DEBUG_PRINTF("input=%.19g, compare to float('%s')\n", (double)f, buf); + mp_parse_float_internal(buf, reprlen, &check); + if (!fp_equal(check, f)) { + num_digits++; + DEBUG_PRINTF("Not perfect, retry using more digits (%d)\n", num_digits); + goto try_again; } } - // Rounding. If the next digit to print is >= 5, round up. - if (d >= 5) { - char *rs = s; - rs--; - while (1) { - if (*rs == '.') { - rs--; - continue; - } - if (*rs < '0' || *rs > '9') { - // + or - - rs++; // So we sit on the digit to the right of the sign - break; + + #else + + // The initial decimal mantissa might not have been be completely accurate due + // to the previous loating point operations. The best way to verify this is to + // parse the resulting number and compare against the original + mp_float_t check; + DEBUG_PRINTF("input=%.19g, compare to float('%s')\n", (double)f, buf); + mp_parse_float_internal(buf, reprlen, &check); + mp_float_t diff = fp_diff(check, f); + mp_float_t best_diff = diff; + mp_large_float_uint_t best_mantissa = mantissa; + + if (fp_iszero(diff)) { + // we have a perfect match + DEBUG_PRINTF(MP_FFUINT_FMT ": perfect match (direct)\n", mantissa); + } else { + // In order to get the best possible representation, we will perform a + // dichotomic search for a reversible representation. + // This will also provide optimal rounding on the fly. + unsigned err_range = 1; + if (num_digits > SAFE_MANTISSA_DIGITS) { + err_range <<= 3 * (num_digits - SAFE_MANTISSA_DIGITS); + } + int maxruns = 3 + 3 * (MAX_MANTISSA_DIGITS - SAFE_MANTISSA_DIGITS); + while (maxruns-- > 0) { + // update mantissa according to dichotomic search + if (signbit(diff)) { + mantissa += err_range; + } else { + // mantissa is expected to always have more significant digits than err_range + assert(mantissa >= err_range); + mantissa -= err_range; } - if (*rs < '9') { - (*rs)++; + // retry conversion + reprlen = mp_format_mantissa(mantissa, mantissa_cap, buf, s, num_digits, max_exp_zeros, extra_zeros, dec, e, fmt_flags); + assert(reprlen + 1 <= (int)buf_size); + DEBUG_PRINTF("input=%.19g, compare to float('%s')\n", (double)f, buf); + mp_parse_float_internal(buf, reprlen, &check); + DEBUG_PRINTF("check=%.19g num_digits=%d e=%d mantissa=" MP_FFUINT_FMT "\n", (double)check, num_digits, e, mantissa); + diff = fp_diff(check, f); + if (fp_iszero(diff)) { + // we have a perfect match + DEBUG_PRINTF(MP_FFUINT_FMT ": perfect match\n", mantissa); break; } - *rs = '0'; - if (rs == buf) { - break; + // keep track of our best estimate + mp_float_t delta = MICROPY_FLOAT_C_FUN(fabs)(diff) - MICROPY_FLOAT_C_FUN(fabs)(best_diff); + if (signbit(delta) || (fp_iszero(delta) && !(mantissa % 10u))) { + best_diff = diff; + best_mantissa = mantissa; } - rs--; - } - if (*rs == '0') { - // We need to insert a 1 - if (rs[1] == '.' && fmt != 'f') { - // We're going to round 9.99 to 10.00 - // Move the decimal point - rs[0] = '.'; - rs[1] = '0'; - if (e_sign == '-') { - e--; - if (e == 0) { - e_sign = '+'; - } - } else { - e++; - } + // string repr is not perfect: continue a dichotomic improvement + DEBUG_PRINTF(MP_FFUINT_FMT ": %.19g, err_range=%d\n", mantissa, (double)check, err_range); + if (err_range > 1) { + err_range >>= 1; } else { - // Need at extra digit at the end to make room for the leading '1' - // but if we're at the buffer size limit, just drop the final digit. - if ((size_t)(s + 1 - buf) < buf_size) { - s++; + // We have tried all possible mantissa, without finding a reversible repr. + // Check if we have an alternate precision to try. + if (num_digits < max_digits) { + num_digits++; + DEBUG_PRINTF("Failed to find a perfect match, try with more digits (%d)\n", num_digits); + goto try_again; } + // Otherwise, keep the closest one, which is either the first one or the last one. + if (mantissa == best_mantissa) { + // Last guess is the best one + DEBUG_PRINTF(MP_FFUINT_FMT ": last guess was the best one\n", mantissa); + } else { + // We had a better guess earlier + DEBUG_PRINTF(MP_FFUINT_FMT ": use best guess\n", mantissa); + reprlen = mp_format_mantissa(best_mantissa, mantissa_cap, buf, s, num_digits, max_exp_zeros, extra_zeros, dec, e, fmt_flags); + } + break; } - char *ss = s; - while (ss > rs) { - *ss = ss[-1]; - ss--; - } - *rs = '1'; } } + #endif - // verify that we did not overrun the input buffer so far - assert((size_t)(s + 1 - buf) <= buf_size); - - if (org_fmt == 'g' && prec > 0) { - // Remove trailing zeros and a trailing decimal point - while (s[-1] == '0') { - s--; - } - if (s[-1] == '.') { - s--; - } - } - // Append the exponent - if (e_sign) { - *s++ = e_char; - *s++ = e_sign; - if (FPMIN_BUF_SIZE == 7 && e >= 100) { - *s++ = '0' + (e / 100); - } - *s++ = '0' + ((e / 10) % 10); - *s++ = '0' + (e % 10); - } - *s = '\0'; - - // verify that we did not overrun the input buffer - assert((size_t)(s + 1 - buf) <= buf_size); - - return s - buf; + return buf + reprlen - buf_entry; } #endif // MICROPY_FLOAT_IMPL != MICROPY_FLOAT_IMPL_NONE diff --git a/py/formatfloat.h b/py/formatfloat.h index 9a1643b4ddff5..7b1414672b77b 100644 --- a/py/formatfloat.h +++ b/py/formatfloat.h @@ -29,6 +29,7 @@ #include "py/mpconfig.h" #if MICROPY_PY_BUILTINS_FLOAT +#define MP_FLOAT_REPR_PREC (99) // magic `prec` value for optimal `repr` behaviour int mp_format_float(mp_float_t f, char *buf, size_t bufSize, char fmt, int prec, char sign); #endif diff --git a/py/gc.c b/py/gc.c index fc7de6c4d3b4f..757c27e3de696 100644 --- a/py/gc.c +++ b/py/gc.c @@ -1447,7 +1447,7 @@ void gc_dump_alloc_table(const mp_print_t *print) { } if (bl2 - bl >= 2 * DUMP_BYTES_PER_LINE) { // there are at least 2 lines containing only free blocks, so abbreviate their printing - mp_printf(print, "\n (%u lines all free)", (uint)(bl2 - bl) / DUMP_BYTES_PER_LINE); + mp_printf(print, "\n (%u lines all free)", (uint)((bl2 - bl) / DUMP_BYTES_PER_LINE)); bl = bl2 & (~(DUMP_BYTES_PER_LINE - 1)); if (bl >= area->gc_alloc_table_byte_len * BLOCKS_PER_ATB) { // got to end of heap @@ -1490,7 +1490,7 @@ void gc_dump_alloc_table(const mp_print_t *print) { break; } */ - /* this prints the uPy object type of the head block */ + /* this prints the MicroPython object type of the head block */ case AT_HEAD: { // CIRCUITPY-CHANGE: compiler warning avoidance #pragma GCC diagnostic push diff --git a/py/gc.h b/py/gc.h index 0752478d1f286..f9028d637c5c6 100644 --- a/py/gc.h +++ b/py/gc.h @@ -71,6 +71,11 @@ bool gc_alloc_possible(void); // Use this function to sweep the whole heap and run all finalisers void gc_sweep_all(void); +// These functions are used to manage weakrefs. +void gc_weakref_mark(void *ptr); +void gc_weakref_about_to_be_freed(void *ptr); +void gc_weakref_sweep(void); + enum { GC_ALLOC_FLAG_HAS_FINALISER = 1, // CIRCUITPY-CHANGE diff --git a/py/make_root_pointers.py b/py/make_root_pointers.py index efe398b8227ca..5da343c1270ec 100644 --- a/py/make_root_pointers.py +++ b/py/make_root_pointers.py @@ -6,8 +6,6 @@ "struct _mp_state_vm_t" in py/mpstate.h """ -from __future__ import print_function - import argparse import io import re diff --git a/py/makecompresseddata.py b/py/makecompresseddata.py index 1bce3e8e8374a..a6fa7db00b280 100644 --- a/py/makecompresseddata.py +++ b/py/makecompresseddata.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import collections import re import sys diff --git a/py/makemoduledefs.py b/py/makemoduledefs.py index 29162ab387c8b..c8fdc21f6fbe2 100644 --- a/py/makemoduledefs.py +++ b/py/makemoduledefs.py @@ -14,8 +14,6 @@ the built-in version. """ -from __future__ import print_function - import sys import re import io @@ -87,19 +85,25 @@ def generate_module_table_header(modules): ) ) + # There should always be at least one module (__main__ in runtime.c) + assert mod_defs + print("\n#define MICROPY_REGISTERED_MODULES \\") for mod_def in sorted(mod_defs): print(" {mod_def} \\".format(mod_def=mod_def)) - print("// MICROPY_REGISTERED_MODULES") - print("\n#define MICROPY_REGISTERED_EXTENSIBLE_MODULES \\") + # There are not necessarily any extensible modules (e.g., bare-arm or minimal x86) + print("\n#define MICROPY_HAVE_REGISTERED_EXTENSIBLE_MODULES ", len(extensible_mod_defs)) - for mod_def in sorted(extensible_mod_defs): - print(" {mod_def} \\".format(mod_def=mod_def)) + if extensible_mod_defs: + print("\n#define MICROPY_REGISTERED_EXTENSIBLE_MODULES \\") + + for mod_def in sorted(extensible_mod_defs): + print(" {mod_def} \\".format(mod_def=mod_def)) - print("// MICROPY_REGISTERED_EXTENSIBLE_MODULES") + print("// MICROPY_REGISTERED_EXTENSIBLE_MODULES") def generate_module_delegations(delegations): diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py index 998328bf1d250..3fdb2d5476b8a 100644 --- a/py/makeqstrdata.py +++ b/py/makeqstrdata.py @@ -1,31 +1,13 @@ """ Process raw qstr file and output qstr data with length, hash and data bytes. -This script works with Python 2.7, 3.3 and 3.4. - -CIRCUITPY-CHANGE: -For documentation about the format of compressed translated strings, see -supervisor/shared/translate/translate.h +This script works with Python 3.3+. """ -from __future__ import print_function - - import re import sys -# CIRCUITPY-CHANGE -if hasattr(sys.stdout, "reconfigure"): - sys.stdout.reconfigure(encoding="utf-8") - sys.stderr.reconfigure(errors="backslashreplace") - -# Python 2/3 compatibility: -# - iterating through bytes is different -# - codepoint2name from html.entities is hard-coded -if sys.version_info[0] == 2: - bytes_cons = lambda val, enc=None: bytearray(val) -elif sys.version_info[0] == 3: # Also handles MicroPython - bytes_cons = bytes +bytes_cons = bytes # fmt: off codepoint2name = { @@ -67,7 +49,6 @@ 253: "yacute", 165: "yen", 255: "yuml", 950: "zeta", 8205: "zwj", 8204: "zwnj" } # fmt: on -# end compatibility code codepoint2name[ord("-")] = "hyphen" diff --git a/py/makeqstrdefs.py b/py/makeqstrdefs.py index a403985769489..8c07899baf843 100644 --- a/py/makeqstrdefs.py +++ b/py/makeqstrdefs.py @@ -2,11 +2,9 @@ This script processes the output from the C preprocessor and extracts all qstr. Each qstr is transformed into a qstr definition of the form 'Q(...)'. -This script works with Python 3.x (CIRCUITPY-CHANGE: not 2.x) +This script works with Python 3.3+. """ -from __future__ import print_function - import io import os import re @@ -139,6 +137,7 @@ def qstr_unescape(qstr): return qstr +# CIRCUITPY-CHANGE: output_filename as an arg def process_file(f, output_filename=None): # match gcc-like output (# n "file") and msvc-like output (#line n "file") re_line = re.compile(r"^#(?:line)?\s+\d+\s\"([^\"]+)\"") @@ -292,6 +291,7 @@ class Args: args.input_filename = sys.argv[3] # Unused for command=cat args.output_dir = sys.argv[4] args.output_file = None if len(sys.argv) == 5 else sys.argv[5] # Unused for command=split + # CIRCUITPY-CHANGE if args.output_file == "_": args.output_file = None @@ -306,6 +306,7 @@ class Args: if args.command == "split": with io.open(args.input_filename, encoding="utf-8") as infile: + # CIRCUITPY-CHANGE: pass output_file process_file(infile, args.output_file) if args.command == "cat": diff --git a/py/makeversionhdr.py b/py/makeversionhdr.py index 307bfac1e9adb..94321ef0bad4d 100644 --- a/py/makeversionhdr.py +++ b/py/makeversionhdr.py @@ -5,8 +5,6 @@ This script works with Python 3.7 and newer """ -from __future__ import print_function - import argparse import sys import os diff --git a/py/malloc.c b/py/malloc.c index fcf930ecfaa83..fc9795043a879 100644 --- a/py/malloc.c +++ b/py/malloc.c @@ -54,9 +54,11 @@ // freely accessed - for interfacing with system and 3rd-party libs for // example. On the other hand, some (e.g. bare-metal) ports may use GC // heap as system heap, so, to avoid warnings, we do undef's first. -// CIRCUITPY-CHANGE: Add selective collect support to malloc to optimize GC for large buffers +#undef malloc #undef free #undef realloc +#define malloc(b) gc_alloc((b), false) +#define malloc_with_finaliser(b) gc_alloc((b), true) #define free gc_free #define realloc(ptr, n) gc_realloc(ptr, n, true) #define realloc_ext(ptr, n, mv) gc_realloc(ptr, n, mv) @@ -68,6 +70,7 @@ #error MICROPY_ENABLE_FINALISER requires MICROPY_ENABLE_GC #endif +// CIRCUITPY-CHANGE: Add selective collect support to malloc to optimize GC for large buffers #if MICROPY_ENABLE_SELECTIVE_COLLECT #error MICROPY_ENABLE_SELECTIVE_COLLECT requires MICROPY_ENABLE_GC #endif @@ -85,7 +88,7 @@ static void *realloc_ext(void *ptr, size_t n_bytes, bool allow_move) { #endif // MICROPY_ENABLE_GC -// CIRCUITPY-CHANGE: Add malloc helper with flags instead of a list of bools. +// CIRCUITPY-CHANGE: Add malloc helper to factor out flag handling and allow combinations. void *m_malloc_helper(size_t num_bytes, uint8_t flags) { void *ptr; #if MICROPY_ENABLE_GC @@ -112,7 +115,6 @@ void *m_malloc_helper(size_t num_bytes, uint8_t flags) { MP_STATE_MEM(current_bytes_allocated) += num_bytes; UPDATE_PEAK(); #endif - // CIRCUITPY-CHANGE // If this config is set then the GC clears all memory, so we don't need to. #if !MICROPY_GC_CONSERVATIVE_CLEAR if (flags & M_MALLOC_ENSURE_ZEROED) { @@ -124,26 +126,37 @@ void *m_malloc_helper(size_t num_bytes, uint8_t flags) { } void *m_malloc(size_t num_bytes) { - // CIRCUITPY-CHANGE + // CIRCUITPY-CHANGE: use helper return m_malloc_helper(num_bytes, M_MALLOC_RAISE_ERROR | M_MALLOC_COLLECT); } void *m_malloc_maybe(size_t num_bytes) { - // CIRCUITPY-CHANGE + // CIRCUITPY-CHANGE: use helper return m_malloc_helper(num_bytes, M_MALLOC_COLLECT); } +#if MICROPY_ENABLE_FINALISER +void *m_malloc_with_finaliser(size_t num_bytes) { + // CIRCUITPY-CHANGE: use helper + return m_malloc_helper(num_bytes, M_MALLOC_COLLECT | M_MALLOC_WITH_FINALISER); +} +#endif + void *m_malloc0(size_t num_bytes) { - return m_malloc_helper(num_bytes, M_MALLOC_ENSURE_ZEROED | M_MALLOC_RAISE_ERROR | M_MALLOC_COLLECT); + // CIRCUITPY-CHANGE: use helper + return m_malloc_helper(num_bytes, + (MICROPY_GC_CONSERVATIVE_CLEAR ? 0 : M_MALLOC_ENSURE_ZEROED) + | M_MALLOC_RAISE_ERROR | M_MALLOC_COLLECT); } +// CIRCUITPY-CHANGE: add selective collect void *m_malloc_without_collect(size_t num_bytes) { - // CIRCUITPY-CHANGE return m_malloc_helper(num_bytes, M_MALLOC_RAISE_ERROR); } +// CIRCUITPY-CHANGE: add selective collect void *m_malloc_maybe_without_collect(size_t num_bytes) { - // CIRCUITPY-CHANGE + return m_malloc_helper(num_bytes, 0); } @@ -224,6 +237,31 @@ void m_free(void *ptr) #if MICROPY_TRACKED_ALLOC +#if MICROPY_PY_THREAD && !MICROPY_PY_THREAD_GIL +// If there's no GIL, use the GC recursive mutex to protect the tracked node linked list +// under m_tracked_head. +// +// (For ports with GIL, the expectation is to only call tracked alloc functions +// while holding the GIL.) + +static inline void m_tracked_node_lock(void) { + mp_thread_recursive_mutex_lock(&MP_STATE_MEM(gc_mutex), 1); +} + +static inline void m_tracked_node_unlock(void) { + mp_thread_recursive_mutex_unlock(&MP_STATE_MEM(gc_mutex)); +} + +#else + +static inline void m_tracked_node_lock(void) { +} + +static inline void m_tracked_node_unlock(void) { +} + +#endif + #define MICROPY_TRACKED_ALLOC_STORE_SIZE (!MICROPY_ENABLE_GC) typedef struct _m_tracked_node_t { @@ -237,6 +275,7 @@ typedef struct _m_tracked_node_t { #if MICROPY_DEBUG_VERBOSE static size_t m_tracked_count_links(size_t *nb) { + m_tracked_node_lock(); m_tracked_node_t *node = MP_STATE_VM(m_tracked_head); size_t n = 0; *nb = 0; @@ -249,6 +288,7 @@ static size_t m_tracked_count_links(size_t *nb) { #endif node = node->next; } + m_tracked_node_unlock(); return n; } #endif @@ -263,12 +303,14 @@ void *m_tracked_calloc(size_t nmemb, size_t size) { size_t n = m_tracked_count_links(&nb); DEBUG_printf("m_tracked_calloc(%u, %u) -> (%u;%u) %p\n", (int)nmemb, (int)size, (int)n, (int)nb, node); #endif + m_tracked_node_lock(); if (MP_STATE_VM(m_tracked_head) != NULL) { MP_STATE_VM(m_tracked_head)->prev = node; } node->prev = NULL; node->next = MP_STATE_VM(m_tracked_head); MP_STATE_VM(m_tracked_head) = node; + m_tracked_node_unlock(); #if MICROPY_TRACKED_ALLOC_STORE_SIZE node->size = nmemb * size; #endif @@ -294,7 +336,8 @@ void m_tracked_free(void *ptr_in) { size_t nb; size_t n = m_tracked_count_links(&nb); DEBUG_printf("m_tracked_free(%p, [%p, %p], nbytes=%u, links=%u;%u)\n", node, node->prev, node->next, (int)data_bytes, (int)n, (int)nb); - #endif + #endif // MICROPY_DEBUG_VERBOSE + m_tracked_node_lock(); if (node->next != NULL) { node->next->prev = node->prev; } @@ -303,6 +346,7 @@ void m_tracked_free(void *ptr_in) { } else { MP_STATE_VM(m_tracked_head) = node->next; } + m_tracked_node_unlock(); m_free(node #if MICROPY_MALLOC_USES_ALLOCATED_SIZE #if MICROPY_TRACKED_ALLOC_STORE_SIZE diff --git a/py/misc.h b/py/misc.h index 22f32147124fc..d5d7950574f90 100644 --- a/py/misc.h +++ b/py/misc.h @@ -26,6 +26,8 @@ #ifndef MICROPY_INCLUDED_PY_MISC_H #define MICROPY_INCLUDED_PY_MISC_H +#include "py/mpconfig.h" + // a mini library of useful types and functions /** types *******************************************************/ @@ -33,13 +35,25 @@ #include #include #include - -// CIRCUITPY-CHANGE: include directly instead of depending on previous includes -#include "mpconfig.h" +// CIRCUITPY-CHANGE: #ifdef instead of #if +#ifdef __cplusplus // Required on at least one compiler to get ULLONG_MAX +#include +#else +#include +#endif typedef unsigned char byte; typedef unsigned int uint; +#ifndef __has_builtin +#define __has_builtin(x) (0) +#endif +#ifndef __has_feature +// This macro is supported by Clang and gcc>=14 +#define __has_feature(x) (0) +#endif + + /** generic ops *************************************************/ #ifndef MIN @@ -54,6 +68,7 @@ typedef unsigned int uint; #define MP_STRINGIFY(x) MP_STRINGIFY_HELPER(x) // Static assertion macro +// CIRCUITPY-CHANGE: defined() #if defined(__cplusplus) #define MP_STATIC_ASSERT(cond) static_assert((cond), #cond) #elif __GNUC__ >= 5 || __STDC_VERSION__ >= 201112L @@ -70,6 +85,7 @@ typedef unsigned int uint; #if defined(_MSC_VER) || defined(__cplusplus) #define MP_STATIC_ASSERT_NONCONSTEXPR(cond) ((void)1) #else +// CIRCUITPY-CHANGE: defined() #if defined(__clang__) #pragma GCC diagnostic ignored "-Wgnu-folding-constant" #endif @@ -98,9 +114,9 @@ typedef unsigned int uint; #define m_new0(type, num) ((type *)(m_malloc0(sizeof(type) * (num)))) #define m_new_obj(type) (m_new(type, 1)) #define m_new_obj_maybe(type) (m_new_maybe(type, 1)) -#define m_new_obj_var(obj_type, var_field, var_type, var_num) ((obj_type *)m_malloc_helper(offsetof(obj_type, var_field) + sizeof(var_type) * (var_num), M_MALLOC_RAISE_ERROR | M_MALLOC_COLLECT)) -#define m_new_obj_var0(obj_type, var_field, var_type, var_num) ((obj_type *)m_malloc_helper(offsetof(obj_type, var_field) + sizeof(var_type) * (var_num), M_MALLOC_ENSURE_ZEROED | M_MALLOC_RAISE_ERROR | M_MALLOC_COLLECT)) -#define m_new_obj_var_maybe(obj_type, var_field, var_type, var_num) ((obj_type *)m_malloc_helper(offsetof(obj_type, var_field) + sizeof(var_type) * (var_num), M_MALLOC_COLLECT)) +#define m_new_obj_var(obj_type, var_field, var_type, var_num) ((obj_type *)m_malloc(offsetof(obj_type, var_field) + sizeof(var_type) * (var_num))) +#define m_new_obj_var0(obj_type, var_field, var_type, var_num) ((obj_type *)m_malloc0(offsetof(obj_type, var_field) + sizeof(var_type) * (var_num))) +#define m_new_obj_var_maybe(obj_type, var_field, var_type, var_num) ((obj_type *)m_malloc_maybe(offsetof(obj_type, var_field) + sizeof(var_type) * (var_num))) #if MICROPY_MALLOC_USES_ALLOCATED_SIZE #define m_renew(type, ptr, old_num, new_num) ((type *)(m_realloc((ptr), sizeof(type) * (old_num), sizeof(type) * (new_num)))) #define m_renew_maybe(type, ptr, old_num, new_num, allow_move) ((type *)(m_realloc_maybe((ptr), sizeof(type) * (old_num), sizeof(type) * (new_num), (allow_move)))) @@ -126,9 +142,13 @@ typedef unsigned int uint; void *m_malloc_helper(size_t num_bytes, uint8_t flags); void *m_malloc(size_t num_bytes); void *m_malloc_maybe(size_t num_bytes); +void *m_malloc_with_finaliser(size_t num_bytes); void *m_malloc0(size_t num_bytes); + +// CIRCUITPY-CHANGE: _without_collect void *m_malloc_without_collect(size_t num_bytes); void *m_malloc_maybe_without_collect(size_t num_bytes); + #if MICROPY_MALLOC_USES_ALLOCATED_SIZE void *m_realloc(void *ptr, size_t old_num_bytes, size_t new_num_bytes); void *m_realloc_maybe(void *ptr, size_t old_num_bytes, size_t new_num_bytes, bool allow_move); @@ -138,7 +158,7 @@ void *m_realloc(void *ptr, size_t new_num_bytes); void *m_realloc_maybe(void *ptr, size_t new_num_bytes, bool allow_move); void m_free(void *ptr); #endif -NORETURN void m_malloc_fail(size_t num_bytes); +MP_NORETURN void m_malloc_fail(size_t num_bytes); #if MICROPY_TRACKED_ALLOC // These alloc/free functions track the pointers in a linked list so the GC does not reclaim @@ -161,7 +181,7 @@ size_t m_get_peak_bytes_allocated(void); // align ptr to the nearest multiple of "alignment" #define MP_ALIGN(ptr, alignment) (void *)(((uintptr_t)(ptr) + ((alignment) - 1)) & ~((alignment) - 1)) -// CIRCUITPY-CHANGE +// CIRCUITPY-CHANGE: addition #define sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER)) /** unichar / UTF-8 *********************************************/ @@ -308,6 +328,25 @@ typedef union _mp_float_union_t { mp_float_uint_t i; } mp_float_union_t; +#if MICROPY_FLOAT_FORMAT_IMPL == MICROPY_FLOAT_FORMAT_IMPL_EXACT + +#if MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_FLOAT +// Exact float conversion requires using internally a bigger sort of floating point +typedef double mp_large_float_t; +#elif MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_DOUBLE +typedef long double mp_large_float_t; +#endif +// Always use a 64 bit mantissa for formatting and parsing +typedef uint64_t mp_large_float_uint_t; + +#else // MICROPY_FLOAT_FORMAT_IMPL != MICROPY_FLOAT_FORMAT_IMPL_EXACT + +// No bigger floating points +typedef mp_float_t mp_large_float_t; +typedef mp_float_uint_t mp_large_float_uint_t; + +#endif + #endif // MICROPY_PY_BUILTINS_FLOAT /** ROM string compression *************/ @@ -413,26 +452,28 @@ static inline bool mp_check(bool value) { static inline uint32_t mp_popcount(uint32_t x) { return __popcnt(x); } -#else +#else // _MSC_VER #define mp_clz(x) __builtin_clz(x) #define mp_clzl(x) __builtin_clzl(x) #define mp_clzll(x) __builtin_clzll(x) #define mp_ctz(x) __builtin_ctz(x) #define mp_check(x) (x) -#if defined __has_builtin #if __has_builtin(__builtin_popcount) #define mp_popcount(x) __builtin_popcount(x) -#endif -#endif -#if !defined(mp_popcount) +#else static inline uint32_t mp_popcount(uint32_t x) { x = x - ((x >> 1) & 0x55555555); x = (x & 0x33333333) + ((x >> 2) & 0x33333333); x = (x + (x >> 4)) & 0x0F0F0F0F; - return x * 0x01010101; + return (x * 0x01010101) >> 24; } -#endif -#endif +#endif // __has_builtin(__builtin_popcount) +#endif // _MSC_VER + +#define MP_FIT_UNSIGNED(bits, value) (((value) & (~0U << (bits))) == 0) +#define MP_FIT_SIGNED(bits, value) \ + (MP_FIT_UNSIGNED(((bits) - 1), (value)) || \ + (((value) & (~0U << ((bits) - 1))) == (~0U << ((bits) - 1)))) // mp_int_t can be larger than long, i.e. Windows 64-bit, nan-box variants static inline uint32_t mp_clz_mpi(mp_int_t x) { @@ -448,16 +489,107 @@ static inline uint32_t mp_clz_mpi(mp_int_t x) { } return zeroes; #else - MP_STATIC_ASSERT(sizeof(mp_int_t) == sizeof(long long) - || sizeof(mp_int_t) == sizeof(long)); + #if MP_INT_MAX == INT_MAX + return mp_clz((unsigned)x); + #elif MP_INT_MAX == LONG_MAX + return mp_clzl((unsigned long)x); + #elif MP_INT_MAX == LLONG_MAX + return mp_clzll((unsigned long long)x); + #else + #error Unexpected MP_INT_MAX value + #endif + #endif +} - // ugly, but should compile to single intrinsic unless O0 is set - if (mp_check(sizeof(mp_int_t) == sizeof(long))) { - return mp_clzl((unsigned long)x); +// Overflow-checked operations + +// Integer overflow builtins were added to GCC 5, but __has_builtin only in GCC 10 +// +// Note that the builtins has a defined result when overflow occurs, whereas the custom +// functions below don't update the result if an overflow would occur (to avoid UB). +#define MP_GCC_HAS_BUILTIN_OVERFLOW (__GNUC__ >= 5) + +#if MICROPY_USE_GCC_MUL_OVERFLOW_INTRINSIC + +#define mp_mul_ull_overflow __builtin_umulll_overflow +#define mp_mul_ll_overflow __builtin_smulll_overflow +static inline bool mp_mul_mp_int_t_overflow(mp_int_t x, mp_int_t y, mp_int_t *res) { + // __builtin_mul_overflow is a type-generic function, this inline ensures the argument + // types are checked to match mp_int_t. + return __builtin_mul_overflow(x, y, res); +} + +#else + +bool mp_mul_ll_overflow(long long int x, long long int y, long long int *res); +bool mp_mul_mp_int_t_overflow(mp_int_t x, mp_int_t y, mp_int_t *res); +static inline bool mp_mul_ull_overflow(unsigned long long int x, unsigned long long int y, unsigned long long int *res) { + if (y > 0 && x > (ULLONG_MAX / y)) { + return true; // overflow + } + *res = x * y; + return false; +} + +#endif + +#if __has_builtin(__builtin_saddll_overflow) || MP_GCC_HAS_BUILTIN_OVERFLOW +#define mp_add_ll_overflow __builtin_saddll_overflow +#else +static inline bool mp_add_ll_overflow(long long int lhs, long long int rhs, long long int *res) { + bool overflow; + + if (rhs > 0) { + overflow = (lhs > LLONG_MAX - rhs); } else { - return mp_clzll((unsigned long long)x); + overflow = (lhs < LLONG_MIN - rhs); } - #endif + + if (!overflow) { + *res = lhs + rhs; + } + + return overflow; +} +#endif + +#if __has_builtin(__builtin_ssubll_overflow) || MP_GCC_HAS_BUILTIN_OVERFLOW +#define mp_sub_ll_overflow __builtin_ssubll_overflow +#else +static inline bool mp_sub_ll_overflow(long long int lhs, long long int rhs, long long int *res) { + bool overflow; + + if (rhs > 0) { + overflow = (lhs < LLONG_MIN + rhs); + } else { + overflow = (lhs > LLONG_MAX + rhs); + } + + if (!overflow) { + *res = lhs - rhs; + } + + return overflow; } +#endif + + +// Helper macros for detecting if sanitizers are enabled +// +// Use sparingly, not for masking issues reported by sanitizers! +// +// Can be detected automatically in Clang and gcc>=14, need to be +// set manually otherwise. +#ifndef MP_UBSAN +#define MP_UBSAN __has_feature(undefined_behavior_sanitizer) +#endif + +#ifndef MP_ASAN +#define MP_ASAN __has_feature(address_sanitizer) +#endif + +#ifndef MP_SANITIZER_BUILD +#define MP_SANITIZER_BUILD (MP_UBSAN || MP_ASAN) +#endif #endif // MICROPY_INCLUDED_PY_MISC_H diff --git a/py/mkrules.cmake b/py/mkrules.cmake index 4374b8b4da3cb..e3d769cc59b5c 100644 --- a/py/mkrules.cmake +++ b/py/mkrules.cmake @@ -14,6 +14,9 @@ set(MICROPY_MODULEDEFS "${MICROPY_GENHDR_DIR}/moduledefs.h") set(MICROPY_ROOT_POINTERS_SPLIT "${MICROPY_GENHDR_DIR}/root_pointers.split") set(MICROPY_ROOT_POINTERS_COLLECTED "${MICROPY_GENHDR_DIR}/root_pointers.collected") set(MICROPY_ROOT_POINTERS "${MICROPY_GENHDR_DIR}/root_pointers.h") +set(MICROPY_COMPRESSED_SPLIT "${MICROPY_GENHDR_DIR}/compressed.split") +set(MICROPY_COMPRESSED_COLLECTED "${MICROPY_GENHDR_DIR}/compressed.collected") +set(MICROPY_COMPRESSED_DATA "${MICROPY_GENHDR_DIR}/compressed.data.h") if(NOT MICROPY_PREVIEW_VERSION_2) set(MICROPY_PREVIEW_VERSION_2 0) @@ -32,6 +35,13 @@ if(MICROPY_BOARD) ) endif() +# Need to do this before extracting MICROPY_CPP_DEF below. +if(MICROPY_ROM_TEXT_COMPRESSION) + target_compile_definitions(${MICROPY_TARGET} PUBLIC + MICROPY_ROM_TEXT_COMPRESSION=\(1\) + ) +endif() + # Need to do this before extracting MICROPY_CPP_DEF below. Rest of frozen # manifest handling is at the end of this file. if(MICROPY_FROZEN_MANIFEST) @@ -84,6 +94,18 @@ target_sources(${MICROPY_TARGET} PRIVATE ${MICROPY_ROOT_POINTERS} ) +if(MICROPY_ROM_TEXT_COMPRESSION) + target_sources(${MICROPY_TARGET} PRIVATE + ${MICROPY_COMPRESSED_DATA} + ) +endif() + +# Ensure genhdr directory is removed on clean + +set_property(TARGET ${MICROPY_TARGET} APPEND PROPERTY ADDITIONAL_CLEAN_FILES + "${MICROPY_GENHDR_DIR}" +) + # Command to force the build of another command # Generate mpversion.h @@ -197,11 +219,41 @@ add_custom_command( DEPENDS ${MICROPY_ROOT_POINTERS_COLLECTED} ${MICROPY_PY_DIR}/make_root_pointers.py ) +# Generate compressed.data.h + +add_custom_command( + OUTPUT ${MICROPY_COMPRESSED_SPLIT} + COMMAND ${Python3_EXECUTABLE} ${MICROPY_PY_DIR}/makeqstrdefs.py split compress ${MICROPY_QSTRDEFS_LAST} ${MICROPY_GENHDR_DIR}/compress _ + COMMAND touch ${MICROPY_COMPRESSED_SPLIT} + DEPENDS ${MICROPY_QSTRDEFS_LAST} + VERBATIM + COMMAND_EXPAND_LISTS +) + +add_custom_command( + OUTPUT ${MICROPY_COMPRESSED_COLLECTED} + COMMAND ${Python3_EXECUTABLE} ${MICROPY_PY_DIR}/makeqstrdefs.py cat compress _ ${MICROPY_GENHDR_DIR}/compress ${MICROPY_COMPRESSED_COLLECTED} + BYPRODUCTS "${MICROPY_COMPRESSED_COLLECTED}.hash" + DEPENDS ${MICROPY_COMPRESSED_SPLIT} + VERBATIM + COMMAND_EXPAND_LISTS +) + +add_custom_command( + OUTPUT ${MICROPY_COMPRESSED_DATA} + COMMAND ${Python3_EXECUTABLE} ${MICROPY_PY_DIR}/makecompresseddata.py ${MICROPY_COMPRESSED_COLLECTED} > ${MICROPY_COMPRESSED_DATA} + DEPENDS ${MICROPY_COMPRESSED_COLLECTED} ${MICROPY_PY_DIR}/makecompresseddata.py +) + # Build frozen code if enabled if(MICROPY_FROZEN_MANIFEST) set(MICROPY_FROZEN_CONTENT "${CMAKE_BINARY_DIR}/frozen_content.c") + set_property(TARGET ${MICROPY_TARGET} APPEND PROPERTY ADDITIONAL_CLEAN_FILES + "${CMAKE_BINARY_DIR}/frozen_mpy" + ) + target_sources(${MICROPY_TARGET} PRIVATE ${MICROPY_FROZEN_CONTENT} ) diff --git a/py/mkrules.mk b/py/mkrules.mk index f364297f0f209..03988996c71ce 100644 --- a/py/mkrules.mk +++ b/py/mkrules.mk @@ -11,6 +11,7 @@ ifeq ($(MICROPY_PREVIEW_VERSION_2),1) CFLAGS += -DMICROPY_PREVIEW_VERSION_2=1 endif +# CIRCUITPY-CHANGE: point to CircuitPython documentation HELP_BUILD_ERROR ?= "See \033[1;31mhttps://learn.adafruit.com/building-circuitpython; Adafruit Discord \#circuitpython-dev\033[0m" HELP_MPY_LIB_SUBMODULE ?= "\033[1;31mError: micropython-lib submodule is not initialized.\033[0m Run 'make submodules'" @@ -112,6 +113,14 @@ $(BUILD)/%.pp: %.c $(STEPECHO) "PreProcess $<" $(Q)$(CPP) $(CFLAGS) -Wp,-C,-dD,-dI -o $@ $< +.PHONY: $(BUILD)/%.sz +$(BUILD)/%.sz: $(BUILD)/%.o + $(Q)$(SIZE) $< + +# Special case for compiling auto-generated source files. +$(BUILD)/%.o: $(BUILD)/%.c + $(call compile_c) + # The following rule uses | to create an order only prerequisite. Order only # prerequisites only get built if they don't exist. They don't cause timestamp # checking to be performed. @@ -179,7 +188,7 @@ $(HEADER_BUILD)/compressed.collected: $(HEADER_BUILD)/compressed.split # will be created if they don't exist. OBJ_DIRS = $(sort $(dir $(OBJ))) $(OBJ): | $(OBJ_DIRS) -// CIRCUITPY-CHANGE: use $(Q) +# CIRCUITPY-CHANGE: use $(Q) $(OBJ_DIRS): $(Q)$(MKDIR) -p $@ @@ -260,7 +269,7 @@ submodules: $(ECHO) "Updating submodules: $(GIT_SUBMODULES)" ifneq ($(GIT_SUBMODULES),) $(Q)cd $(TOP) && git submodule sync $(GIT_SUBMODULES) - $(Q)cd $(TOP) && git submodule update --init --filter=blob:none $(GIT_SUBMODULES) || \ + $(Q)cd $(TOP) && git submodule update --init --filter=blob:none $(GIT_SUBMODULES) 2>/dev/null || \ git submodule update --init $(GIT_SUBMODULES) endif .PHONY: submodules diff --git a/py/modio.c b/py/modio.c index d3e563dbcf447..9aeb42d30aa13 100644 --- a/py/modio.c +++ b/py/modio.c @@ -169,12 +169,13 @@ static mp_obj_t bufwriter_flush(mp_obj_t self_in) { int err; mp_uint_t out_sz = mp_stream_write_exactly(self->stream, self->buf, self->len, &err); (void)out_sz; - // TODO: try to recover from a case of non-blocking stream, e.g. move - // remaining chunk to the beginning of buffer. - assert(out_sz == self->len); - self->len = 0; if (err != 0) { mp_raise_OSError(err); + } else { + // TODO: try to recover from a case of non-blocking stream, e.g. move + // remaining chunk to the beginning of buffer. + assert(out_sz == self->len); + self->len = 0; } } diff --git a/py/modmath.c b/py/modmath.c index 2b41bbcd7d15e..2a9ed1b7f2291 100644 --- a/py/modmath.c +++ b/py/modmath.c @@ -37,7 +37,7 @@ #define MP_PI_4 MICROPY_FLOAT_CONST(0.78539816339744830962) #define MP_3_PI_4 MICROPY_FLOAT_CONST(2.35619449019234492885) -static NORETURN void math_error(void) { +static MP_NORETURN void math_error(void) { mp_raise_ValueError(MP_ERROR_TEXT("math domain error")); } @@ -99,12 +99,16 @@ mp_float_t MICROPY_FLOAT_C_FUN(log2)(mp_float_t x) { MATH_FUN_1(sqrt, sqrt) // pow(x, y): returns x to the power of y #if MICROPY_PY_MATH_POW_FIX_NAN -mp_float_t pow_func(mp_float_t x, mp_float_t y) { +mp_float_t MICROPY_FLOAT_C_FUN(pow_func)(mp_float_t x, mp_float_t y) { // pow(base, 0) returns 1 for any base, even when base is NaN // pow(+1, exponent) returns 1 for any exponent, even when exponent is NaN if (x == MICROPY_FLOAT_CONST(1.0) || y == MICROPY_FLOAT_CONST(0.0)) { return MICROPY_FLOAT_CONST(1.0); } + // pow(base, NaN) returns NaN for any other value of base + if (isnan(y)) { + return y; + } return MICROPY_FLOAT_C_FUN(pow)(x, y); } MATH_FUN_2(pow, pow_func) @@ -161,6 +165,11 @@ MATH_FUN_2(atan2, atan2) MATH_FUN_1_TO_INT(ceil, ceil) // copysign(x, y) static mp_float_t MICROPY_FLOAT_C_FUN(copysign_func)(mp_float_t x, mp_float_t y) { + #if MICROPY_PY_MATH_COPYSIGN_FIX_NAN + if (isnan(y)) { + y = 0.0; + } + #endif return MICROPY_FLOAT_C_FUN(copysign)(x, y); } MATH_FUN_2(copysign, copysign_func) diff --git a/py/modmicropython.c b/py/modmicropython.c index ff25af8ff7eec..67bb32626c3fa 100644 --- a/py/modmicropython.c +++ b/py/modmicropython.c @@ -100,7 +100,7 @@ static mp_obj_t mp_micropython_qstr_info(size_t n_args, const mp_obj_t *args) { size_t n_pool, n_qstr, n_str_data_bytes, n_total_bytes; qstr_pool_info(&n_pool, &n_qstr, &n_str_data_bytes, &n_total_bytes); mp_printf(&mp_plat_print, "qstr pool: n_pool=%u, n_qstr=%u, n_str_data_bytes=%u, n_total_bytes=%u\n", - n_pool, n_qstr, n_str_data_bytes, n_total_bytes); + (uint)n_pool, (uint)n_qstr, (uint)n_str_data_bytes, (uint)n_total_bytes); if (n_args == 1) { // arg given means dump qstr data qstr_dump_data(); diff --git a/py/modsys.c b/py/modsys.c index 2adbc0b7bd66f..bbae93d0c3f78 100644 --- a/py/modsys.c +++ b/py/modsys.c @@ -95,8 +95,16 @@ static const MP_DEFINE_STR_OBJ(mp_sys_implementation_machine_obj, MICROPY_BANNER MP_ROM_PTR(&mp_sys_implementation_machine_obj) #if MICROPY_PERSISTENT_CODE_LOAD +// Note: Adding architecture flags to _mpy will break if the flags information +// takes up more bits than what is available in a small-int value. +#if MICROPY_EMIT_RV32 +#include "py/asmrv32.h" +#define MPY_FILE_ARCH_FLAGS (MICROPY_RV32_EXTENSIONS << 16) +#else +#define MPY_FILE_ARCH_FLAGS (0) +#endif #define SYS_IMPLEMENTATION_ELEMS__MPY \ - , MP_ROM_INT(MPY_FILE_HEADER_INT) + , MP_ROM_INT(MPY_FILE_HEADER_INT | MPY_FILE_ARCH_FLAGS) #else #define SYS_IMPLEMENTATION_ELEMS__MPY #endif @@ -113,6 +121,18 @@ static const MP_DEFINE_STR_OBJ(mp_sys_implementation__build_obj, MICROPY_BOARD_B #define SYS_IMPLEMENTATION_ELEMS__BUILD #endif +#if MICROPY_PY_THREAD +#if MICROPY_PY_THREAD_GIL +#define SYS_IMPLEMENTATION_ELEMS__THREAD \ + , MP_ROM_QSTR(MP_QSTR_GIL) +#else +#define SYS_IMPLEMENTATION_ELEMS__THREAD \ + , MP_ROM_QSTR(MP_QSTR_unsafe) +#endif +#else +#define SYS_IMPLEMENTATION_ELEMS__THREAD +#endif + #if MICROPY_PREVIEW_VERSION_2 #define SYS_IMPLEMENTATION_ELEMS__V2 \ , MP_ROM_TRUE @@ -130,6 +150,9 @@ static const qstr impl_fields[] = { #if defined(MICROPY_BOARD_BUILD_NAME) MP_QSTR__build, #endif + #if MICROPY_PY_THREAD + MP_QSTR__thread, + #endif #if MICROPY_PREVIEW_VERSION_2 MP_QSTR__v2, #endif @@ -137,20 +160,21 @@ static const qstr impl_fields[] = { static MP_DEFINE_ATTRTUPLE( mp_sys_implementation_obj, impl_fields, - 3 + MICROPY_PERSISTENT_CODE_LOAD + MICROPY_BOARD_BUILD + MICROPY_PREVIEW_VERSION_2, + 3 + MICROPY_PERSISTENT_CODE_LOAD + MICROPY_BOARD_BUILD + MICROPY_PY_THREAD + MICROPY_PREVIEW_VERSION_2, SYS_IMPLEMENTATION_ELEMS_BASE SYS_IMPLEMENTATION_ELEMS__MPY SYS_IMPLEMENTATION_ELEMS__BUILD + SYS_IMPLEMENTATION_ELEMS__THREAD SYS_IMPLEMENTATION_ELEMS__V2 ); #else static const mp_rom_obj_tuple_t mp_sys_implementation_obj = { {&mp_type_tuple}, 3 + MICROPY_PERSISTENT_CODE_LOAD, - // Do not include SYS_IMPLEMENTATION_ELEMS__BUILD or SYS_IMPLEMENTATION_ELEMS__V2 - // because SYS_IMPLEMENTATION_ELEMS__MPY may be empty if + // Do not include SYS_IMPLEMENTATION_ELEMS__BUILD, SYS_IMPLEMENTATION_ELEMS__THREAD + // or SYS_IMPLEMENTATION_ELEMS__V2 because SYS_IMPLEMENTATION_ELEMS__MPY may be empty if // MICROPY_PERSISTENT_CODE_LOAD is disabled, which means they'll share - // the same index. Cannot query _build or _v2 if MICROPY_PY_ATTRTUPLE is + // the same index. Cannot query _build, _thread or _v2 if MICROPY_PY_ATTRTUPLE is // disabled. { SYS_IMPLEMENTATION_ELEMS_BASE diff --git a/py/mpconfig.h b/py/mpconfig.h index 5ff568e64b00a..69a338d746a2f 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -26,6 +26,14 @@ #ifndef MICROPY_INCLUDED_PY_MPCONFIG_H #define MICROPY_INCLUDED_PY_MPCONFIG_H +#include + +#if defined(__cplusplus) // Required on at least one compiler to get ULLONG_MAX +#include +#else +#include +#endif + // CIRCUITPY-CHANGE // Is this a CircuitPython build? #ifndef CIRCUITPY @@ -40,7 +48,7 @@ // as well as a fallback to generate MICROPY_GIT_TAG if the git repo or tags // are unavailable. #define MICROPY_VERSION_MAJOR 1 -#define MICROPY_VERSION_MINOR 25 +#define MICROPY_VERSION_MINOR 27 #define MICROPY_VERSION_MICRO 0 #define MICROPY_VERSION_PRERELEASE 0 @@ -173,6 +181,78 @@ #define MICROPY_OBJ_IMMEDIATE_OBJS (MICROPY_OBJ_REPR != MICROPY_OBJ_REPR_D) #endif +// Definition of the `mp_int_t` and `mp_uint_t` types and associated macros. +// Normally, it suffices for the platform to do nothing: A type as wide +// as a pointer is chosen, unless nanboxing (REPR_D) is selected, in +// which case a 64-bit type is chosen to match the assumed size of +// double-precision floats. +// +// In the case of exceptions, the port, board, or variant must define +// MP_INT_TYPE as MP_INT_TYPE_OTHER and provide all the typedefs and +// defines. +#define MP_INT_TYPE_INTPTR (0) +#define MP_INT_TYPE_INT64 (1) +#define MP_INT_TYPE_OTHER (2) + +#if !defined(MP_INT_TYPE) +#if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_D +#define MP_INT_TYPE (MP_INT_TYPE_INT64) +#else +#define MP_INT_TYPE (MP_INT_TYPE_INTPTR) +#endif +#endif + +#if MP_INT_TYPE == MP_INT_TYPE_INTPTR +typedef intptr_t mp_int_t; +typedef uintptr_t mp_uint_t; +#define MP_INT_MAX INTPTR_MAX +#define MP_INT_MIN INTPTR_MIN +#define MP_UINT_MAX INTPTR_UMAX +#elif MP_INT_TYPE == MP_INT_TYPE_INT64 +typedef int64_t mp_int_t; +typedef uint64_t mp_uint_t; +#define MP_INT_MAX INT64_MAX +#define MP_INT_MIN INT64_MIN +#define MP_UINT_MAX INT64_UMAX +#endif + +// mp_printf format support for mp_int_t. In the unusual case that MP_INT_MAX doesn't +// match any of the standard C types (int/long/long long), provide all 3 +// macros. Otherwise, rely on these automatic definitions. +#if !defined(INT_FMT) +#if MP_INT_MAX == INT_MAX +#define INT_FMT "%d" +#define UINT_FMT "%u" +#define HEX_FMT "%x" +#elif MP_INT_MAX == LONG_MAX +#define INT_FMT "%ld" +#define UINT_FMT "%lu" +#define HEX_FMT "%lx" +#elif MP_INT_MAX == LLONG_MAX +#define INT_FMT "%lld" +#define UINT_FMT "%llu" +#define HEX_FMT "%llx" +#else +#error Unexpected MP_INT_MAX value +#endif +#endif + +// mp_printf format support for size_t. In the unusual case that SIZE_MAX doesn't +// match any of the standard C types (int/long/long long), provide a +// macro. Otherwise, rely on these automatic definitions. +#if !defined(SIZE_FMT) +#if SIZE_MAX == UINT_MAX +#define SIZE_FMT "%u" +#elif SIZE_MAX == ULONG_MAX +#define SIZE_FMT "%lu" +#elif SIZE_MAX == ULLONG_MAX +#define SIZE_FMT "%llu" +#else +#error Unexpected SIZE_MAX value +#endif +#endif + + /*****************************************************************************/ /* Memory allocation policy */ @@ -418,6 +498,11 @@ #define MICROPY_EMIT_INLINE_XTENSA (0) #endif +// Whether to support uncommon Xtensa inline assembler opcodes +#ifndef MICROPY_EMIT_INLINE_XTENSA_UNCOMMON_OPCODES +#define MICROPY_EMIT_INLINE_XTENSA_UNCOMMON_OPCODES (0) +#endif + // Whether to emit Xtensa-Windowed native code #ifndef MICROPY_EMIT_XTENSAWIN #define MICROPY_EMIT_XTENSAWIN (0) @@ -428,9 +513,9 @@ #define MICROPY_EMIT_RV32 (0) #endif -// CIRCUITPY-CHANGE: make sure MICROPY_EMIT_NATIVE_DEBUG is defined -#ifndef MICROPY_EMIT_NATIVE_DEBUG -#define MICROPY_EMIT_NATIVE_DEBUG (0) +// Whether to emit RISC-V RV32 Zba opcodes in native code +#ifndef MICROPY_EMIT_RV32_ZBA +#define MICROPY_EMIT_RV32_ZBA (0) #endif // Whether to enable the RISC-V RV32 inline assembler @@ -438,6 +523,11 @@ #define MICROPY_EMIT_INLINE_RV32 (0) #endif +// Whether to enable the human-readable native instructions emitter +#ifndef MICROPY_EMIT_NATIVE_DEBUG +#define MICROPY_EMIT_NATIVE_DEBUG (0) +#endif + // Convenience definition for whether any native emitter is enabled #define MICROPY_EMIT_NATIVE (MICROPY_EMIT_X64 || MICROPY_EMIT_X86 || MICROPY_EMIT_THUMB || MICROPY_EMIT_ARM || MICROPY_EMIT_XTENSA || MICROPY_EMIT_XTENSAWIN || MICROPY_EMIT_RV32 || MICROPY_EMIT_NATIVE_DEBUG) @@ -497,6 +587,13 @@ #define MICROPY_COMP_CONST (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif +// Whether to enable float constant folding like 1.2+3.4 (when MICROPY_COMP_CONST_FOLDING is also enabled) +// and constant optimisation like id = const(1.2) (when MICROPY_COMP_CONST is also enabled) +// and constant lookup like math.inf (when MICROPY_COMP_MODULE_CONST is also enabled) +#ifndef MICROPY_COMP_CONST_FLOAT +#define MICROPY_COMP_CONST_FLOAT (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) +#endif + // Whether to enable optimisation of: a, b = c, d // Costs 124 bytes (Thumb2) #ifndef MICROPY_COMP_DOUBLE_TUPLE_ASSIGN @@ -727,6 +824,13 @@ #define MICROPY_STACK_CHECK_MARGIN (0) #endif +// The size of a separate stack used for hard IRQ handlers, which should be +// checked instead of the main stack when running a hard callback. 0 implies +// there is no separate ISR stack to check. +#ifndef MICROPY_STACK_SIZE_HARD_IRQ +#define MICROPY_STACK_SIZE_HARD_IRQ (0) +#endif + // Whether to have an emergency exception buffer #ifndef MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF #define MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF (0) @@ -896,8 +1000,25 @@ typedef double mp_float_t; #define MICROPY_PY_BUILTINS_COMPLEX (MICROPY_PY_BUILTINS_FLOAT) #endif -#ifndef MICROPY_PY_DOUBLE_TYPECODE -#define MICROPY_PY_DOUBLE_TYPECODE (MICROPY_PY_BUILTINS_FLOAT) +// Float to string conversion implementations +// +// Note that the EXACT method is only available if the compiler supports +// floating points larger than mp_float_t: +// - with MICROPY_FLOAT_IMPL_FLOAT, the compiler needs to support `double` +// - with MICROPY_FLOAT_IMPL_DOUBLE, the compiler needs to support `long double` +// +#define MICROPY_FLOAT_FORMAT_IMPL_BASIC (0) // smallest code, but inexact +#define MICROPY_FLOAT_FORMAT_IMPL_APPROX (1) // slightly bigger, almost perfect +#define MICROPY_FLOAT_FORMAT_IMPL_EXACT (2) // bigger code, and 100% exact repr + +#ifndef MICROPY_FLOAT_FORMAT_IMPL +#if MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_FLOAT +#define MICROPY_FLOAT_FORMAT_IMPL (MICROPY_FLOAT_FORMAT_IMPL_APPROX) +#elif defined(__SIZEOF_LONG_DOUBLE__) && __SIZEOF_LONG_DOUBLE__ > __SIZEOF_DOUBLE__ +#define MICROPY_FLOAT_FORMAT_IMPL (MICROPY_FLOAT_FORMAT_IMPL_EXACT) +#else +#define MICROPY_FLOAT_FORMAT_IMPL (MICROPY_FLOAT_FORMAT_IMPL_APPROX) +#endif #endif // Whether to use the native _Float16 for 16-bit float support @@ -932,6 +1053,64 @@ typedef double mp_float_t; #define MICROPY_FULL_CHECKS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif +// Ports can choose to use timestamps based on 2000-01-01 or 1970-01-01 +// Default is timestamps based on 2000-01-01 +#if !defined(MICROPY_EPOCH_IS_2000) && !defined(MICROPY_EPOCH_IS_1970) +#define MICROPY_EPOCH_IS_2000 (1) +#define MICROPY_EPOCH_IS_1970 (0) +#elif !defined(MICROPY_EPOCH_IS_1970) +#define MICROPY_EPOCH_IS_1970 (1 - (MICROPY_EPOCH_IS_2000)) +#elif !defined(MICROPY_EPOCH_IS_2000) +#define MICROPY_EPOCH_IS_2000 (1 - (MICROPY_EPOCH_IS_1970)) +#endif + +// To maintain reasonable compatibility with CPython on embedded systems, +// and avoid breaking anytime soon, time functions are defined to work +// at least between 1970 and 2099 (included) on any machine. +// +// Specific ports can enable extended date support +// - after 2099 using MICROPY_TIME_SUPPORT_Y2100_AND_BEYOND +// - before 1970 using MICROPY_TIME_SUPPORT_Y1969_AND_BEFORE +// The largest possible range is year 1600 to year 3000 +// +// By default, extended date support is only enabled for machines using 64 bit pointers, +// but it can be enabled by specific ports +#ifndef MICROPY_TIME_SUPPORT_Y1969_AND_BEFORE +#if MP_SSIZE_MAX > 2147483647 +#define MICROPY_TIME_SUPPORT_Y1969_AND_BEFORE (1) +#else +#define MICROPY_TIME_SUPPORT_Y1969_AND_BEFORE (0) +#endif +#endif + +// When support for dates <1970 is enabled, supporting >=2100 does not cost anything +#ifndef MICROPY_TIME_SUPPORT_Y2100_AND_BEYOND +#define MICROPY_TIME_SUPPORT_Y2100_AND_BEYOND (MICROPY_TIME_SUPPORT_Y1969_AND_BEFORE) +#endif + +// The type to be used to represent platform-specific timestamps depends on the choices above +#define MICROPY_TIMESTAMP_IMPL_LONG_LONG (0) +#define MICROPY_TIMESTAMP_IMPL_UINT (1) +#define MICROPY_TIMESTAMP_IMPL_TIME_T (2) + +#ifndef MICROPY_TIMESTAMP_IMPL +#if MICROPY_TIME_SUPPORT_Y2100_AND_BEYOND || MICROPY_TIME_SUPPORT_Y1969_AND_BEFORE || MICROPY_EPOCH_IS_2000 +#define MICROPY_TIMESTAMP_IMPL (MICROPY_TIMESTAMP_IMPL_LONG_LONG) +#else +#define MICROPY_TIMESTAMP_IMPL (MICROPY_TIMESTAMP_IMPL_UINT) +#endif +#endif + +// `mp_timestamp_t` is the type that should be used by the port +// to represent timestamps, and is referenced to the platform epoch +#if MICROPY_TIMESTAMP_IMPL == MICROPY_TIMESTAMP_IMPL_LONG_LONG +typedef long long mp_timestamp_t; +#elif MICROPY_TIMESTAMP_IMPL == MICROPY_TIMESTAMP_IMPL_UINT +typedef mp_uint_t mp_timestamp_t; +#elif MICROPY_TIMESTAMP_IMPL == MICROPY_TIMESTAMP_IMPL_TIME_T +typedef time_t mp_timestamp_t; +#endif + // Whether POSIX-semantics non-blocking streams are supported #ifndef MICROPY_STREAMS_NON_BLOCK #define MICROPY_STREAMS_NON_BLOCK (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) @@ -943,6 +1122,16 @@ typedef double mp_float_t; #define MICROPY_STREAMS_POSIX_API (0) #endif +// Whether to process __all__ when importing all public symbols from a module. +#ifndef MICROPY_MODULE___ALL__ +#define MICROPY_MODULE___ALL__ (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_BASIC_FEATURES) +#endif + +// Whether to set __file__ on imported modules. +#ifndef MICROPY_MODULE___FILE__ +#define MICROPY_MODULE___FILE__ (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) +#endif + // Whether modules can use MP_REGISTER_MODULE_DELEGATION() to delegate failed // attribute lookups to a custom handler function. #ifndef MICROPY_MODULE_ATTR_DELEGATION @@ -1030,6 +1219,16 @@ typedef double mp_float_t; #define MICROPY_ENABLE_VM_ABORT (0) #endif +// Whether to handle abort behavior in pyexec code +#ifndef MICROPY_PYEXEC_ENABLE_VM_ABORT +#define MICROPY_PYEXEC_ENABLE_VM_ABORT (0) +#endif + +// Whether to set exit codes according to the exit reason (keyboard interrupt, crash, normal exit, ...) +#ifndef MICROPY_PYEXEC_ENABLE_EXIT_CODE_HANDLING +#define MICROPY_PYEXEC_ENABLE_EXIT_CODE_HANDLING (0) +#endif + // Support for internal scheduler #ifndef MICROPY_ENABLE_SCHEDULER #define MICROPY_ENABLE_SCHEDULER (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) @@ -1065,6 +1264,11 @@ typedef double mp_float_t; #define MICROPY_VFS_POSIX (0) #endif +// Whether to include support for writable POSIX filesystems. +#ifndef MICROPY_VFS_POSIX_WRITABLE +#define MICROPY_VFS_POSIX_WRITABLE (1) +#endif + // Support for VFS FAT component, to mount a FAT filesystem within VFS #ifndef MICROPY_VFS_FAT #define MICROPY_VFS_FAT (0) @@ -1105,7 +1309,15 @@ typedef double mp_float_t; #define MICROPY_PY_FUNCTION_ATTRS_CODE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_FULL_FEATURES) #endif -// Whether to support the descriptors __get__, __set__, __delete__ +// Whether bound_method can just use == (feature disabled), or requires a call to +// mp_obj_equal (feature enabled), to test equality of the self and meth entities. +// This is only needed if objects and functions can be identical without being the +// same thing, eg when using an object proxy. +#ifndef MICROPY_PY_BOUND_METHOD_FULL_EQUALITY_CHECK +#define MICROPY_PY_BOUND_METHOD_FULL_EQUALITY_CHECK (0) +#endif + +// Whether to support the descriptors __get__, __set__, __delete__, __set_name__ // This costs some code size and makes load/store/delete of instance // attributes slower for the classes that use this feature #ifndef MICROPY_PY_DESCRIPTORS @@ -1362,11 +1574,6 @@ typedef double mp_float_t; #define MICROPY_PY_BUILTINS_HELP_MODULES (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif -// Whether to set __file__ for imported modules -#ifndef MICROPY_PY___FILE__ -#define MICROPY_PY___FILE__ (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) -#endif - // Whether to provide mem-info related functions in micropython module #ifndef MICROPY_PY_MICROPYTHON_MEM_INFO #define MICROPY_PY_MICROPYTHON_MEM_INFO (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) @@ -1482,6 +1689,7 @@ typedef double mp_float_t; #endif // Whether to provide fix for pow(1, NaN) and pow(NaN, 0), which both should be 1 not NaN. +// Also fixes pow(base, NaN) to return NaN for other values of base. #ifndef MICROPY_PY_MATH_POW_FIX_NAN #define MICROPY_PY_MATH_POW_FIX_NAN (0) #endif @@ -1518,7 +1726,7 @@ typedef double mp_float_t; // Whether to provide "io.IOBase" class to support user streams #ifndef MICROPY_PY_IO_IOBASE -#define MICROPY_PY_IO_IOBASE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +#define MICROPY_PY_IO_IOBASE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif // Whether to provide "io.BytesIO" class @@ -1536,15 +1744,16 @@ typedef double mp_float_t; #define MICROPY_PY_STRUCT (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif -// CIRCUITPY-CHANGE -// Whether to provide non-standard typecodes in "struct" module -#ifndef MICROPY_NONSTANDARD_TYPECODES -#define MICROPY_NONSTANDARD_TYPECODES (1) +// Whether struct module provides unsafe and non-standard typecodes O, P, S. +// These typecodes are not in CPython and can cause crashes by accessing arbitrary +// memory. +#ifndef MICROPY_PY_STRUCT_UNSAFE_TYPECODES +#define MICROPY_PY_STRUCT_UNSAFE_TYPECODES (1) #endif // Whether to provide "sys" module #ifndef MICROPY_PY_SYS -#define MICROPY_PY_SYS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) +#define MICROPY_PY_SYS (1) #endif // Whether to initialise "sys.path" and "sys.argv" to their defaults in mp_init() @@ -1800,11 +2009,11 @@ typedef double mp_float_t; #endif #ifndef MICROPY_PY_HASHLIB_MD5 -#define MICROPY_PY_HASHLIB_MD5 (0) +#define MICROPY_PY_HASHLIB_MD5 (MICROPY_PY_SSL) #endif #ifndef MICROPY_PY_HASHLIB_SHA1 -#define MICROPY_PY_HASHLIB_SHA1 (0) +#define MICROPY_PY_HASHLIB_SHA1 (MICROPY_PY_SSL) #endif #ifndef MICROPY_PY_HASHLIB_SHA256 @@ -1812,7 +2021,7 @@ typedef double mp_float_t; #endif #ifndef MICROPY_PY_CRYPTOLIB -#define MICROPY_PY_CRYPTOLIB (0) +#define MICROPY_PY_CRYPTOLIB (MICROPY_PY_SSL) #endif // Depends on MICROPY_PY_CRYPTOLIB @@ -1935,6 +2144,11 @@ typedef double mp_float_t; #define MICROPY_PY_SSL_MBEDTLS_NEED_ACTIVE_CONTEXT (MICROPY_PY_SSL_ECDSA_SIGN_ALT) #endif +// Whether to support DTLS protocol (non-CPython feature) +#ifndef MICROPY_PY_SSL_DTLS +#define MICROPY_PY_SSL_DTLS (MICROPY_SSL_MBEDTLS && MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +#endif + // Whether to provide the "vfs" module #ifndef MICROPY_PY_VFS #define MICROPY_PY_VFS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES && MICROPY_VFS) @@ -2027,7 +2241,7 @@ typedef double mp_float_t; /*****************************************************************************/ /* Miscellaneous settings */ -// All uPy objects in ROM must be aligned on at least a 4 byte boundary +// All MicroPython objects in ROM must be aligned on at least a 4 byte boundary // so that the small-int/qstr/pointer distinction can be made. For machines // that don't do this (eg 16-bit CPU), define the following macro to something // like __attribute__((aligned(4))). @@ -2166,25 +2380,13 @@ typedef double mp_float_t; #define MP_SSIZE_MAX SSIZE_MAX #endif -// printf format spec to use for mp_int_t and friends -#ifndef INT_FMT -#if defined(__LP64__) -// Archs where mp_int_t == long, long != int -#define UINT_FMT "%lu" -#define INT_FMT "%ld" -#elif defined(_WIN64) -#define UINT_FMT "%llu" -#define INT_FMT "%lld" -#else -// Archs where mp_int_t == int -#define UINT_FMT "%u" -#define INT_FMT "%d" +// Modifier for function which doesn't return +#ifndef MP_NORETURN +#define MP_NORETURN __attribute__((noreturn)) #endif -#endif // INT_FMT -// Modifier for function which doesn't return -#ifndef NORETURN -#define NORETURN __attribute__((noreturn)) +#if !MICROPY_PREVIEW_VERSION_2 +#define NORETURN MP_NORETURN #endif // Modifier for weak functions @@ -2285,4 +2487,23 @@ typedef double mp_float_t; #define MP_WARN_CAT(x) (NULL) #endif +// If true, use __builtin_mul_overflow (a gcc intrinsic supported by clang) for +// overflow checking when multiplying two small ints. Otherwise, use a portable +// algorithm. +// +// Most MCUs have a 32x32->64 bit multiply instruction, in which case the +// intrinsic is likely to be faster and generate smaller code. The main exception is +// cortex-m0 with __ARM_ARCH_ISA_THUMB == 1. +// +// The intrinsic is in GCC starting with version 5. +#ifndef MICROPY_USE_GCC_MUL_OVERFLOW_INTRINSIC +#if defined(__ARM_ARCH_ISA_THUMB) && (__GNUC__ >= 5) +#define MICROPY_USE_GCC_MUL_OVERFLOW_INTRINSIC (__ARM_ARCH_ISA_THUMB >= 2) +#elif (__GNUC__ >= 5) +#define MICROPY_USE_GCC_MUL_OVERFLOW_INTRINSIC (1) +#else +#define MICROPY_USE_GCC_MUL_OVERFLOW_INTRINSIC (0) +#endif +#endif + #endif // MICROPY_INCLUDED_PY_MPCONFIG_H diff --git a/py/mphal.h b/py/mphal.h index 95289ac856cb2..3907641be838a 100644 --- a/py/mphal.h +++ b/py/mphal.h @@ -27,6 +27,7 @@ #define MICROPY_INCLUDED_PY_MPHAL_H #include +#include #include "py/mpconfig.h" #ifdef MICROPY_MPHALPORT_H diff --git a/py/mpprint.c b/py/mpprint.c index e4e25f5a82e43..62e80d1e8e85c 100644 --- a/py/mpprint.c +++ b/py/mpprint.c @@ -40,8 +40,20 @@ #include "py/formatfloat.h" #endif -static const char pad_spaces[] = " "; -static const char pad_zeroes[] = "0000000000000000"; +static const char pad_spaces[16] = {' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '}; +#define pad_spaces_size (sizeof(pad_spaces)) +static const char pad_common[23] = {'0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '_', '0', '0', '0', ',', '0', '0'}; +// The contents of pad_common is arranged to provide the following padding +// strings with minimal flash size: +// 0000000000000000 <- pad_zeroes +// 0000_000 <- pad_zeroes_underscore (offset: 12, size 5) +// 000,00 <- pad_zeroes_comma (offset: 17, size 4) +#define pad_zeroes (pad_common + 0) +#define pad_zeroes_size (16) +#define pad_zeroes_underscore (pad_common + 12) +#define pad_zeroes_underscore_size (5) +#define pad_zeroes_comma (pad_common + 17) +#define pad_zeroes_comma_size (4) static void plat_print_strn(void *env, const char *str, size_t len) { (void)env; @@ -58,20 +70,37 @@ int mp_print_str(const mp_print_t *print, const char *str) { return len; } -int mp_print_strn(const mp_print_t *print, const char *str, size_t len, int flags, char fill, int width) { +int mp_print_strn(const mp_print_t *print, const char *str, size_t len, unsigned int flags, char fill, int width) { int left_pad = 0; int right_pad = 0; int pad = width - len; int pad_size; int total_chars_printed = 0; const char *pad_chars; + char grouping = flags >> PF_FLAG_SEP_POS; if (!fill || fill == ' ') { pad_chars = pad_spaces; - pad_size = sizeof(pad_spaces) - 1; - } else if (fill == '0') { + pad_size = pad_spaces_size; + } else if (fill == '0' && !grouping) { pad_chars = pad_zeroes; - pad_size = sizeof(pad_zeroes) - 1; + pad_size = pad_zeroes_size; + } else if (fill == '0') { + if (grouping == '_') { + pad_chars = pad_zeroes_underscore; + pad_size = pad_zeroes_underscore_size; + } else { + pad_chars = pad_zeroes_comma; + pad_size = pad_zeroes_comma_size; + } + // The result will never start with a grouping character. An extra leading zero is added. + // width is dead after this so we can use it in calculation + if (width % pad_size == 0) { + pad++; + width++; + } + // position the grouping character correctly within the pad repetition + pad_chars += pad_size - 1 - width % pad_size; } else { // Other pad characters are fairly unusual, so we'll take the hit // and output them 1 at a time. @@ -201,7 +230,7 @@ static int mp_print_int(const mp_print_t *print, mp_uint_t x, int sgn, int base, return len; } -int mp_print_mp_int(const mp_print_t *print, mp_obj_t x, int base, int base_char, int flags, char fill, int width, int prec) { +int mp_print_mp_int(const mp_print_t *print, mp_obj_t x, unsigned int base, int base_char, int flags, char fill, int width, int prec) { // These are the only values for "base" that are required to be supported by this // function, since Python only allows the user to format integers in these bases. // If needed this function could be generalised to handle other values. @@ -248,10 +277,7 @@ int mp_print_mp_int(const mp_print_t *print, mp_obj_t x, int base, int base_char int prefix_len = prefix - prefix_buf; prefix = prefix_buf; - char comma = '\0'; - if (flags & PF_FLAG_SHOW_COMMA) { - comma = ','; - } + char comma = flags >> PF_FLAG_SEP_POS; // The size of this buffer is rather arbitrary. If it's not large // enough, a dynamic one will be allocated. @@ -340,8 +366,8 @@ int mp_print_mp_int(const mp_print_t *print, mp_obj_t x, int base, int base_char } #if MICROPY_PY_BUILTINS_FLOAT -int mp_print_float(const mp_print_t *print, mp_float_t f, char fmt, int flags, char fill, int width, int prec) { - char buf[32]; +int mp_print_float(const mp_print_t *print, mp_float_t f, char fmt, unsigned int flags, char fill, int width, int prec) { + char buf[36]; char sign = '\0'; int chrs = 0; @@ -352,11 +378,17 @@ int mp_print_float(const mp_print_t *print, mp_float_t f, char fmt, int flags, c sign = ' '; } - int len = mp_format_float(f, buf, sizeof(buf), fmt, prec, sign); + int len = mp_format_float(f, buf, sizeof(buf) - 3, fmt, prec, sign); char *s = buf; - if ((flags & PF_FLAG_ADD_PERCENT) && (size_t)(len + 1) < sizeof(buf)) { + if ((flags & PF_FLAG_ALWAYS_DECIMAL) && strchr(buf, '.') == NULL && strchr(buf, 'e') == NULL && strchr(buf, 'n') == NULL) { + buf[len++] = '.'; + buf[len++] = '0'; + buf[len] = '\0'; + } + + if (flags & PF_FLAG_ADD_PERCENT) { buf[len++] = '%'; buf[len] = '\0'; } @@ -376,14 +408,6 @@ int mp_print_float(const mp_print_t *print, mp_float_t f, char fmt, int flags, c } #endif -// CIRCUITPY-CHANGE -static int print_str_common(const mp_print_t *print, const char *str, int prec, size_t len, int flags, int fill, int width) { - if (prec >= 0 && (size_t)prec < len) { - len = prec; - } - return mp_print_strn(print, str, len, flags, fill, width); -} - int mp_printf(const mp_print_t *print, const char *fmt, ...) { va_list ap; va_start(ap, fmt); @@ -424,8 +448,6 @@ int mp_vprintf(const mp_print_t *print, const char *fmt, va_list args) { flags |= PF_FLAG_SHOW_SIGN; } else if (*fmt == ' ') { flags |= PF_FLAG_SPACE_SIGN; - } else if (*fmt == '!') { - flags |= PF_FLAG_NO_TRAILZ; } else if (*fmt == '0') { flags |= PF_FLAG_PAD_AFTER_SIGN; fill = '0'; @@ -459,16 +481,36 @@ int mp_vprintf(const mp_print_t *print, const char *fmt, va_list args) { } } - // parse long specifiers (only for LP64 model where they make a difference) - #ifndef __LP64__ - const + // parse long and long long specifiers (only where they make a difference) + #if (MP_INT_MAX > INT_MAX && MP_INT_MAX == LONG_MAX) || defined(MICROPY_UNIX_COVERAGE) + #define SUPPORT_L_FORMAT (1) + #else + #define SUPPORT_L_FORMAT (0) #endif + #if SUPPORT_L_FORMAT bool long_arg = false; + #endif + + #if (MP_INT_MAX > LONG_MAX) || defined(MICROPY_UNIX_COVERAGE) + #define SUPPORT_LL_FORMAT (1) + #else + #define SUPPORT_LL_FORMAT (0) + #endif + #if SUPPORT_LL_FORMAT + bool long_long_arg = false; + #endif + if (*fmt == 'l') { ++fmt; - #ifdef __LP64__ + #if SUPPORT_L_FORMAT long_arg = true; #endif + #if SUPPORT_LL_FORMAT + if (*fmt == 'l') { + ++fmt; + long_long_arg = true; + } + #endif } if (*fmt == '\0') { @@ -492,27 +534,19 @@ int mp_vprintf(const mp_print_t *print, const char *fmt, va_list args) { qstr qst = va_arg(args, qstr); size_t len; const char *str = (const char *)qstr_data(qst, &len); - // CIRCUITPY-CHANGE - chrs += print_str_common(print, str, prec, len, flags, fill, width); - break; - } - // CIRCUITPY-CHANGE: new code to print compressed strings - case 'S': { - mp_rom_error_text_t arg = va_arg(args, mp_rom_error_text_t); - size_t len_with_nul = decompress_length(arg); - size_t len = len_with_nul - 1; - char str[len_with_nul]; - decompress(arg, str); - chrs += print_str_common(print, str, prec, len, flags, fill, width); + if (prec >= 0 && (size_t)prec < len) { + len = prec; + } + chrs += mp_print_strn(print, str, len, flags, fill, width); break; } case 's': { const char *str = va_arg(args, const char *); #ifndef NDEBUG // With debugging enabled, catch printing of null string pointers - if (str == NULL) { - // CIRCUITPY-CHANGE - str = "(null)"; + if (prec != 0 && str == NULL) { + chrs += mp_print_strn(print, "(null)", 6, flags, fill, width); + break; } #endif size_t len = strlen(str); @@ -522,41 +556,58 @@ int mp_vprintf(const mp_print_t *print, const char *fmt, va_list args) { chrs += mp_print_strn(print, str, len, flags, fill, width); break; } - case 'd': { - mp_int_t val; - if (long_arg) { - val = va_arg(args, long int); - } else { - val = va_arg(args, int); - } - chrs += mp_print_int(print, val, 1, 10, 'a', flags, fill, width); - break; - } + case 'd': + case 'p': + case 'P': case 'u': case 'x': case 'X': { - int base = 16 - ((*fmt + 1) & 6); // maps char u/x/X to base 10/16/16 - char fmt_c = (*fmt & 0xf0) - 'P' + 'A'; // maps char u/x/X to char a/a/A + char fmt_chr = *fmt; mp_uint_t val; - if (long_arg) { - val = va_arg(args, unsigned long int); + if (fmt_chr == 'p' || fmt_chr == 'P') { + val = va_arg(args, uintptr_t); + } + #if SUPPORT_LL_FORMAT + else if (long_long_arg) { + val = va_arg(args, unsigned long long); + } + #endif + #if SUPPORT_L_FORMAT + else if (long_arg) { + if (sizeof(long) != sizeof(mp_uint_t) && fmt_chr == 'd') { + val = va_arg(args, long); + } else { + val = va_arg(args, unsigned long); + } + } + #endif + else { + if (sizeof(int) != sizeof(mp_uint_t) && fmt_chr == 'd') { + val = va_arg(args, int); + } else { + val = va_arg(args, unsigned); + } + } + int base; + // Map format char x/p/X/P to a/a/A/A for hex letters. + // It doesn't matter what d/u map to. + char fmt_c = (fmt_chr & 0xf0) - 'P' + 'A'; + if (fmt_chr == 'd' || fmt_chr == 'u') { + base = 10; } else { - val = va_arg(args, unsigned int); + base = 16; + } + // CIRCUITPY-CHANGE: include "0x" for 'p' and 'P'. + if (fmt_chr == 'p' || fmt_chr == 'P') { + #if SUPPORT_INT_BASE_PREFIX + chrs += mp_print_int(print, va_arg(args, unsigned long int), 0, 16, 'a', flags | PF_FLAG_SHOW_PREFIX, fill, width); + #else + chrs += mp_print_strn(print, "0x", 2, flags, fill, width); + #endif } - chrs += mp_print_int(print, val, 0, base, fmt_c, flags, fill, width); + chrs += mp_print_int(print, val, fmt_chr == 'd', base, fmt_c, flags, fill, width); break; } - case 'p': - case 'P': // don't bother to handle upcase for 'P' - // Use unsigned long int to work on both ILP32 and LP64 systems - // CIRCUITPY-CHANGE: print 0x prefix - #if SUPPORT_INT_BASE_PREFIX - chrs += mp_print_int(print, va_arg(args, unsigned long int), 0, 16, 'a', flags | PF_FLAG_SHOW_PREFIX, fill, width); - #else - print->print_strn(print->data, "0x", 2); - chrs += mp_print_int(print, va_arg(args, unsigned long int), 0, 16, 'a', flags, fill, width) + 2; - #endif - break; #if MICROPY_PY_BUILTINS_FLOAT case 'e': case 'E': @@ -573,18 +624,21 @@ int mp_vprintf(const mp_print_t *print, const char *fmt, va_list args) { break; } #endif - // Because 'l' is eaten above, another 'l' means %ll. We need to support - // this length specifier for OBJ_REPR_D (64-bit NaN boxing). - // TODO Either enable this unconditionally, or provide a specific config var. - #if (MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_D) || defined(_WIN64) - case 'l': { - unsigned long long int arg_value = va_arg(args, unsigned long long int); - ++fmt; - assert(*fmt == 'u' || *fmt == 'd' || !"unsupported fmt char"); - chrs += mp_print_int(print, arg_value, *fmt == 'd', 10, 'a', flags, fill, width); + + // CIRCUITPY-CHANGE: new format code to print compressed strings + case 'S': { + mp_rom_error_text_t arg = va_arg(args, mp_rom_error_text_t); + size_t len_with_nul = decompress_length(arg); + size_t len = len_with_nul - 1; + char str[len_with_nul]; + decompress(arg, str); + if (prec >= 0 && (size_t)prec < len) { + len = prec; + } + chrs += mp_print_strn(print, str, len, flags, fill, width); break; } - #endif + default: // if it's not %% then it's an unsupported format character assert(*fmt == '%' || !"unsupported fmt char"); diff --git a/py/mpprint.h b/py/mpprint.h index e883cc27047f1..614c61ea41537 100644 --- a/py/mpprint.h +++ b/py/mpprint.h @@ -31,13 +31,13 @@ #define PF_FLAG_LEFT_ADJUST (0x001) #define PF_FLAG_SHOW_SIGN (0x002) #define PF_FLAG_SPACE_SIGN (0x004) -#define PF_FLAG_NO_TRAILZ (0x008) -#define PF_FLAG_SHOW_PREFIX (0x010) -#define PF_FLAG_SHOW_COMMA (0x020) -#define PF_FLAG_PAD_AFTER_SIGN (0x040) -#define PF_FLAG_CENTER_ADJUST (0x080) -#define PF_FLAG_ADD_PERCENT (0x100) -#define PF_FLAG_SHOW_OCTAL_LETTER (0x200) +#define PF_FLAG_SHOW_PREFIX (0x008) +#define PF_FLAG_PAD_AFTER_SIGN (0x010) +#define PF_FLAG_CENTER_ADJUST (0x020) +#define PF_FLAG_ADD_PERCENT (0x040) +#define PF_FLAG_SHOW_OCTAL_LETTER (0x080) +#define PF_FLAG_ALWAYS_DECIMAL (0x100) +#define PF_FLAG_SEP_POS (9) // must be above all the above PF_FLAGs #if MICROPY_PY_IO && MICROPY_PY_SYS_STDFILES #define MP_PYTHON_PRINTER &mp_sys_stdout_print @@ -69,9 +69,9 @@ extern const mp_print_t mp_sys_stdout_print; #endif int mp_print_str(const mp_print_t *print, const char *str); -int mp_print_strn(const mp_print_t *print, const char *str, size_t len, int flags, char fill, int width); +int mp_print_strn(const mp_print_t *print, const char *str, size_t len, unsigned int flags, char fill, int width); #if MICROPY_PY_BUILTINS_FLOAT -int mp_print_float(const mp_print_t *print, mp_float_t f, char fmt, int flags, char fill, int width, int prec); +int mp_print_float(const mp_print_t *print, mp_float_t f, char fmt, unsigned int flags, char fill, int width, int prec); #endif int mp_printf(const mp_print_t *print, const char *fmt, ...); diff --git a/py/mpstate.h b/py/mpstate.h index e5e5f8d9fa3d1..7934e843f0651 100644 --- a/py/mpstate.h +++ b/py/mpstate.h @@ -71,6 +71,12 @@ enum { // This structure contains dynamic configuration for the compiler. #if MICROPY_DYNAMIC_COMPILER typedef struct mp_dynamic_compiler_t { + // This is used to let mpy-cross pass options to the emitter chosen with + // `native_arch`. The main use case for the time being is to give the + // RV32 emitter extended information about which extensions can be + // optionally used, in order to generate code that's better suited for the + // hardware platform the code will run on. + void *backend_options; uint8_t small_int_bits; // must be <= host small_int_bits uint8_t native_arch; uint8_t nlr_buf_num_regs; diff --git a/py/mpz.c b/py/mpz.c index 7d8bc03ca8610..74fd175463600 100644 --- a/py/mpz.c +++ b/py/mpz.c @@ -1541,7 +1541,8 @@ mp_int_t mpz_hash(const mpz_t *z) { mp_uint_t val = 0; mpz_dig_t *d = z->dig + z->len; - while (d-- > z->dig) { + while (d > z->dig) { + d--; val = (val << DIG_SIZE) | *d; } @@ -1556,11 +1557,12 @@ bool mpz_as_int_checked(const mpz_t *i, mp_int_t *value) { mp_uint_t val = 0; mpz_dig_t *d = i->dig + i->len; - while (d-- > i->dig) { + while (d > i->dig) { if (val > (~(MP_OBJ_WORD_MSBIT_HIGH) >> DIG_SIZE)) { // will overflow return false; } + d--; val = (val << DIG_SIZE) | *d; } @@ -1581,11 +1583,12 @@ bool mpz_as_uint_checked(const mpz_t *i, mp_uint_t *value) { mp_uint_t val = 0; mpz_dig_t *d = i->dig + i->len; - while (d-- > i->dig) { + while (d > i->dig) { if (val > (~(MP_OBJ_WORD_MSBIT_HIGH) >> (DIG_SIZE - 1))) { // will overflow return false; } + d--; val = (val << DIG_SIZE) | *d; } @@ -1646,7 +1649,8 @@ mp_float_t mpz_as_float(const mpz_t *i) { mp_float_t val = 0; mpz_dig_t *d = i->dig + i->len; - while (d-- > i->dig) { + while (d > i->dig) { + d--; val = val * DIG_BASE + *d; } @@ -1676,6 +1680,8 @@ size_t mpz_as_str_inpl(const mpz_t *i, unsigned int base, const char *prefix, ch size_t ilen = i->len; + int n_comma = (base == 10) ? 3 : 4; + char *s = str; if (ilen == 0) { if (prefix) { @@ -1721,7 +1727,7 @@ size_t mpz_as_str_inpl(const mpz_t *i, unsigned int base, const char *prefix, ch break; } } - if (!done && comma && (s - last_comma) == 3) { + if (!done && comma && (s - last_comma) == n_comma) { *s++ = comma; last_comma = s; } diff --git a/py/nativeglue.h b/py/nativeglue.h index 00ed9f3f4fcb7..01825ac60edb4 100644 --- a/py/nativeglue.h +++ b/py/nativeglue.h @@ -143,7 +143,7 @@ typedef struct _mp_fun_table_t { int (*printf_)(const mp_print_t *print, const char *fmt, ...); int (*vprintf_)(const mp_print_t *print, const char *fmt, va_list args); #if defined(__GNUC__) - NORETURN // Only certain compilers support no-return attributes in function pointer declarations + MP_NORETURN // Only certain compilers support no-return attributes in function pointer declarations #endif // CIRCUITPY-CHANGE: raise_msg_str instead of raise_msg void (*raise_msg_str)(const mp_obj_type_t *exc_type, const char *msg); diff --git a/py/nlr.c b/py/nlr.c index 7ab0c0955a294..de2a38ceff3e9 100644 --- a/py/nlr.c +++ b/py/nlr.c @@ -81,7 +81,7 @@ void nlr_call_jump_callbacks(nlr_buf_t *nlr) { } #if MICROPY_ENABLE_VM_ABORT -NORETURN void nlr_jump_abort(void) { +MP_NORETURN void nlr_jump_abort(void) { MP_STATE_THREAD(nlr_top) = MP_STATE_VM(nlr_abort); nlr_jump(NULL); } diff --git a/py/nlr.h b/py/nlr.h index 340627b7aa1f0..446e78c7ebbf7 100644 --- a/py/nlr.h +++ b/py/nlr.h @@ -233,18 +233,18 @@ unsigned int nlr_push(nlr_buf_t *); unsigned int nlr_push_tail(nlr_buf_t *top); void nlr_pop(void); -NORETURN void nlr_jump(void *val); +MP_NORETURN void nlr_jump(void *val); #if MICROPY_ENABLE_VM_ABORT #define nlr_set_abort(buf) MP_STATE_VM(nlr_abort) = buf #define nlr_get_abort() MP_STATE_VM(nlr_abort) -NORETURN void nlr_jump_abort(void); +MP_NORETURN void nlr_jump_abort(void); #endif // This must be implemented by a port. It's called by nlr_jump // if no nlr buf has been pushed. It must not return, but rather // should bail out with a fatal error. -NORETURN void nlr_jump_fail(void *val); +MP_NORETURN void nlr_jump_fail(void *val); // use nlr_raise instead of nlr_jump so that debugging is easier #ifndef MICROPY_DEBUG_NLR diff --git a/py/nlraarch64.c b/py/nlraarch64.c index d6d87ebc50db8..3318004b5e037 100644 --- a/py/nlraarch64.c +++ b/py/nlraarch64.c @@ -56,7 +56,7 @@ __asm( #endif ); -NORETURN void nlr_jump(void *val) { +MP_NORETURN void nlr_jump(void *val) { MP_NLR_JUMP_HEAD(val, top) MP_STATIC_ASSERT(offsetof(nlr_buf_t, regs) == 16); // asm assumes it diff --git a/py/nlrmips.c b/py/nlrmips.c index cba52b16a266a..5c55db7e268ec 100644 --- a/py/nlrmips.c +++ b/py/nlrmips.c @@ -57,7 +57,7 @@ __asm( ".end nlr_push \n" ); -NORETURN void nlr_jump(void *val) { +MP_NORETURN void nlr_jump(void *val) { MP_NLR_JUMP_HEAD(val, top) __asm( "move $4, %0 \n" diff --git a/py/nlrpowerpc.c b/py/nlrpowerpc.c index 8a69fe1eeca6b..cf140400e68f5 100644 --- a/py/nlrpowerpc.c +++ b/py/nlrpowerpc.c @@ -78,7 +78,7 @@ unsigned int nlr_push(nlr_buf_t *nlr) { return 0; } -NORETURN void nlr_jump(void *val) { +MP_NORETURN void nlr_jump(void *val) { MP_NLR_JUMP_HEAD(val, top) __asm__ volatile ( @@ -167,7 +167,7 @@ unsigned int nlr_push(nlr_buf_t *nlr) { return 0; } -NORETURN void nlr_jump(void *val) { +MP_NORETURN void nlr_jump(void *val) { MP_NLR_JUMP_HEAD(val, top) __asm__ volatile ( diff --git a/py/nlrrv32.c b/py/nlrrv32.c index 9a12ede400daa..565a8629db5f0 100644 --- a/py/nlrrv32.c +++ b/py/nlrrv32.c @@ -50,7 +50,7 @@ __attribute__((naked)) unsigned int nlr_push(nlr_buf_t *nlr) { ); } -NORETURN void nlr_jump(void *val) { +MP_NORETURN void nlr_jump(void *val) { MP_NLR_JUMP_HEAD(val, top) __asm volatile ( "add x10, x0, %0 \n" // Load nlr_buf address. diff --git a/py/nlrrv64.c b/py/nlrrv64.c index e7ba79797b857..b7d1467b8f6f6 100644 --- a/py/nlrrv64.c +++ b/py/nlrrv64.c @@ -50,7 +50,7 @@ __attribute__((naked)) unsigned int nlr_push(nlr_buf_t *nlr) { ); } -NORETURN void nlr_jump(void *val) { +MP_NORETURN void nlr_jump(void *val) { MP_NLR_JUMP_HEAD(val, top) __asm volatile ( "add x10, x0, %0 \n" // Load nlr_buf address. diff --git a/py/nlrthumb.c b/py/nlrthumb.c index 265052568ffe4..1791d8b49294e 100644 --- a/py/nlrthumb.c +++ b/py/nlrthumb.c @@ -104,7 +104,7 @@ __attribute__((naked)) unsigned int nlr_push(nlr_buf_t *nlr) { #endif } -NORETURN void nlr_jump(void *val) { +MP_NORETURN void nlr_jump(void *val) { MP_NLR_JUMP_HEAD(val, top) __asm volatile ( diff --git a/py/nlrx64.c b/py/nlrx64.c index d1ad91ff7d718..51224729fc90d 100644 --- a/py/nlrx64.c +++ b/py/nlrx64.c @@ -100,7 +100,7 @@ unsigned int nlr_push(nlr_buf_t *nlr) { #endif } -NORETURN void nlr_jump(void *val) { +MP_NORETURN void nlr_jump(void *val) { MP_NLR_JUMP_HEAD(val, top) __asm volatile ( diff --git a/py/nlrx86.c b/py/nlrx86.c index 085e30d2034a1..26bf0dc6ccb85 100644 --- a/py/nlrx86.c +++ b/py/nlrx86.c @@ -78,7 +78,7 @@ unsigned int nlr_push(nlr_buf_t *nlr) { #endif } -NORETURN void nlr_jump(void *val) { +MP_NORETURN void nlr_jump(void *val) { MP_NLR_JUMP_HEAD(val, top) __asm volatile ( diff --git a/py/nlrxtensa.c b/py/nlrxtensa.c index ff7af6edeef98..2d1bf35e381a3 100644 --- a/py/nlrxtensa.c +++ b/py/nlrxtensa.c @@ -55,7 +55,7 @@ unsigned int nlr_push(nlr_buf_t *nlr) { return 0; // needed to silence compiler warning } -NORETURN void nlr_jump(void *val) { +MP_NORETURN void nlr_jump(void *val) { MP_NLR_JUMP_HEAD(val, top) __asm volatile ( diff --git a/py/obj.c b/py/obj.c index 29ae76557f8b5..ba3e6a9a9964d 100644 --- a/py/obj.c +++ b/py/obj.c @@ -157,7 +157,7 @@ void mp_obj_print_helper(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t if (MP_OBJ_TYPE_HAS_SLOT(type, print)) { MP_OBJ_TYPE_GET_SLOT(type, print)((mp_print_t *)print, o_in, kind); } else { - mp_printf(print, "<%q>", type->name); + mp_printf(print, "<%q>", (qstr)type->name); } } @@ -417,6 +417,36 @@ mp_int_t mp_obj_get_int(mp_const_obj_t arg) { return val; } +#if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_NONE +mp_uint_t mp_obj_get_uint(mp_const_obj_t arg) { + if (!mp_obj_is_exact_type(arg, &mp_type_int)) { + mp_obj_t as_int = mp_unary_op(MP_UNARY_OP_INT_MAYBE, (mp_obj_t)arg); + if (as_int == MP_OBJ_NULL) { + mp_raise_TypeError_int_conversion(arg); + } + arg = as_int; + } + return mp_obj_int_get_uint_checked(arg); +} + +long long mp_obj_get_ll(mp_const_obj_t arg) { + if (!mp_obj_is_exact_type(arg, &mp_type_int)) { + mp_obj_t as_int = mp_unary_op(MP_UNARY_OP_INT_MAYBE, (mp_obj_t)arg); + if (as_int == MP_OBJ_NULL) { + mp_raise_TypeError_int_conversion(arg); + } + arg = as_int; + } + if (mp_obj_is_small_int(arg)) { + return MP_OBJ_SMALL_INT_VALUE(arg); + } else { + long long res; + mp_obj_int_to_bytes_impl((mp_obj_t)arg, MP_ENDIANNESS_BIG, sizeof(res), (byte *)&res); + return res; + } +} +#endif + mp_int_t mp_obj_get_int_truncated(mp_const_obj_t arg) { if (mp_obj_is_int(arg)) { return mp_obj_int_get_truncated(arg); diff --git a/py/obj.h b/py/obj.h index 38db17cba449b..eb7143bd43f62 100644 --- a/py/obj.h +++ b/py/obj.h @@ -184,13 +184,15 @@ static inline bool mp_obj_is_small_int(mp_const_obj_t o) { #define MP_OBJ_NEW_SMALL_INT(small_int) ((mp_obj_t)((((mp_uint_t)(small_int)) << 1) | 1)) #if MICROPY_PY_BUILTINS_FLOAT -#define MP_OBJ_NEW_CONST_FLOAT(f) MP_ROM_PTR((mp_obj_t)((((((uint64_t)f) & ~3) | 2) + 0x80800000) & 0xffffffff)) +#include +// note: MP_OBJ_NEW_CONST_FLOAT should be a MP_ROM_PTR but that macro isn't available yet +#define MP_OBJ_NEW_CONST_FLOAT(f) ((mp_obj_t)((((((uint64_t)f) & ~3) | 2) + 0x80800000) & 0xffffffff)) #define mp_const_float_e MP_OBJ_NEW_CONST_FLOAT(0x402df854) #define mp_const_float_pi MP_OBJ_NEW_CONST_FLOAT(0x40490fdb) +#define mp_const_float_nan MP_OBJ_NEW_CONST_FLOAT(0x7fc00000) #if MICROPY_PY_MATH_CONSTANTS #define mp_const_float_tau MP_OBJ_NEW_CONST_FLOAT(0x40c90fdb) #define mp_const_float_inf MP_OBJ_NEW_CONST_FLOAT(0x7f800000) -#define mp_const_float_nan MP_OBJ_NEW_CONST_FLOAT(0xffc00000) #endif static inline bool mp_obj_is_float(mp_const_obj_t o) { @@ -204,9 +206,17 @@ static inline mp_float_t mp_obj_float_get(mp_const_obj_t o) { mp_float_t f; mp_uint_t u; } num = {.u = ((mp_uint_t)o - 0x80800000u) & ~3u}; + // Rather than always truncating toward zero, which creates a strong + // bias, copy the two previous bits to fill in the two missing bits. + // This appears to be a pretty good heuristic. + num.u |= (num.u >> 2) & 3u; return num.f; } static inline mp_obj_t mp_obj_new_float(mp_float_t f) { + if (isnan(f)) { + // prevent creation of bad nanboxed pointers via array.array or struct + return mp_const_float_nan; + } union { mp_float_t f; mp_uint_t u; @@ -257,8 +267,10 @@ static inline bool mp_obj_is_immediate_obj(mp_const_obj_t o) { #error MICROPY_OBJ_REPR_D requires MICROPY_FLOAT_IMPL_DOUBLE #endif +#include #define mp_const_float_e {((mp_obj_t)((uint64_t)0x4005bf0a8b145769 + 0x8004000000000000))} #define mp_const_float_pi {((mp_obj_t)((uint64_t)0x400921fb54442d18 + 0x8004000000000000))} +#define mp_const_float_nan {((mp_obj_t)((uint64_t)0x7ff8000000000000 + 0x8004000000000000))} #if MICROPY_PY_MATH_CONSTANTS #define mp_const_float_tau {((mp_obj_t)((uint64_t)0x401921fb54442d18 + 0x8004000000000000))} #define mp_const_float_inf {((mp_obj_t)((uint64_t)0x7ff0000000000000 + 0x8004000000000000))} @@ -276,6 +288,13 @@ static inline mp_float_t mp_obj_float_get(mp_const_obj_t o) { return num.f; } static inline mp_obj_t mp_obj_new_float(mp_float_t f) { + if (isnan(f)) { + // prevent creation of bad nanboxed pointers via array.array or struct + struct { + uint64_t r; + } num = mp_const_float_nan; + return num.r; + } union { mp_float_t f; uint64_t r; @@ -371,25 +390,25 @@ typedef struct _mp_rom_obj_t { mp_const_obj_t o; } mp_rom_obj_t; #define MP_DEFINE_CONST_FUN_OBJ_0(obj_name, fun_name) \ const mp_obj_fun_builtin_fixed_t obj_name = \ - {{&mp_type_fun_builtin_0}, .fun._0 = fun_name} + {.base = {.type = &mp_type_fun_builtin_0}, .fun = {._0 = fun_name}} #define MP_DEFINE_CONST_FUN_OBJ_1(obj_name, fun_name) \ const mp_obj_fun_builtin_fixed_t obj_name = \ - {{&mp_type_fun_builtin_1}, .fun._1 = fun_name} + {.base = {.type = &mp_type_fun_builtin_1}, .fun = {._1 = fun_name}} #define MP_DEFINE_CONST_FUN_OBJ_2(obj_name, fun_name) \ const mp_obj_fun_builtin_fixed_t obj_name = \ - {{&mp_type_fun_builtin_2}, .fun._2 = fun_name} + {.base = {.type = &mp_type_fun_builtin_2}, .fun = {._2 = fun_name}} #define MP_DEFINE_CONST_FUN_OBJ_3(obj_name, fun_name) \ const mp_obj_fun_builtin_fixed_t obj_name = \ - {{&mp_type_fun_builtin_3}, .fun._3 = fun_name} + {.base = {.type = &mp_type_fun_builtin_3}, .fun = {._3 = fun_name}} #define MP_DEFINE_CONST_FUN_OBJ_VAR(obj_name, n_args_min, fun_name) \ const mp_obj_fun_builtin_var_t obj_name = \ - {{&mp_type_fun_builtin_var}, MP_OBJ_FUN_MAKE_SIG(n_args_min, MP_OBJ_FUN_ARGS_MAX, false), .fun.var = fun_name} + {.base = {.type = &mp_type_fun_builtin_var}, .sig = MP_OBJ_FUN_MAKE_SIG(n_args_min, MP_OBJ_FUN_ARGS_MAX, false), .fun = {.var = fun_name}} #define MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(obj_name, n_args_min, n_args_max, fun_name) \ const mp_obj_fun_builtin_var_t obj_name = \ - {{&mp_type_fun_builtin_var}, MP_OBJ_FUN_MAKE_SIG(n_args_min, n_args_max, false), .fun.var = fun_name} + {.base = {.type = &mp_type_fun_builtin_var}, .sig = MP_OBJ_FUN_MAKE_SIG(n_args_min, n_args_max, false), .fun = {.var = fun_name}} #define MP_DEFINE_CONST_FUN_OBJ_KW(obj_name, n_args_min, fun_name) \ const mp_obj_fun_builtin_var_t obj_name = \ - {{&mp_type_fun_builtin_var}, MP_OBJ_FUN_MAKE_SIG(n_args_min, MP_OBJ_FUN_ARGS_MAX, true), .fun.kw = fun_name} + {.base = {.type = &mp_type_fun_builtin_var}, .sig = MP_OBJ_FUN_MAKE_SIG(n_args_min, MP_OBJ_FUN_ARGS_MAX, true), .fun = {.kw = fun_name}} // CIRCUITPY-CHANGE #define MP_DEFINE_CONST_PROP_GET(obj_name, fun_name) \ @@ -516,9 +535,7 @@ static inline bool mp_map_slot_is_filled(const mp_map_t *map, size_t pos) { void mp_map_init(mp_map_t *map, size_t n); void mp_map_init_fixed_table(mp_map_t *map, size_t n, const mp_obj_t *table); -mp_map_t *mp_map_new(size_t n); void mp_map_deinit(mp_map_t *map); -void mp_map_free(mp_map_t *map); mp_map_elem_t *mp_map_lookup(mp_map_t *map, mp_obj_t index, mp_map_lookup_kind_t lookup_kind); void mp_map_clear(mp_map_t *map); void mp_map_dump(mp_map_t *map); @@ -555,6 +572,10 @@ typedef mp_obj_t (*mp_fun_var_t)(size_t n, const mp_obj_t *); typedef mp_obj_t (*mp_fun_kw_t)(size_t n, const mp_obj_t *, mp_map_t *); // Flags for type behaviour (mp_obj_type_t.flags) +// If MP_TYPE_FLAG_IS_SUBCLASSED is set, then subclasses of this class have been created. +// Mutations to this class that would require updating all subclasses must be rejected. +// If MP_TYPE_FLAG_HAS_SPECIAL_ACCESSORS is set, then attribute lookups involving this +// class need to additionally check for special accessor methods, such as from descriptors. // If MP_TYPE_FLAG_EQ_NOT_REFLEXIVE is clear then __eq__ is reflexive (A==A returns True). // If MP_TYPE_FLAG_EQ_CHECKS_OTHER_TYPE is clear then the type can't be equal to an // instance of any different class that also clears this flag. If this flag is set @@ -572,6 +593,8 @@ typedef mp_obj_t (*mp_fun_kw_t)(size_t n, const mp_obj_t *, mp_map_t *); // If MP_TYPE_FLAG_ITER_IS_STREAM is set then the type implicitly gets a "return self" // getiter, and mp_stream_unbuffered_iter for iternext. // If MP_TYPE_FLAG_INSTANCE_TYPE is set then this is an instance type (i.e. defined in Python). +// If MP_TYPE_FLAG_SUBSCR_ALLOWS_STACK_SLICE is set then the "subscr" slot allows a stack +// allocated slice to be passed in (no references to it will be retained after the call). #define MP_TYPE_FLAG_NONE (0x0000) #define MP_TYPE_FLAG_IS_SUBCLASSED (0x0001) #define MP_TYPE_FLAG_HAS_SPECIAL_ACCESSORS (0x0002) @@ -585,8 +608,9 @@ typedef mp_obj_t (*mp_fun_kw_t)(size_t n, const mp_obj_t *, mp_map_t *); #define MP_TYPE_FLAG_ITER_IS_CUSTOM (0x0100) #define MP_TYPE_FLAG_ITER_IS_STREAM (MP_TYPE_FLAG_ITER_IS_ITERNEXT | MP_TYPE_FLAG_ITER_IS_CUSTOM) #define MP_TYPE_FLAG_INSTANCE_TYPE (0x0200) +#define MP_TYPE_FLAG_SUBSCR_ALLOWS_STACK_SLICE (0x0400) // CIRCUITPY-CHANGE: check for valid types in json dumps -#define MP_TYPE_FLAG_PRINT_JSON (0x0400) +#define MP_TYPE_FLAG_PRINT_JSON (0x0800) typedef enum { PRINT_STR = 0, @@ -827,7 +851,7 @@ typedef struct _mp_obj_full_type_t { #define MP_DEFINE_CONST_OBJ_TYPE_NARGS(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, _25, _26, _27, _28, N, ...) MP_DEFINE_CONST_OBJ_TYPE_NARGS_##N // This macros is used to define a object type in ROM. -// Invoke as MP_DEFINE_CONST_OBJ_TYPE(_typename, _name, _flags, _make_new [, slot, func]*) +// Invoke as MP_DEFINE_CONST_OBJ_TYPE(_typename, _name, _flags, [, slot, func]*) // It uses the number of arguments to select which MP_DEFINE_CONST_OBJ_TYPE_* // macro to use based on the number of arguments. It works by shifting the // numeric values 12, 11, ... 0 by the number of arguments, such that the @@ -847,7 +871,7 @@ extern const mp_obj_type_t mp_type_bytearray; extern const mp_obj_type_t mp_type_memoryview; extern const mp_obj_type_t mp_type_float; extern const mp_obj_type_t mp_type_complex; -// CIRCUITPY-CHANGE +// CIRCUITPY-CHANGE: add traceback support extern const mp_obj_type_t mp_type_traceback; extern const mp_obj_type_t mp_type_tuple; extern const mp_obj_type_t mp_type_list; @@ -909,7 +933,7 @@ extern const mp_obj_type_t mp_type_ImportError; extern const mp_obj_type_t mp_type_IndentationError; extern const mp_obj_type_t mp_type_IndexError; extern const mp_obj_type_t mp_type_KeyboardInterrupt; -// CIRCUITPY-CHANGE +// CIRCUITPY-CHANGE: add ReloadException extern const mp_obj_type_t mp_type_ReloadException; extern const mp_obj_type_t mp_type_KeyError; extern const mp_obj_type_t mp_type_LookupError; @@ -917,9 +941,9 @@ extern const mp_obj_type_t mp_type_MemoryError; extern const mp_obj_type_t mp_type_NameError; extern const mp_obj_type_t mp_type_NotImplementedError; extern const mp_obj_type_t mp_type_OSError; -// CIRCUITPY-CHANGE +// CIRCUITPY-CHANGE: add ConnectionError extern const mp_obj_type_t mp_type_ConnectionError; -// CIRCUITPY-CHANGE +// CIRCUITPY-CHANGE: add BrokenPipeError extern const mp_obj_type_t mp_type_BrokenPipeError; extern const mp_obj_type_t mp_type_OverflowError; extern const mp_obj_type_t mp_type_RuntimeError; @@ -927,6 +951,7 @@ extern const mp_obj_type_t mp_type_StopAsyncIteration; extern const mp_obj_type_t mp_type_StopIteration; extern const mp_obj_type_t mp_type_SyntaxError; extern const mp_obj_type_t mp_type_SystemExit; +// CIRCUITPY-CHANGE: add TimeoutError extern const mp_obj_type_t mp_type_TimeoutError; extern const mp_obj_type_t mp_type_TypeError; extern const mp_obj_type_t mp_type_UnicodeError; @@ -991,11 +1016,11 @@ void *mp_obj_malloc_helper(size_t num_bytes, const mp_obj_type_t *type); // Object allocation macros for allocating objects that have a finaliser. #if MICROPY_ENABLE_FINALISER #define mp_obj_malloc_with_finaliser(struct_type, obj_type) ((struct_type *)mp_obj_malloc_with_finaliser_helper(sizeof(struct_type), obj_type)) -#define mp_obj_malloc_var_with_finaliser(struct_type, var_type, var_num, obj_type) ((struct_type *)mp_obj_malloc_with_finaliser_helper(sizeof(struct_type) + sizeof(var_type) * (var_num), obj_type)) +#define mp_obj_malloc_var_with_finaliser(struct_type, var_field, var_type, var_num, obj_type) ((struct_type *)mp_obj_malloc_with_finaliser_helper(offsetof(struct_type, var_field) + sizeof(var_type) * (var_num), obj_type)) void *mp_obj_malloc_with_finaliser_helper(size_t num_bytes, const mp_obj_type_t *type); #else #define mp_obj_malloc_with_finaliser(struct_type, obj_type) mp_obj_malloc(struct_type, obj_type) -#define mp_obj_malloc_var_with_finaliser(struct_type, var_type, var_num, obj_type) mp_obj_malloc_var(struct_type, var_type, var_num, obj_type) +#define mp_obj_malloc_var_with_finaliser(struct_type, var_field, var_type, var_num, obj_type) mp_obj_malloc_var(struct_type, var_field, var_type, var_num, obj_type) #endif // These macros are derived from more primitive ones and are used to @@ -1032,7 +1057,6 @@ bool mp_obj_is_dict_or_ordereddict(mp_obj_t o); // type check is done on iter method to allow tuple, namedtuple, attrtuple #define mp_obj_is_tuple_compatible(o) (MP_OBJ_TYPE_GET_SLOT_OR_NULL(mp_obj_get_type(o), iter) == mp_obj_tuple_getiter) -mp_obj_t mp_obj_new_type(qstr name, mp_obj_t bases_tuple, mp_obj_t locals_dict); static inline mp_obj_t mp_obj_new_bool(mp_int_t x) { return x ? mp_const_true : mp_const_false; } @@ -1053,10 +1077,10 @@ mp_obj_t mp_obj_new_str_from_utf8_vstr(vstr_t *vstr); // input data must be vali #endif mp_obj_t mp_obj_new_bytes_from_vstr(vstr_t *vstr); mp_obj_t mp_obj_new_bytes(const byte *data, size_t len); -// CIRCUITPY-CHANGE +// CIRCUITPY-CHANGE: new routine mp_obj_t mp_obj_new_bytes_of_zeros(size_t len); mp_obj_t mp_obj_new_bytearray(size_t n, const void *items); -// CIRCUITPY-CHANGE +// CIRCUITPY-CHANGE: new routine mp_obj_t mp_obj_new_bytearray_of_zeros(size_t n); mp_obj_t mp_obj_new_bytearray_by_ref(size_t n, void *items); #if MICROPY_PY_BUILTINS_FLOAT @@ -1092,7 +1116,7 @@ mp_obj_t mp_obj_new_memoryview(byte typecode, size_t nitems, void *items); const mp_obj_type_t *mp_obj_get_type(mp_const_obj_t o_in); const char *mp_obj_get_type_str(mp_const_obj_t o_in); -// CIRCUITPY-CHANGE +// CIRCUITPY-CHANGE: new routine #define mp_obj_get_type_qstr(o_in) (mp_obj_get_type((o_in))->name) bool mp_obj_is_subclass_fast(mp_const_obj_t object, mp_const_obj_t classinfo); // arguments should be type objects mp_obj_t mp_obj_cast_to_native_base(mp_obj_t self_in, mp_const_obj_t native_type); @@ -1100,7 +1124,7 @@ mp_obj_t mp_obj_cast_to_native_base(mp_obj_t self_in, mp_const_obj_t native_type void mp_obj_print_helper(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind); void mp_obj_print(mp_obj_t o, mp_print_kind_t kind); void mp_obj_print_exception(const mp_print_t *print, mp_obj_t exc); -// CIRCUITPY-CHANGE +// CIRCUITPY-CHANGE: new routine void mp_obj_print_exception_with_limit(const mp_print_t *print, mp_obj_t exc, mp_int_t limit); bool mp_obj_is_true(mp_obj_t arg); @@ -1114,6 +1138,8 @@ static inline bool mp_obj_is_integer(mp_const_obj_t o) { } mp_int_t mp_obj_get_int(mp_const_obj_t arg); +mp_uint_t mp_obj_get_uint(mp_const_obj_t arg); +long long mp_obj_get_ll(mp_const_obj_t arg); mp_int_t mp_obj_get_int_truncated(mp_const_obj_t arg); bool mp_obj_get_int_maybe(mp_const_obj_t arg, mp_int_t *value); #if MICROPY_PY_BUILTINS_FLOAT @@ -1163,7 +1189,7 @@ bool mp_obj_exception_match(mp_obj_t exc, mp_const_obj_t exc_type); void mp_obj_exception_clear_traceback(mp_obj_t self_in); void mp_obj_exception_add_traceback(mp_obj_t self_in, qstr file, size_t line, qstr block); void mp_obj_exception_get_traceback(mp_obj_t self_in, size_t *n, size_t **values); -// CIRCUITPY-CHANGE +// CIRCUITPY-CHANGE: new routine mp_obj_t mp_obj_exception_get_traceback_obj(mp_obj_t self_in); mp_obj_t mp_obj_exception_get_value(mp_obj_t self_in); mp_obj_t mp_obj_exception_make_new(const mp_obj_type_t *type_in, size_t n_args, size_t n_kw, const mp_obj_t *args); @@ -1240,7 +1266,7 @@ void mp_obj_tuple_del(mp_obj_t self_in); mp_int_t mp_obj_tuple_hash(mp_obj_t self_in); // list -// CIRCUITPY-CHANGE +// CIRCUITPY-CHANGE: public routine mp_obj_t mp_obj_list_clear(mp_obj_t self_in); mp_obj_t mp_obj_list_append(mp_obj_t self_in, mp_obj_t arg); mp_obj_t mp_obj_list_remove(mp_obj_t self_in, mp_obj_t value); @@ -1336,7 +1362,7 @@ typedef struct _mp_rom_obj_static_class_method_t { } mp_rom_obj_static_class_method_t; // property -// CIRCUITPY-CHANGE +// CIRCUITPY-CHANGE: extra args const mp_obj_t *mp_obj_property_get(mp_obj_t self_in, size_t *n_proxy); // sequence helpers diff --git a/py/objarray.c b/py/objarray.c index 0be1947167d6e..1e259a20ac8cb 100644 --- a/py/objarray.c +++ b/py/objarray.c @@ -126,6 +126,19 @@ static mp_obj_array_t *array_new(char typecode, size_t n) { #endif #if MICROPY_PY_BUILTINS_BYTEARRAY || MICROPY_PY_ARRAY +static void array_extend_impl(mp_obj_array_t *array, mp_obj_t arg, char typecode, size_t len) { + mp_obj_t iterable = mp_getiter(arg, NULL); + mp_obj_t item; + size_t i = 0; + while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) { + if (len == 0) { + array_append(MP_OBJ_FROM_PTR(array), item); + } else { + mp_binary_set_val_array(typecode, array->items, i++, item); + } + } +} + static mp_obj_t array_construct(char typecode, mp_obj_t initializer) { // bytearrays can be raw-initialised from anything with the buffer protocol // other arrays can only be raw-initialised from bytes and bytearray objects @@ -158,18 +171,7 @@ static mp_obj_t array_construct(char typecode, mp_obj_t initializer) { } mp_obj_array_t *array = array_new(typecode, len); - - mp_obj_t iterable = mp_getiter(initializer, NULL); - mp_obj_t item; - size_t i = 0; - while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) { - if (len == 0) { - array_append(MP_OBJ_FROM_PTR(array), item); - } else { - mp_binary_set_val_array(typecode, array->items, i++, item); - } - } - + array_extend_impl(array, initializer, typecode, len); return MP_OBJ_FROM_PTR(array); } #endif @@ -298,7 +300,7 @@ static void memoryview_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { mp_obj_array_t *self = MP_OBJ_TO_PTR(self_in); dest[0] = MP_OBJ_NEW_SMALL_INT(mp_binary_get_size('@', self->typecode & TYPECODE_MASK, NULL)); } - // CIRCUITPY-CHANGE: prevent warning + // CIRCUITPY-CHANGE: add MICROPY_CPYTHON_COMPAT #if MICROPY_PY_BUILTINS_BYTES_HEX || MICROPY_CPYTHON_COMPAT else { // Need to forward to locals dict. @@ -493,39 +495,42 @@ static mp_obj_t array_extend(mp_obj_t self_in, mp_obj_t arg_in) { // allow to extend by anything that has the buffer protocol (extension to CPython) mp_buffer_info_t arg_bufinfo; - // CIRCUITPY-CHANGE: allow appending an iterable - if (mp_get_buffer(arg_in, &arg_bufinfo, MP_BUFFER_READ)) { - size_t sz = mp_binary_get_size('@', self->typecode, NULL); - - // convert byte count to element count - size_t len = arg_bufinfo.len / sz; - - // make sure we have enough room to extend - // TODO: alloc policy; at the moment we go conservative - if (self->free < len) { - self->items = m_renew(byte, self->items, (self->len + self->free) * sz, (self->len + len) * sz); - self->free = 0; - } else { - self->free -= len; - } + if (!mp_get_buffer(arg_in, &arg_bufinfo, MP_BUFFER_READ)) { + array_extend_impl(self, arg_in, 0, 0); + return mp_const_none; + } - // extend - mp_seq_copy((byte *)self->items + self->len * sz, arg_bufinfo.buf, len * sz, byte); - self->len += len; - } else { - // Otherwise argument must be an iterable of items to append - mp_obj_t iterable = mp_getiter(arg_in, NULL); - mp_obj_t item; - while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) { - array_append(self_in, item); + size_t sz = mp_binary_get_size('@', self->typecode, NULL); + + // convert byte count to element count + size_t len = arg_bufinfo.len / sz; + + // make sure we have enough room to extend + // TODO: alloc policy; at the moment we go conservative + if (self->free < len) { + self->items = m_renew(byte, self->items, (self->len + self->free) * sz, (self->len + len) * sz); + self->free = 0; + + if (self_in == arg_in) { + // Get arg_bufinfo again in case self->items has moved + // + // (Note not possible to handle case that arg_in is a memoryview into self) + mp_get_buffer_raise(arg_in, &arg_bufinfo, MP_BUFFER_READ); } + } else { + self->free -= len; } + + // extend + mp_seq_copy((byte *)self->items + self->len * sz, arg_bufinfo.buf, len * sz, byte); + self->len += len; + return mp_const_none; } MP_DEFINE_CONST_FUN_OBJ_2(mp_obj_array_extend_obj, array_extend); #endif -// CIRCUITPY-CHANGE: buffer_finder used belo +// CIRCUITPY-CHANGE: buffer_finder used below #if MICROPY_PY_BUILTINS_BYTEARRAY && MICROPY_CPYTHON_COMPAT static mp_obj_t buffer_finder(size_t n_args, const mp_obj_t *args, int direction, bool is_index) { mp_check_self(mp_obj_is_type(args[0], &mp_type_bytearray)); @@ -564,7 +569,6 @@ static mp_obj_t buffer_finder(size_t n_args, const mp_obj_t *args, int direction } return MP_OBJ_NEW_SMALL_INT(p - (const byte *)haystack_bufinfo.buf); } - // CIRCUITPY-CHANGE: provides find, rfind, index static mp_obj_t buffer_find(size_t n_args, const mp_obj_t *args) { return buffer_finder(n_args, args, 1, false); @@ -667,7 +671,7 @@ static mp_obj_t array_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_obj_t value mp_seq_replace_slice_no_grow(dest_items, o->len, slice.start, slice.stop, src_items + src_offs, src_len, item_sz); // CIRCUITPY-CHANGE - #if MICROPY_NONSTANDARD_TYPECODES + #if MICROPY_PY_STRUCT_UNSAFE_TYPECODES // Clear "freed" elements at the end of list // TODO: This is actually only needed for typecode=='O' mp_seq_clear(dest_items, o->len + len_adj, o->len, item_sz); @@ -803,7 +807,7 @@ MP_DEFINE_CONST_OBJ_TYPE( MP_DEFINE_CONST_OBJ_TYPE( mp_type_bytearray, MP_QSTR_bytearray, - MP_TYPE_FLAG_EQ_CHECKS_OTHER_TYPE | MP_TYPE_FLAG_ITER_IS_GETITER, + MP_TYPE_FLAG_EQ_CHECKS_OTHER_TYPE | MP_TYPE_FLAG_ITER_IS_GETITER | MP_TYPE_FLAG_SUBSCR_ALLOWS_STACK_SLICE, make_new, bytearray_make_new, print, array_print, iter, array_iterator_new, @@ -842,7 +846,7 @@ MP_DEFINE_CONST_DICT(memoryview_locals_dict, memoryview_locals_dict_table); MP_DEFINE_CONST_OBJ_TYPE( mp_type_memoryview, MP_QSTR_memoryview, - MP_TYPE_FLAG_EQ_CHECKS_OTHER_TYPE | MP_TYPE_FLAG_ITER_IS_GETITER, + MP_TYPE_FLAG_EQ_CHECKS_OTHER_TYPE | MP_TYPE_FLAG_ITER_IS_GETITER | MP_TYPE_FLAG_SUBSCR_ALLOWS_STACK_SLICE, make_new, memoryview_make_new, iter, array_iterator_new, unary_op, array_unary_op, diff --git a/py/objboundmeth.c b/py/objboundmeth.c index e3503ff154a65..6df67f7bf9e0c 100644 --- a/py/objboundmeth.c +++ b/py/objboundmeth.c @@ -102,7 +102,11 @@ static mp_obj_t bound_meth_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_ } mp_obj_bound_meth_t *lhs = MP_OBJ_TO_PTR(lhs_in); mp_obj_bound_meth_t *rhs = MP_OBJ_TO_PTR(rhs_in); + #if MICROPY_PY_BOUND_METHOD_FULL_EQUALITY_CHECK + return mp_obj_new_bool(mp_obj_equal(lhs->self, rhs->self) && mp_obj_equal(lhs->meth, rhs->meth)); + #else return mp_obj_new_bool(lhs->self == rhs->self && lhs->meth == rhs->meth); + #endif } #if MICROPY_PY_FUNCTION_ATTRS diff --git a/py/objcell.c b/py/objcell.c index 95966c7917cb7..5c030c7405a80 100644 --- a/py/objcell.c +++ b/py/objcell.c @@ -30,7 +30,7 @@ static void cell_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind) { (void)kind; mp_obj_cell_t *o = MP_OBJ_TO_PTR(o_in); - mp_printf(print, "obj); + mp_printf(print, "obj); if (o->obj == MP_OBJ_NULL) { mp_print_str(print, "(nil)"); } else { diff --git a/py/objcode.c b/py/objcode.c index 9b98a696798d4..09904f10f3683 100644 --- a/py/objcode.c +++ b/py/objcode.c @@ -69,6 +69,7 @@ static mp_obj_tuple_t *code_consts(const mp_module_context_t *context, const mp_ return consts; } +#if !MICROPY_PREVIEW_VERSION_2 static mp_obj_t raw_code_lnotab(const mp_raw_code_t *rc) { // const mp_bytecode_prelude_t *prelude = &rc->prelude; uint start = 0; @@ -106,6 +107,68 @@ static mp_obj_t raw_code_lnotab(const mp_raw_code_t *rc) { m_del(byte, buffer, buffer_size); return o; } +#endif + +static mp_obj_t code_colines_iter(mp_obj_t); +static mp_obj_t code_colines_next(mp_obj_t); +typedef struct _mp_obj_colines_iter_t { + mp_obj_base_t base; + mp_fun_1_t iternext; + const mp_raw_code_t *rc; + mp_uint_t bc; + mp_uint_t source_line; + const byte *ci; +} mp_obj_colines_iter_t; + +static mp_obj_t code_colines_iter(mp_obj_t self_in) { + mp_obj_code_t *self = MP_OBJ_TO_PTR(self_in); + mp_obj_colines_iter_t *iter = mp_obj_malloc(mp_obj_colines_iter_t, &mp_type_polymorph_iter); + iter->iternext = code_colines_next; + iter->rc = self->rc; + iter->bc = 0; + iter->source_line = 1; + iter->ci = self->rc->prelude.line_info; + return MP_OBJ_FROM_PTR(iter); +} +static MP_DEFINE_CONST_FUN_OBJ_1(code_colines_obj, code_colines_iter); + +static mp_obj_t code_colines_next(mp_obj_t iter_in) { + mp_obj_colines_iter_t *iter = MP_OBJ_TO_PTR(iter_in); + const byte *ci_end = iter->rc->prelude.line_info_top; + + mp_uint_t start = iter->bc; + mp_uint_t line_no = iter->source_line; + bool another = true; + + while (another && iter->ci < ci_end) { + another = false; + mp_code_lineinfo_t decoded = mp_bytecode_decode_lineinfo(&iter->ci); + iter->bc += decoded.bc_increment; + iter->source_line += decoded.line_increment; + + if (decoded.bc_increment == 0) { + line_no = iter->source_line; + another = true; + } else if (decoded.line_increment == 0) { + another = true; + } + } + + if (another) { + mp_uint_t prelude_size = (iter->rc->prelude.opcodes - (const byte *)iter->rc->fun_data); + mp_uint_t bc_end = iter->rc->fun_data_len - prelude_size; + if (iter->bc >= bc_end) { + return MP_OBJ_STOP_ITERATION; + } else { + iter->bc = bc_end; + } + } + + mp_uint_t end = iter->bc; + mp_obj_t next[3] = {MP_OBJ_NEW_SMALL_INT(start), MP_OBJ_NEW_SMALL_INT(end), MP_OBJ_NEW_SMALL_INT(line_no)}; + + return mp_obj_new_tuple(MP_ARRAY_SIZE(next), next); +} static void code_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { if (dest[0] != MP_OBJ_NULL) { @@ -137,12 +200,18 @@ static void code_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { case MP_QSTR_co_names: dest[0] = MP_OBJ_FROM_PTR(o->dict_locals); break; + #if !MICROPY_PREVIEW_VERSION_2 case MP_QSTR_co_lnotab: if (!o->lnotab) { o->lnotab = raw_code_lnotab(rc); } dest[0] = o->lnotab; break; + #endif + case MP_QSTR_co_lines: + dest[0] = MP_OBJ_FROM_PTR(&code_colines_obj); + dest[1] = self_in; + break; } } @@ -168,7 +237,9 @@ mp_obj_t mp_obj_new_code(const mp_module_context_t *context, const mp_raw_code_t o->context = context; o->rc = rc; o->dict_locals = mp_locals_get(); // this is a wrong! how to do this properly? + #if !MICROPY_PREVIEW_VERSION_2 o->lnotab = MP_OBJ_NULL; + #endif return MP_OBJ_FROM_PTR(o); } diff --git a/py/objcode.h b/py/objcode.h index 8db9a34b6e1c9..7be15e23f5271 100644 --- a/py/objcode.h +++ b/py/objcode.h @@ -72,15 +72,16 @@ static inline const void *mp_code_get_proto_fun(mp_obj_code_t *self) { #include "py/emitglue.h" -#define MP_CODE_QSTR_MAP(context, idx) (context->constants.qstr_table[idx]) +#define MP_CODE_QSTR_MAP(context, idx) ((qstr)(context->constants.qstr_table[idx])) typedef struct _mp_obj_code_t { - // TODO this was 4 words mp_obj_base_t base; const mp_module_context_t *context; const mp_raw_code_t *rc; mp_obj_dict_t *dict_locals; + #if !MICROPY_PREVIEW_VERSION_2 mp_obj_t lnotab; + #endif } mp_obj_code_t; mp_obj_t mp_obj_new_code(const mp_module_context_t *context, const mp_raw_code_t *rc, bool result_required); diff --git a/py/objcomplex.c b/py/objcomplex.c index d7287efdc13dd..88f4444f09e4d 100644 --- a/py/objcomplex.c +++ b/py/objcomplex.c @@ -49,29 +49,18 @@ typedef struct _mp_obj_complex_t { static void complex_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind) { (void)kind; mp_obj_complex_t *o = MP_OBJ_TO_PTR(o_in); - #if MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_FLOAT - char buf[16]; - #if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_C - const int precision = 6; - #else - const int precision = 7; - #endif - #else - char buf[32]; - const int precision = 16; - #endif - if (o->real == 0) { - mp_format_float(o->imag, buf, sizeof(buf), 'g', precision, '\0'); - mp_printf(print, "%sj", buf); + const char *suffix; + int flags = 0; + if (o->real != 0) { + mp_print_str(print, "("); + mp_print_float(print, o->real, 'g', 0, '\0', -1, MP_FLOAT_REPR_PREC); + flags = PF_FLAG_SHOW_SIGN; + suffix = "j)"; } else { - mp_format_float(o->real, buf, sizeof(buf), 'g', precision, '\0'); - mp_printf(print, "(%s", buf); - if (o->imag >= 0 || isnan(o->imag)) { - mp_print_str(print, "+"); - } - mp_format_float(o->imag, buf, sizeof(buf), 'g', precision, '\0'); - mp_printf(print, "%sj)", buf); + suffix = "j"; } + mp_print_float(print, o->imag, 'g', flags, '\0', -1, MP_FLOAT_REPR_PREC); + mp_print_str(print, suffix); } static mp_obj_t complex_make_new(const mp_obj_type_t *type_in, size_t n_args, size_t n_kw, const mp_obj_t *args) { diff --git a/py/objdict.c b/py/objdict.c index 79a606f09707b..d459d1ae48647 100644 --- a/py/objdict.c +++ b/py/objdict.c @@ -101,7 +101,7 @@ static void dict_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_ #endif } if (MICROPY_PY_COLLECTIONS_ORDEREDDICT && self->base.type != &mp_type_dict && kind != PRINT_JSON) { - mp_printf(print, "%q(", self->base.type->name); + mp_printf(print, "%q(", (qstr)self->base.type->name); } mp_print_str(print, "{"); size_t cur = 0; @@ -616,7 +616,8 @@ static mp_obj_t dict_view_unary_op(mp_unary_op_t op, mp_obj_t o_in) { if (op == MP_UNARY_OP_HASH && o->kind == MP_DICT_VIEW_VALUES) { return MP_OBJ_NEW_SMALL_INT((mp_uint_t)o_in); } - return MP_OBJ_NULL; + // delegate all other ops to dict unary op handler + return dict_unary_op(op, o->dict); } static mp_obj_t dict_view_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_in) { diff --git a/py/objfloat.c b/py/objfloat.c index 3610c2b85862d..ecc0f61a6abc9 100644 --- a/py/objfloat.c +++ b/py/objfloat.c @@ -34,6 +34,11 @@ #if MICROPY_PY_BUILTINS_FLOAT +// Workaround a bug in Windows SDK version 10.0.26100.0, where NAN is no longer constant. +#if defined(_MSC_VER) && !defined(_UCRT_NOISY_NAN) +#define _UCRT_NOISY_NAN +#endif + #include #include "py/formatfloat.h" @@ -51,13 +56,6 @@ #define M_PI (3.14159265358979323846) #endif -// Workaround a bug in recent MSVC where NAN is no longer constant. -// (By redefining back to the previous MSVC definition of NAN) -#if defined(_MSC_VER) && _MSC_VER >= 1942 -#undef NAN -#define NAN (-(float)(((float)(1e+300 * 1e+300)) * 0.0F)) -#endif - typedef struct _mp_obj_float_t { mp_obj_base_t base; mp_float_t value; @@ -116,23 +114,7 @@ mp_int_t mp_float_hash(mp_float_t src) { static void float_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind) { (void)kind; mp_float_t o_val = mp_obj_float_get(o_in); - #if MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_FLOAT - char buf[16]; - #if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_C - const int precision = 6; - #else - const int precision = 7; - #endif - #else - char buf[32]; - const int precision = 16; - #endif - mp_format_float(o_val, buf, sizeof(buf), 'g', precision, '\0'); - mp_print_str(print, buf); - if (strchr(buf, '.') == NULL && strchr(buf, 'e') == NULL && strchr(buf, 'n') == NULL) { - // Python floats always have decimal point (unless inf or nan) - mp_print_str(print, ".0"); - } + mp_print_float(print, o_val, 'g', PF_FLAG_ALWAYS_DECIMAL, '\0', -1, MP_FLOAT_REPR_PREC); } static mp_obj_t float_make_new(const mp_obj_type_t *type_in, size_t n_args, size_t n_kw, const mp_obj_t *args) { @@ -324,6 +306,10 @@ mp_obj_t mp_obj_float_binary_op(mp_binary_op_t op, mp_float_t lhs_val, mp_obj_t lhs_val = MICROPY_FLOAT_CONST(1.0); break; } + if (isnan(rhs_val)) { + lhs_val = rhs_val; + break; + } #endif lhs_val = MICROPY_FLOAT_C_FUN(pow)(lhs_val, rhs_val); break; diff --git a/py/objfun.c b/py/objfun.c index e6a923d59e886..34565cf633632 100644 --- a/py/objfun.c +++ b/py/objfun.c @@ -184,7 +184,8 @@ static mp_obj_t fun_bc_make_new(const mp_obj_type_t *type, size_t n_args, size_t static void fun_bc_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind) { (void)kind; mp_obj_fun_bc_t *o = MP_OBJ_TO_PTR(o_in); - mp_printf(print, "", mp_obj_fun_get_name(o_in), o); + // CIRCUITPY-CHANGE: %p already prints "0x", so don't include it explicitly. + mp_printf(print, "", mp_obj_fun_get_name(o_in), o); } #endif diff --git a/py/objint.c b/py/objint.c index b12f09c9d38a7..cabbd7b8f0e12 100644 --- a/py/objint.c +++ b/py/objint.c @@ -117,9 +117,15 @@ static mp_fp_as_int_class_t mp_classify_fp_as_int(mp_float_t val) { } // 8 * sizeof(uintptr_t) counts the number of bits for a small int // TODO provide a way to configure this properly + #if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_B + if (e <= ((8 * sizeof(uintptr_t) + MP_FLOAT_EXP_BIAS - 4) << MP_FLOAT_EXP_SHIFT_I32)) { + return MP_FP_CLASS_FIT_SMALLINT; + } + #else if (e <= ((8 * sizeof(uintptr_t) + MP_FLOAT_EXP_BIAS - 3) << MP_FLOAT_EXP_SHIFT_I32)) { return MP_FP_CLASS_FIT_SMALLINT; } + #endif #if MICROPY_LONGINT_IMPL == MICROPY_LONGINT_IMPL_LONGLONG if (e <= (((sizeof(long long) * MP_BITS_PER_BYTE) + MP_FLOAT_EXP_BIAS - 2) << MP_FLOAT_EXP_SHIFT_I32)) { return MP_FP_CLASS_FIT_LONGINT; @@ -211,7 +217,7 @@ static const uint8_t log_base2_floor[] = { size_t mp_int_format_size(size_t num_bits, int base, const char *prefix, char comma) { assert(2 <= base && base <= 16); size_t num_digits = num_bits / log_base2_floor[base - 1] + 1; - size_t num_commas = comma ? num_digits / 3 : 0; + size_t num_commas = comma ? (base == 10 ? num_digits / 3 : num_digits / 4): 0; size_t prefix_len = prefix ? strlen(prefix) : 0; return num_digits + num_commas + prefix_len + 2; // +1 for sign, +1 for null byte } @@ -249,10 +255,11 @@ char *mp_obj_int_formatted(char **buf, size_t *buf_size, size_t *fmt_size, mp_co char sign = '\0'; if (num < 0) { - num = -num; + num = -(fmt_uint_t)num; sign = '-'; } + int n_comma = (base == 10) ? 3 : 4; size_t needed_size = mp_int_format_size(sizeof(fmt_int_t) * 8, base, prefix, comma); if (needed_size > *buf_size) { *buf = m_new(char, needed_size); @@ -277,7 +284,7 @@ char *mp_obj_int_formatted(char **buf, size_t *buf_size, size_t *fmt_size, mp_co c += '0'; } *(--b) = c; - if (comma && num != 0 && b > str && (last_comma - b) == 3) { + if (comma && num != 0 && b > str && (last_comma - b) == n_comma) { *(--b) = comma; last_comma = b; } diff --git a/py/objint_longlong.c b/py/objint_longlong.c index 0fad693c7adcd..f24aa0cc18664 100644 --- a/py/objint_longlong.c +++ b/py/objint_longlong.c @@ -43,6 +43,10 @@ const mp_obj_int_t mp_sys_maxsize_obj = {{&mp_type_int}, MP_SSIZE_MAX}; #endif +static void raise_long_long_overflow(void) { + mp_raise_msg(&mp_type_OverflowError, MP_ERROR_TEXT("result overflows long long storage")); +} + // CIRCUITPY-CHANGE: bit_length mp_obj_t mp_obj_int_bit_length_impl(mp_obj_t self_in) { assert(mp_obj_is_type(self_in, &mp_type_int)); @@ -132,7 +136,6 @@ mp_obj_t mp_obj_int_unary_op(mp_unary_op_t op, mp_obj_t o_in) { // small int if the value fits without truncation case MP_UNARY_OP_HASH: return MP_OBJ_NEW_SMALL_INT((mp_int_t)o->val); - case MP_UNARY_OP_POSITIVE: return o_in; case MP_UNARY_OP_NEGATIVE: @@ -159,6 +162,8 @@ mp_obj_t mp_obj_int_unary_op(mp_unary_op_t op, mp_obj_t o_in) { mp_obj_t mp_obj_int_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_in) { long long lhs_val; long long rhs_val; + bool overflow = false; + long long result; if (mp_obj_is_small_int(lhs_in)) { lhs_val = MP_OBJ_SMALL_INT_VALUE(lhs_in); @@ -171,21 +176,41 @@ mp_obj_t mp_obj_int_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_i rhs_val = MP_OBJ_SMALL_INT_VALUE(rhs_in); } else if (mp_obj_is_exact_type(rhs_in, &mp_type_int)) { rhs_val = ((mp_obj_int_t *)rhs_in)->val; + #if MICROPY_PY_BUILTINS_FLOAT + } else if (mp_obj_is_float(rhs_in)) { + return mp_obj_float_binary_op(op, (mp_float_t)lhs_val, rhs_in); + #endif + #if MICROPY_PY_BUILTINS_COMPLEX + } else if (mp_obj_is_type(rhs_in, &mp_type_complex)) { + return mp_obj_complex_binary_op(op, (mp_float_t)lhs_val, 0, rhs_in); + #endif } else { // delegate to generic function to check for extra cases return mp_obj_int_binary_op_extra_cases(op, lhs_in, rhs_in); } + #if MICROPY_PY_BUILTINS_FLOAT + if (op == MP_BINARY_OP_TRUE_DIVIDE || op == MP_BINARY_OP_INPLACE_TRUE_DIVIDE) { + if (rhs_val == 0) { + goto zero_division; + } + return mp_obj_new_float((mp_float_t)lhs_val / (mp_float_t)rhs_val); + } + #endif + switch (op) { case MP_BINARY_OP_ADD: case MP_BINARY_OP_INPLACE_ADD: - return mp_obj_new_int_from_ll(lhs_val + rhs_val); + overflow = mp_add_ll_overflow(lhs_val, rhs_val, &result); + break; case MP_BINARY_OP_SUBTRACT: case MP_BINARY_OP_INPLACE_SUBTRACT: - return mp_obj_new_int_from_ll(lhs_val - rhs_val); + overflow = mp_sub_ll_overflow(lhs_val, rhs_val, &result); + break; case MP_BINARY_OP_MULTIPLY: case MP_BINARY_OP_INPLACE_MULTIPLY: - return mp_obj_new_int_from_ll(lhs_val * rhs_val); + overflow = mp_mul_ll_overflow(lhs_val, rhs_val, &result); + break; case MP_BINARY_OP_FLOOR_DIVIDE: case MP_BINARY_OP_INPLACE_FLOOR_DIVIDE: if (rhs_val == 0) { @@ -211,9 +236,21 @@ mp_obj_t mp_obj_int_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_i case MP_BINARY_OP_LSHIFT: case MP_BINARY_OP_INPLACE_LSHIFT: - return mp_obj_new_int_from_ll(lhs_val << (int)rhs_val); + if (rhs_val < 0) { + // negative shift not allowed + mp_raise_ValueError(MP_ERROR_TEXT("negative shift count")); + } + overflow = rhs_val >= (sizeof(long long) * MP_BITS_PER_BYTE) + || lhs_val > (LLONG_MAX >> rhs_val) + || lhs_val < (LLONG_MIN >> rhs_val); + result = (unsigned long long)lhs_val << rhs_val; + break; case MP_BINARY_OP_RSHIFT: case MP_BINARY_OP_INPLACE_RSHIFT: + if ((int)rhs_val < 0) { + // negative shift not allowed + mp_raise_ValueError(MP_ERROR_TEXT("negative shift count")); + } return mp_obj_new_int_from_ll(lhs_val >> (int)rhs_val); case MP_BINARY_OP_POWER: @@ -225,18 +262,18 @@ mp_obj_t mp_obj_int_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_i mp_raise_ValueError(MP_ERROR_TEXT("negative power with no float support")); #endif } - long long ans = 1; - while (rhs_val > 0) { + result = 1; + while (rhs_val > 0 && !overflow) { if (rhs_val & 1) { - ans *= lhs_val; + overflow = mp_mul_ll_overflow(result, lhs_val, &result); } - if (rhs_val == 1) { + if (rhs_val == 1 || overflow) { break; } rhs_val /= 2; - lhs_val *= lhs_val; + overflow = mp_mul_ll_overflow(lhs_val, lhs_val, &lhs_val); } - return mp_obj_new_int_from_ll(ans); + break; } case MP_BINARY_OP_LESS: @@ -254,8 +291,14 @@ mp_obj_t mp_obj_int_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_i return MP_OBJ_NULL; // op not supported } + if (overflow) { + raise_long_long_overflow(); + } + + return mp_obj_new_int_from_ll(result); + zero_division: - mp_raise_ZeroDivisionError(); + mp_raise_msg(&mp_type_ZeroDivisionError, MP_ERROR_TEXT("divide by zero")); } mp_obj_t mp_obj_new_int(mp_int_t value) { @@ -277,22 +320,12 @@ mp_obj_t mp_obj_new_int_from_ll(long long val) { } mp_obj_t mp_obj_new_int_from_ull(unsigned long long val) { - // TODO raise an exception if the unsigned long long won't fit if (val >> (sizeof(unsigned long long) * 8 - 1) != 0) { - mp_raise_msg(&mp_type_OverflowError, MP_ERROR_TEXT("ulonglong too large")); + raise_long_long_overflow(); } return mp_obj_new_int_from_ll(val); } -mp_obj_t mp_obj_new_int_from_str_len(const char **str, size_t len, bool neg, unsigned int base) { - // TODO this does not honor the given length of the string, but it all cases it should anyway be null terminated - // TODO check overflow - char *endptr; - mp_obj_t result = mp_obj_new_int_from_ll(strtoll(*str, &endptr, base)); - *str = endptr; - return result; -} - mp_int_t mp_obj_int_get_truncated(mp_const_obj_t self_in) { if (mp_obj_is_small_int(self_in)) { return MP_OBJ_SMALL_INT_VALUE(self_in); @@ -303,8 +336,33 @@ mp_int_t mp_obj_int_get_truncated(mp_const_obj_t self_in) { } mp_int_t mp_obj_int_get_checked(mp_const_obj_t self_in) { - // TODO: Check overflow - return mp_obj_int_get_truncated(self_in); + if (mp_obj_is_small_int(self_in)) { + return MP_OBJ_SMALL_INT_VALUE(self_in); + } else { + const mp_obj_int_t *self = self_in; + long long value = self->val; + mp_int_t truncated = (mp_int_t)value; + if ((long long)truncated == value) { + return truncated; + } + } + mp_raise_msg(&mp_type_OverflowError, MP_ERROR_TEXT("overflow converting long int to machine word")); +} + +mp_uint_t mp_obj_int_get_uint_checked(mp_const_obj_t self_in) { + if (mp_obj_is_small_int(self_in)) { + if (MP_OBJ_SMALL_INT_VALUE(self_in) >= 0) { + return MP_OBJ_SMALL_INT_VALUE(self_in); + } + } else { + const mp_obj_int_t *self = self_in; + long long value = self->val; + mp_uint_t truncated = (mp_uint_t)value; + if (value >= 0 && (long long)truncated == value) { + return truncated; + } + } + mp_raise_msg(&mp_type_OverflowError, MP_ERROR_TEXT("overflow converting long int to machine word")); } #if MICROPY_PY_BUILTINS_FLOAT diff --git a/py/objint_mpz.c b/py/objint_mpz.c index 111f53009fb1f..1bede811a4aaa 100644 --- a/py/objint_mpz.c +++ b/py/objint_mpz.c @@ -364,9 +364,10 @@ static mpz_t *mp_mpz_for_int(mp_obj_t arg, mpz_t *temp) { mp_obj_t mp_obj_int_pow3(mp_obj_t base, mp_obj_t exponent, mp_obj_t modulus) { if (!mp_obj_is_int(base) || !mp_obj_is_int(exponent) || !mp_obj_is_int(modulus)) { mp_raise_TypeError(MP_ERROR_TEXT("pow() with 3 arguments requires integers")); + } else if (modulus == MP_OBJ_NEW_SMALL_INT(0)) { + mp_raise_ValueError(MP_ERROR_TEXT("divide by zero")); } else { - mp_obj_t result = mp_obj_new_int_from_ull(0); // Use the _from_ull version as this forces an mpz int - mp_obj_int_t *res_p = (mp_obj_int_t *)MP_OBJ_TO_PTR(result); + mp_obj_int_t *res_p = mp_obj_int_new_mpz(); mpz_t l_temp, r_temp, m_temp; mpz_t *lhs = mp_mpz_for_int(base, &l_temp); @@ -389,7 +390,7 @@ mp_obj_t mp_obj_int_pow3(mp_obj_t base, mp_obj_t exponent, mp_obj_t modulus) { if (mod == &m_temp) { mpz_deinit(mod); } - return result; + return MP_OBJ_FROM_PTR(res_p); } } #endif diff --git a/py/objlist.c b/py/objlist.c index 3137e9fa53483..a29fc51b10472 100644 --- a/py/objlist.c +++ b/py/objlist.c @@ -118,7 +118,7 @@ static mp_obj_t list_unary_op(mp_unary_op_t op, mp_obj_t self_in) { } static mp_obj_t list_binary_op(mp_binary_op_t op, mp_obj_t lhs, mp_obj_t rhs) { - // CIRCUITPY-CHANGE + // CIRCUITPY-CHANGE: allow subclassing mp_obj_list_t *o = native_list(lhs); switch (op) { case MP_BINARY_OP_ADD: { @@ -142,7 +142,7 @@ static mp_obj_t list_binary_op(mp_binary_op_t op, mp_obj_t lhs, mp_obj_t rhs) { if (n < 0) { n = 0; } - // CIRCUITPY-CHANGE + // CIRCUITPY-CHANGE: overflow check (PR#1279) size_t new_len = mp_seq_multiply_len(o->len, n); mp_obj_list_t *s = list_new(new_len); mp_seq_multiply(o->items, sizeof(*o->items), o->len, n, s->items); @@ -171,76 +171,66 @@ static mp_obj_t list_binary_op(mp_binary_op_t op, mp_obj_t lhs, mp_obj_t rhs) { } static mp_obj_t list_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { - // CIRCUITPY-CHANGE - mp_obj_list_t *self = native_list(self_in); - if (value == MP_OBJ_NULL) { - // delete - #if MICROPY_PY_BUILTINS_SLICE - if (mp_obj_is_type(index, &mp_type_slice)) { - mp_bound_slice_t slice; - if (!mp_seq_get_fast_slice_indexes(self->len, index, &slice)) { - mp_raise_NotImplementedError(NULL); + #if MICROPY_PY_BUILTINS_SLICE + if (mp_obj_is_type(index, &mp_type_slice)) { + // CIRCUITPY-CHANGE: handle subclassing + mp_obj_list_t *self = native_list(self_in); + mp_bound_slice_t slice; + bool fast = mp_seq_get_fast_slice_indexes(self->len, index, &slice); + if (value == MP_OBJ_SENTINEL) { + // load + if (!fast) { + return mp_seq_extract_slice(self->items, &slice); } - - mp_int_t len_adj = slice.start - slice.stop; - assert(len_adj <= 0); - mp_seq_replace_slice_no_grow(self->items, self->len, slice.start, slice.stop, self->items /*NULL*/, 0, sizeof(*self->items)); + mp_obj_list_t *res = list_new(slice.stop - slice.start); + mp_seq_copy(res->items, self->items + slice.start, res->len, mp_obj_t); + return MP_OBJ_FROM_PTR(res); + } + // assign/delete + if (value == MP_OBJ_NULL) { + // delete is equivalent to slice assignment of an empty sequence + value = mp_const_empty_tuple; + } + if (!fast) { + mp_raise_NotImplementedError(NULL); + } + size_t value_len; + mp_obj_t *value_items; + mp_obj_get_array(value, &value_len, &value_items); + mp_int_t len_adj = value_len - (slice.stop - slice.start); + if (len_adj > 0) { + if (self->len + len_adj > self->alloc) { + // TODO: Might optimize memory copies here by checking if block can + // be grown inplace or not + self->items = m_renew(mp_obj_t, self->items, self->alloc, self->len + len_adj); + self->alloc = self->len + len_adj; + } + mp_seq_replace_slice_grow_inplace(self->items, self->len, + slice.start, slice.stop, value_items, value_len, len_adj, sizeof(*self->items)); + } else { + mp_seq_replace_slice_no_grow(self->items, self->len, + slice.start, slice.stop, value_items, value_len, sizeof(*self->items)); // Clear "freed" elements at the end of list mp_seq_clear(self->items, self->len + len_adj, self->len, sizeof(*self->items)); - self->len += len_adj; - return mp_const_none; + // TODO: apply allocation policy re: alloc_size } - #endif - // CIRCUITPY-CHANGE - mp_obj_t args[2] = {MP_OBJ_FROM_PTR(self), index}; + self->len += len_adj; + return mp_const_none; + } + #endif + if (value == MP_OBJ_NULL) { + // delete + // CIRCUITPY-CHANGE: handle subclassing + mp_obj_t args[2] = {MP_OBJ_FROM_PTR(self_in), index}; list_pop(2, args); return mp_const_none; } else if (value == MP_OBJ_SENTINEL) { // load - #if MICROPY_PY_BUILTINS_SLICE - if (mp_obj_is_type(index, &mp_type_slice)) { - mp_bound_slice_t slice; - if (!mp_seq_get_fast_slice_indexes(self->len, index, &slice)) { - return mp_seq_extract_slice(self->items, &slice); - } - mp_obj_list_t *res = list_new(slice.stop - slice.start); - mp_seq_copy(res->items, self->items + slice.start, res->len, mp_obj_t); - return MP_OBJ_FROM_PTR(res); - } - #endif + // CIRCUITPY-CHANGE: handle subclassing + mp_obj_list_t *self = native_list(self_in); size_t index_val = mp_get_index(self->base.type, self->len, index, false); return self->items[index_val]; } else { - #if MICROPY_PY_BUILTINS_SLICE - if (mp_obj_is_type(index, &mp_type_slice)) { - size_t value_len; - mp_obj_t *value_items; - mp_obj_get_array(value, &value_len, &value_items); - mp_bound_slice_t slice_out; - if (!mp_seq_get_fast_slice_indexes(self->len, index, &slice_out)) { - mp_raise_NotImplementedError(NULL); - } - mp_int_t len_adj = value_len - (slice_out.stop - slice_out.start); - if (len_adj > 0) { - if (self->len + len_adj > self->alloc) { - // TODO: Might optimize memory copies here by checking if block can - // be grown inplace or not - self->items = m_renew(mp_obj_t, self->items, self->alloc, self->len + len_adj); - self->alloc = self->len + len_adj; - } - mp_seq_replace_slice_grow_inplace(self->items, self->len, - slice_out.start, slice_out.stop, value_items, value_len, len_adj, sizeof(*self->items)); - } else { - mp_seq_replace_slice_no_grow(self->items, self->len, - slice_out.start, slice_out.stop, value_items, value_len, sizeof(*self->items)); - // Clear "freed" elements at the end of list - mp_seq_clear(self->items, self->len + len_adj, self->len, sizeof(*self->items)); - // TODO: apply allocation policy re: alloc_size - } - self->len += len_adj; - return mp_const_none; - } - #endif mp_obj_list_store(self_in, index, value); return mp_const_none; } @@ -252,7 +242,7 @@ static mp_obj_t list_getiter(mp_obj_t o_in, mp_obj_iter_buf_t *iter_buf) { mp_obj_t mp_obj_list_append(mp_obj_t self_in, mp_obj_t arg) { mp_check_self(mp_obj_is_type(self_in, &mp_type_list)); - // CIRCUITPY-CHANGE + // CIRCUITPY-CHANGE: handle subclassing mp_obj_list_t *self = native_list(self_in); if (self->len >= self->alloc) { self->items = m_renew(mp_obj_t, self->items, self->alloc, self->alloc * 2); @@ -266,7 +256,7 @@ mp_obj_t mp_obj_list_append(mp_obj_t self_in, mp_obj_t arg) { static mp_obj_t list_extend(mp_obj_t self_in, mp_obj_t arg_in) { mp_check_self(mp_obj_is_type(self_in, &mp_type_list)); if (mp_obj_is_type(arg_in, &mp_type_list)) { - // CIRCUITPY-CHANGE + // CIRCUITPY-CHANGE: handle subclassing mp_obj_list_t *self = native_list(self_in); mp_obj_list_t *arg = native_list(arg_in); @@ -285,7 +275,7 @@ static mp_obj_t list_extend(mp_obj_t self_in, mp_obj_t arg_in) { return mp_const_none; // return None, as per CPython } -// CIRCUITPY-CHANGE: used elsewhere so not static; impl is different +// CIRCUITPY-CHANGE: provide version for C use outside this file inline mp_obj_t mp_obj_list_pop(mp_obj_list_t *self, size_t index) { if (self->len == 0) { // CIRCUITPY-CHANGE: more specific mp_raise @@ -303,18 +293,28 @@ inline mp_obj_t mp_obj_list_pop(mp_obj_list_t *self, size_t index) { return ret; } -// CIRCUITPY-CHANGE static mp_obj_t list_pop(size_t n_args, const mp_obj_t *args) { mp_check_self(mp_obj_is_type(args[0], &mp_type_list)); mp_obj_list_t *self = native_list(args[0]); size_t index = mp_get_index(self->base.type, self->len, n_args == 1 ? MP_OBJ_NEW_SMALL_INT(-1) : args[1], false); + // CIRCUITPY-CHANGE: use factored-out pop code return mp_obj_list_pop(self, index); } +// "head" is actually the *exclusive lower bound* of the range to sort. That is, +// the first element to be sorted is `head[1]`, not `head[0]`. Similarly `tail` +// is an *inclusive upper bound* of the range to sort. That is, the final +// element to sort is `tail[0]`, not `tail[-1]`. +// +// The pivot element is always chosen as `tail[0]`. +// +// These unusual choices allows structuring the partitioning +// process as a do/while loop, which generates smaller code than the equivalent +// code with usual C bounds & a while or for loop. static void mp_quicksort(mp_obj_t *head, mp_obj_t *tail, mp_obj_t key_fn, mp_obj_t binop_less_result) { mp_cstack_check(); - while (head < tail) { - mp_obj_t *h = head - 1; + while (tail - head > 1) { // So long as at least 2 elements remain + mp_obj_t *h = head; mp_obj_t *t = tail; mp_obj_t v = key_fn == MP_OBJ_NULL ? tail[0] : mp_call_function_1(key_fn, tail[0]); // get pivot using key_fn for (;;) { @@ -325,19 +325,21 @@ static void mp_quicksort(mp_obj_t *head, mp_obj_t *tail, mp_obj_t key_fn, mp_obj if (h >= t) { break; } + // A pair of objects must be swapped to the other side of the partition mp_obj_t x = h[0]; h[0] = t[0]; t[0] = x; } + // Place the pivot element in the proper position mp_obj_t x = h[0]; h[0] = tail[0]; tail[0] = x; // do the smaller recursive call first, to keep stack within O(log(N)) - if (t - head < tail - h - 1) { + if (t - head < tail - h) { mp_quicksort(head, t, key_fn, binop_less_result); - head = h + 1; + head = h; } else { - mp_quicksort(h + 1, tail, key_fn, binop_less_result); + mp_quicksort(h, tail, key_fn, binop_less_result); tail = t; } } @@ -362,7 +364,7 @@ mp_obj_t mp_obj_list_sort(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_ mp_obj_list_t *self = native_list(pos_args[0]); if (self->len > 1) { - mp_quicksort(self->items, self->items + self->len - 1, + mp_quicksort(self->items - 1, self->items + self->len - 1, args.key.u_obj == mp_const_none ? MP_OBJ_NULL : args.key.u_obj, args.reverse.u_bool ? mp_const_false : mp_const_true); } @@ -370,10 +372,10 @@ mp_obj_t mp_obj_list_sort(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_ return mp_const_none; } -// CIRCUITPY-CHANGE: used elsewhere so not static +// CIRCUITPY-CHANGE: rename and make not static for use elsewhere mp_obj_t mp_obj_list_clear(mp_obj_t self_in) { mp_check_self(mp_obj_is_type(self_in, &mp_type_list)); - // CIRCUITPY-CHANGE + // CIRCUITPY-CHANGE: handle subclassing mp_obj_list_t *self = native_list(self_in); self->len = 0; self->items = m_renew(mp_obj_t, self->items, self->alloc, LIST_MIN_ALLOC); @@ -384,27 +386,28 @@ mp_obj_t mp_obj_list_clear(mp_obj_t self_in) { static mp_obj_t list_copy(mp_obj_t self_in) { mp_check_self(mp_obj_is_type(self_in, &mp_type_list)); - // CIRCUITPY-CHANGE + // CIRCUITPY-CHANGE: handle subclassing mp_obj_list_t *self = native_list(self_in); return mp_obj_new_list(self->len, self->items); } static mp_obj_t list_count(mp_obj_t self_in, mp_obj_t value) { mp_check_self(mp_obj_is_type(self_in, &mp_type_list)); - // CIRCUITPY-CHANGE + // CIRCUITPY-CHANGE: handle subclassing mp_obj_list_t *self = native_list(self_in); return mp_seq_count_obj(self->items, self->len, value); } static mp_obj_t list_index(size_t n_args, const mp_obj_t *args) { mp_check_self(mp_obj_is_type(args[0], &mp_type_list)); - // CIRCUITPY-CHANGE + // CIRCUITPY-CHANGE: handle subclassing mp_obj_list_t *self = native_list(args[0]); return mp_seq_index_obj(self->items, self->len, n_args, args); } -// CIRCUITPY-CHANGE: used elsewhere so not static -inline void mp_obj_list_insert(mp_obj_list_t *self, size_t index, mp_obj_t obj) { +// CIRCUITPY-CHANGE: factor out for C use elsewhere; not meant to emulate Python API +inline void mp_obj_list_insert(mp_obj_list_t *self_in, size_t index, mp_obj_t obj) { + mp_obj_list_t *self = native_list(self_in); mp_obj_list_append(MP_OBJ_FROM_PTR(self), mp_const_none); for (size_t i = self->len - 1; i > index; --i) { @@ -428,7 +431,7 @@ static mp_obj_t list_insert(mp_obj_t self_in, mp_obj_t idx, mp_obj_t obj) { if ((size_t)index > self->len) { index = self->len; } - // CIRCUITPY-CHANGE + // CIRCUITPY-CHANGE: use factored-out insert code mp_obj_list_insert(self, index, obj); return mp_const_none; } diff --git a/py/objmodule.c b/py/objmodule.c index a5c1dee968ea8..84b6a10f9abd4 100644 --- a/py/objmodule.c +++ b/py/objmodule.c @@ -34,11 +34,6 @@ #include "py/runtime.h" #include "py/builtin.h" -// CIRCUITPY-CHANGE -#if CIRCUITPY_WARNINGS -#include "shared-module/warnings/__init__.h" -#endif - static void module_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) { (void)kind; mp_obj_module_t *self = MP_OBJ_TO_PTR(self_in); @@ -49,7 +44,7 @@ static void module_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kin module_name = mp_obj_str_get_str(elem->value); } - #if MICROPY_PY___FILE__ + #if MICROPY_MODULE___FILE__ // If we store __file__ to imported modules then try to lookup this // symbol to give more information about the module. elem = mp_map_lookup(&self->globals->map, MP_OBJ_NEW_QSTR(MP_QSTR___file__), MP_MAP_LOOKUP); @@ -157,11 +152,13 @@ static const mp_rom_map_elem_t mp_builtin_module_table[] = { }; MP_DEFINE_CONST_MAP(mp_builtin_module_map, mp_builtin_module_table); +#if MICROPY_HAVE_REGISTERED_EXTENSIBLE_MODULES static const mp_rom_map_elem_t mp_builtin_extensible_module_table[] = { // built-in modules declared with MP_REGISTER_EXTENSIBLE_MODULE() MICROPY_REGISTERED_EXTENSIBLE_MODULES }; MP_DEFINE_CONST_MAP(mp_builtin_extensible_module_map, mp_builtin_extensible_module_table); +#endif #if MICROPY_MODULE_ATTR_DELEGATION && defined(MICROPY_MODULE_DELEGATIONS) typedef struct _mp_module_delegation_entry_t { @@ -178,13 +175,13 @@ static const mp_module_delegation_entry_t mp_builtin_module_delegation_table[] = // Attempts to find (and initialise) a built-in, otherwise returns // MP_OBJ_NULL. mp_obj_t mp_module_get_builtin(qstr module_name, bool extensible) { - // CIRCUITPY-CHANGE - #if CIRCUITPY_PARALLELDISPLAYBUS && CIRCUITPY_WARNINGS - if (module_name == MP_QSTR_paralleldisplay) { - warnings_warn(&mp_type_FutureWarning, MP_ERROR_TEXT("%q renamed %q"), MP_QSTR_paralleldisplay, MP_QSTR_paralleldisplaybus); - } + #if MICROPY_HAVE_REGISTERED_EXTENSIBLE_MODULES + const mp_map_t *map = extensible ? &mp_builtin_extensible_module_map : &mp_builtin_module_map; + #else + const mp_map_t *map = &mp_builtin_module_map; #endif - mp_map_elem_t *elem = mp_map_lookup((mp_map_t *)(extensible ? &mp_builtin_extensible_module_map : &mp_builtin_module_map), MP_OBJ_NEW_QSTR(module_name), MP_MAP_LOOKUP); + mp_map_elem_t *elem = mp_map_lookup((mp_map_t *)map, MP_OBJ_NEW_QSTR(module_name), MP_MAP_LOOKUP); + if (!elem) { #if MICROPY_PY_SYS // Special case for sys, which isn't extensible but can always be @@ -194,6 +191,7 @@ mp_obj_t mp_module_get_builtin(qstr module_name, bool extensible) { } #endif + #if MICROPY_HAVE_REGISTERED_EXTENSIBLE_MODULES if (extensible) { // At this point we've already tried non-extensible built-ins, the // filesystem, and now extensible built-ins. No match, so fail @@ -210,6 +208,7 @@ mp_obj_t mp_module_get_builtin(qstr module_name, bool extensible) { // the old behaviour of the u-prefix being used to force a built-in // import. // CIRCUITPY-CHANGE: Don't look for `ufoo`. + #endif return MP_OBJ_NULL; } diff --git a/py/objmodule.h b/py/objmodule.h index 8b14cd9fc3d30..221392ccce431 100644 --- a/py/objmodule.h +++ b/py/objmodule.h @@ -32,10 +32,16 @@ // Only include module definitions when not doing qstr extraction, because the // qstr extraction stage also generates this module definition header file. #include "genhdr/moduledefs.h" +// CIRCUITPY-CHANGE: avoid undef warning +#else +#define MICROPY_HAVE_REGISTERED_EXTENSIBLE_MODULES (0) #endif extern const mp_map_t mp_builtin_module_map; + +#if MICROPY_HAVE_REGISTERED_EXTENSIBLE_MODULES extern const mp_map_t mp_builtin_extensible_module_map; +#endif mp_obj_t mp_module_get_builtin(qstr module_name, bool extensible); diff --git a/py/objnamedtuple.c b/py/objnamedtuple.c index b6bbe6e05d8b2..d687dbfda59d6 100644 --- a/py/objnamedtuple.c +++ b/py/objnamedtuple.c @@ -73,7 +73,7 @@ MP_DEFINE_CONST_FUN_OBJ_1(namedtuple_asdict_obj, namedtuple_asdict); void namedtuple_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind) { (void)kind; mp_obj_namedtuple_t *o = MP_OBJ_TO_PTR(o_in); - mp_printf(print, "%q", o->tuple.base.type->name); + mp_printf(print, "%q", (qstr)o->tuple.base.type->name); const qstr *fields = ((mp_obj_namedtuple_type_t *)o->tuple.base.type)->fields; mp_obj_attrtuple_print_helper(print, fields, &o->tuple); } diff --git a/py/objrange.c b/py/objrange.c index bde2ebaabb4ed..dea9fe5f2963a 100644 --- a/py/objrange.c +++ b/py/objrange.c @@ -41,9 +41,9 @@ typedef struct _mp_obj_range_it_t { static mp_obj_t range_it_iternext(mp_obj_t o_in) { mp_obj_range_it_t *o = MP_OBJ_TO_PTR(o_in); if ((o->step > 0 && o->cur < o->stop) || (o->step < 0 && o->cur > o->stop)) { - mp_obj_t o_out = MP_OBJ_NEW_SMALL_INT(o->cur); + mp_int_t cur = o->cur; o->cur += o->step; - return o_out; + return mp_obj_new_int(cur); } else { return MP_OBJ_STOP_ITERATION; } @@ -133,7 +133,7 @@ static mp_obj_t range_unary_op(mp_unary_op_t op, mp_obj_t self_in) { case MP_UNARY_OP_BOOL: return mp_obj_new_bool(len > 0); case MP_UNARY_OP_LEN: - return MP_OBJ_NEW_SMALL_INT(len); + return mp_obj_new_int(len); default: return MP_OBJ_NULL; // op not supported } @@ -165,20 +165,16 @@ static mp_obj_t range_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { #if MICROPY_PY_BUILTINS_SLICE if (mp_obj_is_type(index, &mp_type_slice)) { mp_bound_slice_t slice; - mp_seq_get_fast_slice_indexes(len, index, &slice); + mp_obj_slice_indices(index, len, &slice); mp_obj_range_t *o = mp_obj_malloc(mp_obj_range_t, &mp_type_range); o->start = self->start + slice.start * self->step; o->stop = self->start + slice.stop * self->step; o->step = slice.step * self->step; - if (slice.step < 0) { - // Negative slice steps have inclusive stop, so adjust for exclusive - o->stop -= self->step; - } return MP_OBJ_FROM_PTR(o); } #endif size_t index_val = mp_get_index(self->base.type, len, index, false); - return MP_OBJ_NEW_SMALL_INT(self->start + index_val * self->step); + return mp_obj_new_int(self->start + index_val * self->step); } else { return MP_OBJ_NULL; // op not supported } diff --git a/py/objringio.c b/py/objringio.c index ba1ec25307ea4..0025b26be3b07 100644 --- a/py/objringio.c +++ b/py/objringio.c @@ -39,22 +39,19 @@ typedef struct _micropython_ringio_obj_t { static mp_obj_t micropython_ringio_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { mp_arg_check_num(n_args, n_kw, 1, 1, false); - mp_int_t buff_size = -1; mp_buffer_info_t bufinfo = {NULL, 0, 0}; if (!mp_get_buffer(args[0], &bufinfo, MP_BUFFER_RW)) { - buff_size = mp_obj_get_int(args[0]); + bufinfo.len = mp_obj_get_int(args[0]) + 1; + bufinfo.buf = m_new(uint8_t, bufinfo.len); } - micropython_ringio_obj_t *self = mp_obj_malloc(micropython_ringio_obj_t, type); - if (bufinfo.buf != NULL) { - // buffer passed in, use it directly for ringbuffer. - self->ringbuffer.buf = bufinfo.buf; - self->ringbuffer.size = bufinfo.len; - self->ringbuffer.iget = self->ringbuffer.iput = 0; - } else { - // Allocate new buffer, add one extra to buff_size as ringbuf consumes one byte for tracking. - ringbuf_alloc(&(self->ringbuffer), buff_size + 1); + if (bufinfo.len < 2 || bufinfo.len > UINT16_MAX) { + mp_raise_ValueError(NULL); } + micropython_ringio_obj_t *self = mp_obj_malloc(micropython_ringio_obj_t, type); + self->ringbuffer.buf = bufinfo.buf; + self->ringbuffer.size = bufinfo.len; + self->ringbuffer.iget = self->ringbuffer.iput = 0; return MP_OBJ_FROM_PTR(self); } diff --git a/py/objstr.c b/py/objstr.c index 1874cdb01d03d..bd650bbe18ad6 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -33,6 +33,7 @@ #include "py/objlist.h" #include "py/runtime.h" #include "py/cstack.h" +#include "py/objtuple.h" // CIRCUITPY-CHANGE const char nibble_to_hex_upper[16] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', @@ -46,7 +47,7 @@ static mp_obj_t str_modulo_format(mp_obj_t pattern, size_t n_args, const mp_obj_ #endif static mp_obj_t mp_obj_new_bytes_iterator(mp_obj_t str, mp_obj_iter_buf_t *iter_buf); -static NORETURN void bad_implicit_conversion(mp_obj_t self_in); +static MP_NORETURN void bad_implicit_conversion(mp_obj_t self_in); static mp_obj_t mp_obj_new_str_type_from_vstr(const mp_obj_type_t *type, vstr_t *vstr); @@ -372,8 +373,7 @@ mp_obj_t mp_obj_str_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_i mp_obj_t *args = &rhs_in; size_t n_args = 1; mp_obj_t dict = MP_OBJ_NULL; - if (mp_obj_is_type(rhs_in, &mp_type_tuple)) { - // TODO: Support tuple subclasses? + if (mp_obj_is_tuple_compatible(rhs_in)) { mp_obj_tuple_get(rhs_in, &n_args, &args); } else if (mp_obj_is_type(rhs_in, &mp_type_dict)) { dict = rhs_in; @@ -1018,7 +1018,7 @@ static mp_obj_t arg_as_int(mp_obj_t arg) { #endif #if MICROPY_ERROR_REPORTING <= MICROPY_ERROR_REPORTING_TERSE -static NORETURN void terse_str_format_value_error(void) { +static MP_NORETURN void terse_str_format_value_error(void) { mp_raise_ValueError(MP_ERROR_TEXT("bad format string")); } #else @@ -1205,7 +1205,7 @@ static vstr_t mp_obj_str_format_helper(const char *str, const char *top, int *ar int width = -1; int precision = -1; char type = '\0'; - int flags = 0; + unsigned int flags = 0; if (format_spec) { // The format specifier (from http://docs.python.org/2/library/string.html#formatspec) @@ -1250,8 +1250,9 @@ static vstr_t mp_obj_str_format_helper(const char *str, const char *top, int *ar } } s = str_to_int(s, stop, &width); - if (*s == ',') { - flags |= PF_FLAG_SHOW_COMMA; + if (*s == ',' || *s == '_') { + MP_STATIC_ASSERT((unsigned)'_' << PF_FLAG_SEP_POS >> PF_FLAG_SEP_POS == '_'); + flags |= (unsigned)*s << PF_FLAG_SEP_POS; s++; } if (*s == '.') { @@ -1324,7 +1325,7 @@ static vstr_t mp_obj_str_format_helper(const char *str, const char *top, int *ar } case '\0': // No explicit format type implies 'd' - case 'n': // I don't think we support locales in uPy so use 'd' + case 'n': // I don't think we support locales in MicroPython so use 'd' case 'd': mp_print_mp_int(&print, arg, 10, 'a', flags, fill, width, 0); continue; @@ -2395,7 +2396,7 @@ bool mp_obj_str_equal(mp_obj_t s1, mp_obj_t s2) { } } -static NORETURN void bad_implicit_conversion(mp_obj_t self_in) { +static MP_NORETURN void bad_implicit_conversion(mp_obj_t self_in) { #if MICROPY_ERROR_REPORTING <= MICROPY_ERROR_REPORTING_TERSE mp_raise_TypeError(MP_ERROR_TEXT("can't convert to str implicitly")); #else diff --git a/py/objtuple.c b/py/objtuple.c index e0b31edaf2376..b2ccbbb299082 100644 --- a/py/objtuple.c +++ b/py/objtuple.c @@ -31,9 +31,6 @@ #include "py/objtuple.h" #include "py/runtime.h" -// type check is done on getiter method to allow tuple, namedtuple, attrtuple -#define mp_obj_is_tuple_compatible(o) (MP_OBJ_TYPE_GET_SLOT_OR_NULL(mp_obj_get_type(o), iter) == mp_obj_tuple_getiter) - /******************************************************************************/ /* tuple */ diff --git a/py/objtuple.h b/py/objtuple.h index e2d010e6c5a3d..783522a6222b5 100644 --- a/py/objtuple.h +++ b/py/objtuple.h @@ -57,7 +57,7 @@ extern const mp_obj_type_t mp_type_attrtuple; #define MP_DEFINE_ATTRTUPLE(tuple_obj_name, fields, nitems, ...) \ const mp_rom_obj_tuple_t tuple_obj_name = { \ - .base = {&mp_type_attrtuple}, \ + .base = {.type = &mp_type_attrtuple}, \ .len = nitems, \ .items = { __VA_ARGS__, MP_ROM_PTR((void *)fields) } \ } @@ -68,4 +68,7 @@ void mp_obj_attrtuple_print_helper(const mp_print_t *print, const qstr *fields, mp_obj_t mp_obj_new_attrtuple(const qstr *fields, size_t n, const mp_obj_t *items); +// type check is done on getiter method to allow tuple, namedtuple, attrtuple +#define mp_obj_is_tuple_compatible(o) (MP_OBJ_TYPE_GET_SLOT_OR_NULL(mp_obj_get_type(o), iter) == mp_obj_tuple_getiter) + #endif // MICROPY_INCLUDED_PY_OBJTUPLE_H diff --git a/py/objtype.c b/py/objtype.c index 074d6f39293db..b1a984b50e057 100644 --- a/py/objtype.c +++ b/py/objtype.c @@ -44,6 +44,7 @@ #define ENABLE_SPECIAL_ACCESSORS \ (MICROPY_PY_DESCRIPTORS || MICROPY_PY_DELATTR_SETATTR || MICROPY_PY_BUILTINS_PROPERTY) +static mp_obj_t mp_obj_new_type(qstr name, mp_obj_t bases_tuple, mp_obj_t locals_dict); static mp_obj_t mp_obj_is_subclass(mp_obj_t object, mp_obj_t classinfo); static mp_obj_t static_class_method_make_new(const mp_obj_type_t *self_in, size_t n_args, size_t n_kw, const mp_obj_t *args); @@ -690,8 +691,8 @@ static void mp_obj_instance_load_attr(mp_obj_t self_in, qstr attr, mp_obj_t *des // try __getattr__ if (attr != MP_QSTR___getattr__) { #if MICROPY_PY_DESCRIPTORS - // With descriptors enabled, don't delegate lookups of __get__/__set__/__delete__. - if (attr == MP_QSTR___get__ || attr == MP_QSTR___set__ || attr == MP_QSTR___delete__) { + // With descriptors enabled, don't delegate lookups of __get__/__set__/__delete__/__set_name__. + if (attr == MP_QSTR___get__ || attr == MP_QSTR___set__ || attr == MP_QSTR___delete__ || attr == MP_QSTR___set_name__) { return; } #endif @@ -996,7 +997,7 @@ static bool check_for_special_accessors(mp_obj_t key, mp_obj_t value) { #endif #if MICROPY_PY_DESCRIPTORS static const uint8_t to_check[] = { - MP_QSTR___get__, MP_QSTR___set__, MP_QSTR___delete__, + MP_QSTR___get__, MP_QSTR___set__, MP_QSTR___delete__, // not needed for MP_QSTR___set_name__ though }; for (size_t i = 0; i < MP_ARRAY_SIZE(to_check); ++i) { mp_obj_t dest_temp[2]; @@ -1010,10 +1011,52 @@ static bool check_for_special_accessors(mp_obj_t key, mp_obj_t value) { } #endif +#if MICROPY_PY_DESCRIPTORS +// Shared data layout for the __set_name__ call and a linked list of calls to be made. +typedef union _setname_list_t setname_list_t; +union _setname_list_t { + mp_obj_t call[4]; + struct { + mp_obj_t _meth; + mp_obj_t _self; + setname_list_t *next; // can use the "owner" argument position temporarily for the linked list + mp_obj_t _name; + }; +}; + +// Append any `__set_name__` method on `value` to the setname list, with its per-attr args +static setname_list_t *setname_maybe_bind_append(setname_list_t *tail, mp_obj_t name, mp_obj_t value) { + // make certain our type-punning is safe: + MP_STATIC_ASSERT_NONCONSTEXPR(offsetof(setname_list_t, next) == offsetof(setname_list_t, call[2])); + + // tail is a blank list entry + mp_load_method_maybe(value, MP_QSTR___set_name__, tail->call); + if (tail->call[1] != MP_OBJ_NULL) { + // Each time a __set_name__ is found, leave it in-place in the former tail and allocate a new tail + tail->next = m_new_obj(setname_list_t); + tail->next->next = NULL; + tail->call[3] = name; + return tail->next; + } else { + return tail; + } +} + +// Execute the captured `__set_name__` calls, destroying the setname list in the process. +static inline void setname_consume_call_all(setname_list_t *head, mp_obj_t owner) { + setname_list_t *next; + while ((next = head->next) != NULL) { + head->call[2] = owner; + mp_call_method_n_kw(2, 0, head->call); + head = next; + } +} +#endif + static void type_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) { (void)kind; mp_obj_type_t *self = MP_OBJ_TO_PTR(self_in); - mp_printf(print, "", self->name); + mp_printf(print, "", (qstr)self->name); } static mp_obj_t type_make_new(const mp_obj_type_t *type_in, size_t n_args, size_t n_kw, const mp_obj_t *args) { @@ -1043,7 +1086,7 @@ static mp_obj_t type_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const mp if (!MP_OBJ_TYPE_HAS_SLOT(self, make_new)) { #if MICROPY_ERROR_REPORTING <= MICROPY_ERROR_REPORTING_TERSE - mp_raise_TypeError(MP_ERROR_TEXT("cannot create instance")); + mp_raise_TypeError(MP_ERROR_TEXT("can't create instance")); #else // CIRCUITPY-CHANGE: more specific mp_raise mp_raise_TypeError_varg(MP_ERROR_TEXT("can't create '%q' instances"), self->name); @@ -1159,7 +1202,7 @@ MP_DEFINE_CONST_OBJ_TYPE( attr, type_attr ); -mp_obj_t mp_obj_new_type(qstr name, mp_obj_t bases_tuple, mp_obj_t locals_dict) { +static mp_obj_t mp_obj_new_type(qstr name, mp_obj_t bases_tuple, mp_obj_t locals_dict) { // Verify input objects have expected type if (!mp_obj_is_type(bases_tuple, &mp_type_tuple)) { mp_raise_TypeError(NULL); @@ -1252,20 +1295,38 @@ mp_obj_t mp_obj_new_type(qstr name, mp_obj_t bases_tuple, mp_obj_t locals_dict) } } + #if MICROPY_PY_DESCRIPTORS + // To avoid any dynamic allocations when no __set_name__ exists, + // the head of this list is kept on the stack (marked blank with `next = NULL`). + setname_list_t setname_list = { .next = NULL }; + setname_list_t *setname_tail = &setname_list; + #endif + #if ENABLE_SPECIAL_ACCESSORS - // Check if the class has any special accessor methods - if (!(o->flags & MP_TYPE_FLAG_HAS_SPECIAL_ACCESSORS)) { - for (size_t i = 0; i < locals_ptr->map.alloc; i++) { - if (mp_map_slot_is_filled(&locals_ptr->map, i)) { - const mp_map_elem_t *elem = &locals_ptr->map.table[i]; - if (check_for_special_accessors(elem->key, elem->value)) { - o->flags |= MP_TYPE_FLAG_HAS_SPECIAL_ACCESSORS; - break; - } + // Check if the class has any special accessor methods, + // and accumulate bound __set_name__ methods that need to be called + for (size_t i = 0; i < locals_ptr->map.alloc; i++) { + #if !MICROPY_PY_DESCRIPTORS + // __set_name__ needs to scan the entire locals map, can't early-terminate + if (o->flags & MP_TYPE_FLAG_HAS_SPECIAL_ACCESSORS) { + break; + } + #endif + + if (mp_map_slot_is_filled(&locals_ptr->map, i)) { + const mp_map_elem_t *elem = &locals_ptr->map.table[i]; + + if (!(o->flags & MP_TYPE_FLAG_HAS_SPECIAL_ACCESSORS) // elidable when the early-termination check is enabled + && check_for_special_accessors(elem->key, elem->value)) { + o->flags |= MP_TYPE_FLAG_HAS_SPECIAL_ACCESSORS; } + + #if MICROPY_PY_DESCRIPTORS + setname_tail = setname_maybe_bind_append(setname_tail, elem->key, elem->value); + #endif } } - #endif + #endif // ENABLE_SPECIAL_ACCESSORS const mp_obj_type_t *native_base; size_t num_native_bases = instance_count_native_bases(o, &native_base); @@ -1273,8 +1334,7 @@ mp_obj_t mp_obj_new_type(qstr name, mp_obj_t bases_tuple, mp_obj_t locals_dict) mp_raise_TypeError(MP_ERROR_TEXT("multiple bases have instance lay-out conflict")); } - mp_map_t *locals_map = &MP_OBJ_TYPE_GET_SLOT(o, locals_dict)->map; - mp_map_elem_t *elem = mp_map_lookup(locals_map, MP_OBJ_NEW_QSTR(MP_QSTR___new__), MP_MAP_LOOKUP); + mp_map_elem_t *elem = mp_map_lookup(&locals_ptr->map, MP_OBJ_NEW_QSTR(MP_QSTR___new__), MP_MAP_LOOKUP); if (elem != NULL) { // __new__ slot exists; check if it is a function if (mp_obj_is_fun(elem->value)) { @@ -1283,6 +1343,10 @@ mp_obj_t mp_obj_new_type(qstr name, mp_obj_t bases_tuple, mp_obj_t locals_dict) } } + #if MICROPY_PY_DESCRIPTORS + setname_consume_call_all(&setname_list, MP_OBJ_FROM_PTR(o)); + #endif + return MP_OBJ_FROM_PTR(o); } diff --git a/py/parse.c b/py/parse.c index a393b0ee8c153..36150ef468bdf 100644 --- a/py/parse.c +++ b/py/parse.c @@ -341,18 +341,34 @@ static uint8_t peek_rule(parser_t *parser, size_t n) { } #endif -bool mp_parse_node_get_int_maybe(mp_parse_node_t pn, mp_obj_t *o) { +#if MICROPY_COMP_CONST_FOLDING || MICROPY_EMIT_INLINE_ASM +static bool mp_parse_node_get_number_maybe(mp_parse_node_t pn, mp_obj_t *o) { if (MP_PARSE_NODE_IS_SMALL_INT(pn)) { *o = MP_OBJ_NEW_SMALL_INT(MP_PARSE_NODE_LEAF_SMALL_INT(pn)); return true; } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pn, RULE_const_object)) { mp_parse_node_struct_t *pns = (mp_parse_node_struct_t *)pn; *o = mp_parse_node_extract_const_object(pns); - return mp_obj_is_int(*o); + return mp_obj_is_int(*o) + #if MICROPY_COMP_CONST_FLOAT + || mp_obj_is_float(*o) + #endif + ; } else { return false; } } +#endif + +#if MICROPY_EMIT_INLINE_ASM +bool mp_parse_node_get_int_maybe(mp_parse_node_t pn, mp_obj_t *o) { + return mp_parse_node_get_number_maybe(pn, o) + #if MICROPY_COMP_CONST_FLOAT + && mp_obj_is_int(*o) + #endif + ; +} +#endif #if MICROPY_COMP_CONST_TUPLE || MICROPY_COMP_CONST static bool mp_parse_node_is_const(mp_parse_node_t pn) { @@ -647,12 +663,32 @@ static const mp_rom_map_elem_t mp_constants_table[] = { #if MICROPY_PY_UCTYPES { MP_ROM_QSTR(MP_QSTR_uctypes), MP_ROM_PTR(&mp_module_uctypes) }, #endif + #if MICROPY_PY_BUILTINS_FLOAT && MICROPY_PY_MATH && MICROPY_COMP_CONST_FLOAT + { MP_ROM_QSTR(MP_QSTR_math), MP_ROM_PTR(&mp_module_math) }, + #endif // Extra constants as defined by a port MICROPY_PORT_CONSTANTS }; static MP_DEFINE_CONST_MAP(mp_constants_map, mp_constants_table); #endif +static bool binary_op_maybe(mp_binary_op_t op, mp_obj_t lhs, mp_obj_t rhs, mp_obj_t *res) { + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + mp_obj_t tmp = mp_binary_op(op, lhs, rhs); + nlr_pop(); + #if MICROPY_PY_BUILTINS_COMPLEX + if (mp_obj_is_type(tmp, &mp_type_complex)) { + return false; + } + #endif + *res = tmp; + return true; + } else { + return false; + } +} + static bool fold_logical_constants(parser_t *parser, uint8_t rule_id, size_t *num_args) { if (rule_id == RULE_or_test || rule_id == RULE_and_test) { @@ -711,7 +747,7 @@ static bool fold_logical_constants(parser_t *parser, uint8_t rule_id, size_t *nu } static bool fold_constants(parser_t *parser, uint8_t rule_id, size_t num_args) { - // this code does folding of arbitrary integer expressions, eg 1 + 2 * 3 + 4 + // this code does folding of arbitrary numeric expressions, eg 1 + 2 * 3 + 4 // it does not do partial folding, eg 1 + 2 + x -> 3 + x mp_obj_t arg0; @@ -721,7 +757,7 @@ static bool fold_constants(parser_t *parser, uint8_t rule_id, size_t num_args) { || rule_id == RULE_power) { // folding for binary ops: | ^ & ** mp_parse_node_t pn = peek_result(parser, num_args - 1); - if (!mp_parse_node_get_int_maybe(pn, &arg0)) { + if (!mp_parse_node_get_number_maybe(pn, &arg0)) { return false; } mp_binary_op_t op; @@ -737,58 +773,45 @@ static bool fold_constants(parser_t *parser, uint8_t rule_id, size_t num_args) { for (ssize_t i = num_args - 2; i >= 0; --i) { pn = peek_result(parser, i); mp_obj_t arg1; - if (!mp_parse_node_get_int_maybe(pn, &arg1)) { + if (!mp_parse_node_get_number_maybe(pn, &arg1)) { return false; } - if (op == MP_BINARY_OP_POWER && mp_obj_int_sign(arg1) < 0) { - // ** can't have negative rhs + if (!binary_op_maybe(op, arg0, arg1, &arg0)) { return false; } - arg0 = mp_binary_op(op, arg0, arg1); } } else if (rule_id == RULE_shift_expr || rule_id == RULE_arith_expr || rule_id == RULE_term) { // folding for binary ops: << >> + - * @ / % // mp_parse_node_t pn = peek_result(parser, num_args - 1); - if (!mp_parse_node_get_int_maybe(pn, &arg0)) { + if (!mp_parse_node_get_number_maybe(pn, &arg0)) { return false; } for (ssize_t i = num_args - 2; i >= 1; i -= 2) { pn = peek_result(parser, i - 1); mp_obj_t arg1; - if (!mp_parse_node_get_int_maybe(pn, &arg1)) { + if (!mp_parse_node_get_number_maybe(pn, &arg1)) { return false; } mp_token_kind_t tok = MP_PARSE_NODE_LEAF_ARG(peek_result(parser, i)); - if (tok == MP_TOKEN_OP_AT || tok == MP_TOKEN_OP_SLASH) { - // Can't fold @ or / - return false; - } mp_binary_op_t op = MP_BINARY_OP_LSHIFT + (tok - MP_TOKEN_OP_DBL_LESS); - int rhs_sign = mp_obj_int_sign(arg1); - if (op <= MP_BINARY_OP_RSHIFT) { - // << and >> can't have negative rhs - if (rhs_sign < 0) { - return false; - } - } else if (op >= MP_BINARY_OP_FLOOR_DIVIDE) { - // % and // can't have zero rhs - if (rhs_sign == 0) { - return false; - } + if (!binary_op_maybe(op, arg0, arg1, &arg0)) { + return false; } - arg0 = mp_binary_op(op, arg0, arg1); } } else if (rule_id == RULE_factor_2) { // folding for unary ops: + - ~ mp_parse_node_t pn = peek_result(parser, 0); - if (!mp_parse_node_get_int_maybe(pn, &arg0)) { + if (!mp_parse_node_get_number_maybe(pn, &arg0)) { return false; } mp_token_kind_t tok = MP_PARSE_NODE_LEAF_ARG(peek_result(parser, 1)); mp_unary_op_t op; if (tok == MP_TOKEN_OP_TILDE) { + if (!mp_obj_is_int(arg0)) { + return false; + } op = MP_UNARY_OP_INVERT; } else { assert(tok == MP_TOKEN_OP_PLUS || tok == MP_TOKEN_OP_MINUS); // should be @@ -860,7 +883,7 @@ static bool fold_constants(parser_t *parser, uint8_t rule_id, size_t num_args) { return false; } // id1.id2 - // look it up in constant table, see if it can be replaced with an integer + // look it up in constant table, see if it can be replaced with an integer or a float mp_parse_node_struct_t *pns1 = (mp_parse_node_struct_t *)pn1; assert(MP_PARSE_NODE_IS_ID(pns1->nodes[0])); qstr q_base = MP_PARSE_NODE_LEAF_ARG(pn0); @@ -871,7 +894,7 @@ static bool fold_constants(parser_t *parser, uint8_t rule_id, size_t num_args) { } mp_obj_t dest[2]; mp_load_method_maybe(elem->value, q_attr, dest); - if (!(dest[0] != MP_OBJ_NULL && mp_obj_is_int(dest[0]) && dest[1] == MP_OBJ_NULL)) { + if (!(dest[0] != MP_OBJ_NULL && (mp_obj_is_int(dest[0]) || mp_obj_is_float(dest[0])) && dest[1] == MP_OBJ_NULL)) { return false; } arg0 = dest[0]; diff --git a/py/parsenum.c b/py/parsenum.c index 874216f08d005..c52f040e1a210 100644 --- a/py/parsenum.c +++ b/py/parsenum.c @@ -28,6 +28,7 @@ #include #include "py/runtime.h" +#include "py/misc.h" #include "py/parsenumbase.h" #include "py/parsenum.h" #include "py/smallint.h" @@ -36,7 +37,7 @@ #include #endif -static NORETURN void raise_exc(mp_obj_t exc, mp_lexer_t *lex) { +static MP_NORETURN void raise_exc(mp_obj_t exc, mp_lexer_t *lex) { // if lex!=NULL then the parser called us and we need to convert the // exception's type from ValueError to SyntaxError and add traceback info if (lex != NULL) { @@ -46,6 +47,27 @@ static NORETURN void raise_exc(mp_obj_t exc, mp_lexer_t *lex) { nlr_raise(exc); } +#if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_LONGLONG +// For the common small integer parsing case, we parse directly to mp_int_t and +// check that the value doesn't overflow a smallint (in which case we fail over +// to bigint parsing if supported) +typedef mp_int_t parsed_int_t; + +#define PARSED_INT_MUL_OVERFLOW mp_mul_mp_int_t_overflow +#define PARSED_INT_FITS MP_SMALL_INT_FITS +#else +// In the special case where bigint support is long long, we save code size by +// parsing directly to long long and then return either a bigint or smallint +// from the same result. +// +// To avoid pulling in (slow) signed 64-bit math routines we do the initial +// parsing to an unsigned long long and only convert to signed at the end. +typedef unsigned long long parsed_int_t; + +#define PARSED_INT_MUL_OVERFLOW mp_mul_ull_overflow +#define PARSED_INT_FITS(I) ((I) <= (unsigned long long)LLONG_MAX + 1) +#endif + mp_obj_t mp_parse_num_integer(const char *restrict str_, size_t len, int base, mp_lexer_t *lex) { const byte *restrict str = (const byte *)str_; const byte *restrict top = str + len; @@ -77,7 +99,7 @@ mp_obj_t mp_parse_num_integer(const char *restrict str_, size_t len, int base, m str += mp_parse_num_base((const char *)str, top - str, &base); // string should be an integer number - mp_int_t int_val = 0; + parsed_int_t parsed_val = 0; const byte *restrict str_val_start = str; for (; str < top; str++) { // get next digit as a value @@ -99,25 +121,32 @@ mp_obj_t mp_parse_num_integer(const char *restrict str_, size_t len, int base, m break; } - // add next digi and check for overflow - if (mp_small_int_mul_overflow(int_val, base)) { + // add next digit and check for overflow + if (PARSED_INT_MUL_OVERFLOW(parsed_val, base, &parsed_val)) { goto overflow; } - int_val = int_val * base + dig; - if (!MP_SMALL_INT_FITS(int_val)) { + parsed_val += dig; + if (!PARSED_INT_FITS(parsed_val)) { goto overflow; } } - // negate value if needed - if (neg) { - int_val = -int_val; + #if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_LONGLONG + // The PARSED_INT_FITS check above ensures parsed_val fits in small int representation + ret_val = MP_OBJ_NEW_SMALL_INT(neg ? (-parsed_val) : parsed_val); +have_ret_val: + #else + // The PARSED_INT_FITS check above ensures parsed_val won't overflow signed long long + long long signed_val = -parsed_val; + if (!neg) { + if (signed_val == LLONG_MIN) { + goto overflow; + } + signed_val = -signed_val; } + ret_val = mp_obj_new_int_from_ll(signed_val); // Could be large or small int + #endif - // create the small int - ret_val = MP_OBJ_NEW_SMALL_INT(int_val); - -have_ret_val: // check we parsed something if (str == str_val_start) { goto value_error; @@ -136,6 +165,7 @@ mp_obj_t mp_parse_num_integer(const char *restrict str_, size_t len, int base, m return ret_val; overflow: + #if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_LONGLONG // reparse using long int { const char *s2 = (const char *)str_val_start; @@ -143,6 +173,9 @@ mp_obj_t mp_parse_num_integer(const char *restrict str_, size_t len, int base, m str = (const byte *)s2; goto have_ret_val; } + #else + mp_raise_msg(&mp_type_OverflowError, MP_ERROR_TEXT("result overflows long long storage")); + #endif value_error: { @@ -167,6 +200,8 @@ mp_obj_t mp_parse_num_integer(const char *restrict str_, size_t len, int base, m } } +#if MICROPY_PY_BUILTINS_FLOAT + enum { REAL_IMAG_STATE_START = 0, REAL_IMAG_STATE_HAVE_REAL = 1, @@ -179,27 +214,81 @@ typedef enum { PARSE_DEC_IN_EXP, } parse_dec_in_t; -#if MICROPY_PY_BUILTINS_FLOAT // MANTISSA_MAX is used to retain precision while not overflowing mantissa -// SMALL_NORMAL_VAL is the smallest power of 10 that is still a normal float -// EXACT_POWER_OF_10 is the largest value of x so that 10^x can be stored exactly in a float -// Note: EXACT_POWER_OF_10 is at least floor(log_5(2^mantissa_length)). Indeed, 10^n = 2^n * 5^n -// so we only have to store the 5^n part in the mantissa (the 2^n part will go into the float's -// exponent). +#define MANTISSA_MAX (sizeof(mp_large_float_uint_t) == 8 ? 0x1999999999999998ULL : 0x19999998U) + +// MAX_EXACT_POWER_OF_5 is the largest value of x so that 5^x can be stored exactly in a float #if MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_FLOAT -#define MANTISSA_MAX 0x19999998U -#define SMALL_NORMAL_VAL (1e-37F) -#define SMALL_NORMAL_EXP (-37) -#define EXACT_POWER_OF_10 (9) +#define MAX_EXACT_POWER_OF_5 (10) #elif MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_DOUBLE -#define MANTISSA_MAX 0x1999999999999998ULL -#define SMALL_NORMAL_VAL (1e-307) -#define SMALL_NORMAL_EXP (-307) -#define EXACT_POWER_OF_10 (22) +#define MAX_EXACT_POWER_OF_5 (22) #endif +// Helper to compute `num * (10.0 ** dec_exp)` +mp_large_float_t mp_decimal_exp(mp_large_float_t num, int dec_exp) { + // CIRCUITPY-CHANGE: ignore float equal warning + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wfloat-equal" + if (dec_exp == 0 || num == (mp_large_float_t)(0.0)) { + return num; + } + #pragma GCC diagnostic pop + + #if MICROPY_FLOAT_FORMAT_IMPL == MICROPY_FLOAT_FORMAT_IMPL_EXACT + + // If the assert below fails, it means you have chosen MICROPY_FLOAT_FORMAT_IMPL_EXACT + // manually on a platform where `larger floats` are not supported, which would + // result in inexact conversions. To fix this issue, change your `mpconfigport.h` + // and select MICROPY_FLOAT_FORMAT_IMPL_APPROX instead + assert(sizeof(mp_large_float_t) > sizeof(mp_float_t)); + + // Perform power using simple multiplications, to avoid + // dependency to higher-precision pow() function + int neg_exp = (dec_exp < 0); + if (neg_exp) { + dec_exp = -dec_exp; + } + mp_large_float_t res = num; + mp_large_float_t expo = (mp_large_float_t)10.0; + while (dec_exp) { + if (dec_exp & 1) { + if (neg_exp) { + res /= expo; + } else { + res *= expo; + } + } + dec_exp >>= 1; + if (dec_exp) { + expo *= expo; + } + } + return res; + + #else + // MICROPY_FLOAT_FORMAT_IMPL != MICROPY_FLOAT_FORMAT_IMPL_EXACT + + mp_float_union_t res = {num}; + // Multiply first by (2.0 ** dec_exp) via the exponent + // - this will ensure that the result of `pow()` is always in mp_float_t range + // when the result is expected to be in mp_float_t range (e.g. during format) + // - we don't need to care about p.exp overflow, as (5.0 ** dec_exp) will anyway + // force the final result toward the proper edge if needed (0.0 or inf) + res.p.exp += dec_exp; + // Use positive exponents when they are more precise then negative + if (dec_exp < 0 && dec_exp >= -MAX_EXACT_POWER_OF_5) { + res.f /= MICROPY_FLOAT_C_FUN(pow)(5, -dec_exp); + } else { + res.f *= MICROPY_FLOAT_C_FUN(pow)(5, dec_exp); + } + return (mp_large_float_t)res.f; + + #endif +} + + // Break out inner digit accumulation routine to ease trailing zero deferral. -static mp_float_uint_t accept_digit(mp_float_uint_t p_mantissa, unsigned int dig, int *p_exp_extra, int in) { +static mp_large_float_uint_t accept_digit(mp_large_float_uint_t p_mantissa, unsigned int dig, int *p_exp_extra, int in) { // Core routine to ingest an additional digit. if (p_mantissa < MANTISSA_MAX) { // dec_val won't overflow so keep accumulating @@ -216,6 +305,85 @@ static mp_float_uint_t accept_digit(mp_float_uint_t p_mantissa, unsigned int dig return p_mantissa; } } + +// Helper to parse an unsigned decimal number into a mp_float_t +const char *mp_parse_float_internal(const char *str, size_t len, mp_float_t *res) { + const char *top = str + len; + + parse_dec_in_t in = PARSE_DEC_IN_INTG; + bool exp_neg = false; + mp_large_float_uint_t mantissa = 0; + int exp_val = 0; + int exp_extra = 0; + int trailing_zeros_intg = 0, trailing_zeros_frac = 0; + while (str < top) { + unsigned int dig = *str++; + if ('0' <= dig && dig <= '9') { + dig -= '0'; + if (in == PARSE_DEC_IN_EXP) { + // don't overflow exp_val when adding next digit, instead just truncate + // it and the resulting float will still be correct, either inf or 0.0 + // (use INT_MAX/2 to allow adding exp_extra at the end without overflow) + if (exp_val < (INT_MAX / 2 - 9) / 10) { + exp_val = 10 * exp_val + dig; + } + } else { + if (dig == 0 || mantissa >= MANTISSA_MAX) { + // Defer treatment of zeros in fractional part. If nothing comes afterwards, ignore them. + // Also, once we reach MANTISSA_MAX, treat every additional digit as a trailing zero. + if (in == PARSE_DEC_IN_INTG) { + ++trailing_zeros_intg; + } else { + ++trailing_zeros_frac; + } + } else { + // Time to un-defer any trailing zeros. Intg zeros first. + while (trailing_zeros_intg) { + mantissa = accept_digit(mantissa, 0, &exp_extra, PARSE_DEC_IN_INTG); + --trailing_zeros_intg; + } + while (trailing_zeros_frac) { + mantissa = accept_digit(mantissa, 0, &exp_extra, PARSE_DEC_IN_FRAC); + --trailing_zeros_frac; + } + mantissa = accept_digit(mantissa, dig, &exp_extra, in); + } + } + } else if (in == PARSE_DEC_IN_INTG && dig == '.') { + in = PARSE_DEC_IN_FRAC; + } else if (in != PARSE_DEC_IN_EXP && ((dig | 0x20) == 'e')) { + in = PARSE_DEC_IN_EXP; + if (str < top) { + if (str[0] == '+') { + str++; + } else if (str[0] == '-') { + str++; + exp_neg = true; + } + } + if (str == top) { + return NULL; + } + } else if (dig == '_') { + continue; + } else { + // unknown character + str--; + break; + } + } + + // work out the exponent + if (exp_neg) { + exp_val = -exp_val; + } + exp_val += exp_extra + trailing_zeros_intg; + + // At this point, we just need to multiply the mantissa by its base 10 exponent. + *res = (mp_float_t)mp_decimal_exp(mantissa, exp_val); + + return str; +} #endif // MICROPY_PY_BUILTINS_FLOAT #if MICROPY_PY_BUILTINS_COMPLEX @@ -253,111 +421,23 @@ parse_start:; const char *str_val_start = str; // determine what the string is - if (str < top && (str[0] | 0x20) == 'i') { - // string starts with 'i', should be 'inf' or 'infinity' (case insensitive) - if (str + 2 < top && (str[1] | 0x20) == 'n' && (str[2] | 0x20) == 'f') { - // inf - str += 3; - dec_val = (mp_float_t)INFINITY; - if (str + 4 < top && (str[0] | 0x20) == 'i' && (str[1] | 0x20) == 'n' && (str[2] | 0x20) == 'i' && (str[3] | 0x20) == 't' && (str[4] | 0x20) == 'y') { - // infinity - str += 5; - } - } - } else if (str < top && (str[0] | 0x20) == 'n') { - // string starts with 'n', should be 'nan' (case insensitive) - if (str + 2 < top && (str[1] | 0x20) == 'a' && (str[2] | 0x20) == 'n') { - // NaN - str += 3; - dec_val = MICROPY_FLOAT_C_FUN(nan)(""); + if (str + 2 < top && (str[0] | 0x20) == 'i' && (str[1] | 0x20) == 'n' && (str[2] | 0x20) == 'f') { + // 'inf' or 'infinity' (case insensitive) + str += 3; + dec_val = (mp_float_t)INFINITY; + if (str + 4 < top && (str[0] | 0x20) == 'i' && (str[1] | 0x20) == 'n' && (str[2] | 0x20) == 'i' && (str[3] | 0x20) == 't' && (str[4] | 0x20) == 'y') { + // infinity + str += 5; } + } else if (str + 2 < top && (str[0] | 0x20) == 'n' && (str[1] | 0x20) == 'a' && (str[2] | 0x20) == 'n') { + // 'nan' (case insensitive) + str += 3; + dec_val = MICROPY_FLOAT_C_FUN(nan)(""); } else { // string should be a decimal number - parse_dec_in_t in = PARSE_DEC_IN_INTG; - bool exp_neg = false; - mp_float_uint_t mantissa = 0; - int exp_val = 0; - int exp_extra = 0; - int trailing_zeros_intg = 0, trailing_zeros_frac = 0; - while (str < top) { - unsigned int dig = *str++; - if ('0' <= dig && dig <= '9') { - dig -= '0'; - if (in == PARSE_DEC_IN_EXP) { - // don't overflow exp_val when adding next digit, instead just truncate - // it and the resulting float will still be correct, either inf or 0.0 - // (use INT_MAX/2 to allow adding exp_extra at the end without overflow) - if (exp_val < (INT_MAX / 2 - 9) / 10) { - exp_val = 10 * exp_val + dig; - } - } else { - if (dig == 0 || mantissa >= MANTISSA_MAX) { - // Defer treatment of zeros in fractional part. If nothing comes afterwards, ignore them. - // Also, once we reach MANTISSA_MAX, treat every additional digit as a trailing zero. - if (in == PARSE_DEC_IN_INTG) { - ++trailing_zeros_intg; - } else { - ++trailing_zeros_frac; - } - } else { - // Time to un-defer any trailing zeros. Intg zeros first. - while (trailing_zeros_intg) { - mantissa = accept_digit(mantissa, 0, &exp_extra, PARSE_DEC_IN_INTG); - --trailing_zeros_intg; - } - while (trailing_zeros_frac) { - mantissa = accept_digit(mantissa, 0, &exp_extra, PARSE_DEC_IN_FRAC); - --trailing_zeros_frac; - } - mantissa = accept_digit(mantissa, dig, &exp_extra, in); - } - } - } else if (in == PARSE_DEC_IN_INTG && dig == '.') { - in = PARSE_DEC_IN_FRAC; - } else if (in != PARSE_DEC_IN_EXP && ((dig | 0x20) == 'e')) { - in = PARSE_DEC_IN_EXP; - if (str < top) { - if (str[0] == '+') { - str++; - } else if (str[0] == '-') { - str++; - exp_neg = true; - } - } - if (str == top) { - goto value_error; - } - } else if (dig == '_') { - continue; - } else { - // unknown character - str--; - break; - } - } - - // work out the exponent - if (exp_neg) { - exp_val = -exp_val; - } - - // apply the exponent, making sure it's not a subnormal value - exp_val += exp_extra + trailing_zeros_intg; - dec_val = (mp_float_t)mantissa; - if (exp_val < SMALL_NORMAL_EXP) { - exp_val -= SMALL_NORMAL_EXP; - dec_val *= SMALL_NORMAL_VAL; - } - - // At this point, we need to multiply the mantissa by its base 10 exponent. If possible, - // we would rather manipulate numbers that have an exact representation in IEEE754. It - // turns out small positive powers of 10 do, whereas small negative powers of 10 don't. - // So in that case, we'll yield a division of exact values rather than a multiplication - // of slightly erroneous values. - if (exp_val < 0 && exp_val >= -EXACT_POWER_OF_10) { - dec_val /= MICROPY_FLOAT_C_FUN(pow)(10, -exp_val); - } else { - dec_val *= MICROPY_FLOAT_C_FUN(pow)(10, exp_val); + str = mp_parse_float_internal(str, top - str, &dec_val); + if (!str) { + goto value_error; } } diff --git a/py/parsenum.h b/py/parsenum.h index f444632d23021..d532cb194a5d8 100644 --- a/py/parsenum.h +++ b/py/parsenum.h @@ -34,6 +34,11 @@ mp_obj_t mp_parse_num_integer(const char *restrict str, size_t len, int base, mp_lexer_t *lex); +#if MICROPY_PY_BUILTINS_FLOAT +mp_large_float_t mp_decimal_exp(mp_large_float_t num, int dec_exp); +const char *mp_parse_float_internal(const char *str, size_t len, mp_float_t *res); +#endif + #if MICROPY_PY_BUILTINS_COMPLEX mp_obj_t mp_parse_num_decimal(const char *str, size_t len, bool allow_imag, bool force_complex, mp_lexer_t *lex); diff --git a/py/persistentcode.c b/py/persistentcode.c index 93f4c33deb4e2..5c3de3174bcd0 100644 --- a/py/persistentcode.c +++ b/py/persistentcode.c @@ -63,6 +63,10 @@ typedef struct _bytecode_prelude_t { uint code_info_size; } bytecode_prelude_t; +#if MICROPY_EMIT_RV32 +#include "py/asmrv32.h" +#endif + #endif // MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE #if MICROPY_PERSISTENT_CODE_LOAD @@ -72,6 +76,8 @@ typedef struct _bytecode_prelude_t { static int read_byte(mp_reader_t *reader); static size_t read_uint(mp_reader_t *reader); +#if MICROPY_EMIT_MACHINE_CODE + #if MICROPY_PERSISTENT_CODE_TRACK_FUN_DATA || MICROPY_PERSISTENT_CODE_TRACK_BSS_RODATA // An mp_obj_list_t that tracks native text/BSS/rodata to prevent the GC from reclaiming them. @@ -86,8 +92,6 @@ static void track_root_pointer(void *ptr) { #endif -#if MICROPY_EMIT_MACHINE_CODE - typedef struct _reloc_info_t { mp_reader_t *reader; mp_module_context_t *context; @@ -422,15 +426,17 @@ static mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co // Relocate and commit code to executable address space reloc_info_t ri = {reader, context, rodata, bss}; + #if MICROPY_PERSISTENT_CODE_TRACK_FUN_DATA + if (native_scope_flags & MP_SCOPE_FLAG_VIPERRELOC) { + // Track the function data memory so it's not reclaimed by the GC. + track_root_pointer(fun_data); + } + #endif #if defined(MP_PLAT_COMMIT_EXEC) void *opt_ri = (native_scope_flags & MP_SCOPE_FLAG_VIPERRELOC) ? &ri : NULL; fun_data = MP_PLAT_COMMIT_EXEC(fun_data, fun_data_len, opt_ri); #else if (native_scope_flags & MP_SCOPE_FLAG_VIPERRELOC) { - #if MICROPY_PERSISTENT_CODE_TRACK_FUN_DATA - // Track the function data memory so it's not reclaimed by the GC. - track_root_pointer(fun_data); - #endif // Do the relocations. mp_native_relocate(&ri, fun_data, (uintptr_t)fun_data); } @@ -480,7 +486,7 @@ void mp_raw_code_load(mp_reader_t *reader, mp_compiled_module_t *cm) { || header[3] > MP_SMALL_INT_BITS) { mp_raise_ValueError(MP_ERROR_TEXT("incompatible .mpy file")); } - if (MPY_FEATURE_DECODE_ARCH(header[2]) != MP_NATIVE_ARCH_NONE) { + if (arch != MP_NATIVE_ARCH_NONE) { if (!MPY_FEATURE_ARCH_TEST(arch)) { if (MPY_FEATURE_ARCH_TEST(MP_NATIVE_ARCH_NONE)) { // On supported ports this can be resolved by enabling feature, eg @@ -492,6 +498,23 @@ void mp_raw_code_load(mp_reader_t *reader, mp_compiled_module_t *cm) { } } + size_t arch_flags = 0; + if (MPY_FEATURE_ARCH_FLAGS_TEST(header[2])) { + #if MICROPY_EMIT_RV32 + arch_flags = read_uint(reader); + + if (MPY_FEATURE_ARCH_TEST(MP_NATIVE_ARCH_RV32IMC)) { + if ((arch_flags & (size_t)asm_rv32_allowed_extensions()) != arch_flags) { + mp_raise_ValueError(MP_ERROR_TEXT("incompatible .mpy file")); + } + } else + #endif + { + (void)arch_flags; + mp_raise_ValueError(MP_ERROR_TEXT("incompatible .mpy file")); + } + } + size_t n_qstr = read_uint(reader); size_t n_obj = read_uint(reader); mp_module_context_alloc_tables(cm->context, n_qstr, n_obj); @@ -513,6 +536,7 @@ void mp_raw_code_load(mp_reader_t *reader, mp_compiled_module_t *cm) { cm->has_native = MPY_FEATURE_DECODE_ARCH(header[2]) != MP_NATIVE_ARCH_NONE; cm->n_qstr = n_qstr; cm->n_obj = n_obj; + cm->arch_flags = arch_flags; #endif // Deregister exception handler and close the reader. @@ -682,7 +706,7 @@ void mp_raw_code_save(mp_compiled_module_t *cm, mp_print_t *print) { byte header[4] = { 'C', MPY_VERSION, - cm->has_native ? MPY_FEATURE_ENCODE_SUB_VERSION(MPY_SUB_VERSION) | MPY_FEATURE_ENCODE_ARCH(MPY_FEATURE_ARCH_DYNAMIC) : 0, + (cm->arch_flags != 0 ? MPY_FEATURE_ARCH_FLAGS : 0) | (cm->has_native ? MPY_FEATURE_ENCODE_SUB_VERSION(MPY_SUB_VERSION) | MPY_FEATURE_ENCODE_ARCH(MPY_FEATURE_ARCH_DYNAMIC) : 0), #if MICROPY_DYNAMIC_COMPILER mp_dynamic_compiler.small_int_bits, #else @@ -691,6 +715,10 @@ void mp_raw_code_save(mp_compiled_module_t *cm, mp_print_t *print) { }; mp_print_bytes(print, header, sizeof(header)); + if (cm->arch_flags) { + mp_print_uint(print, cm->arch_flags); + } + // Number of entries in constant table. mp_print_uint(print, cm->n_qstr); mp_print_uint(print, cm->n_obj); @@ -769,7 +797,7 @@ static void bit_vector_clear(bit_vector_t *self) { static bool bit_vector_is_set(bit_vector_t *self, size_t index) { const size_t bits_size = sizeof(*self->bits) * MP_BITS_PER_BYTE; return index / bits_size < self->alloc - && (self->bits[index / bits_size] & (1 << (index % bits_size))) != 0; + && (self->bits[index / bits_size] & ((uintptr_t)1 << (index % bits_size))) != 0; } static void bit_vector_set(bit_vector_t *self, size_t index) { @@ -780,7 +808,7 @@ static void bit_vector_set(bit_vector_t *self, size_t index) { self->bits = m_renew(uintptr_t, self->bits, self->alloc, new_alloc); self->alloc = new_alloc; } - self->bits[index / bits_size] |= 1 << (index % bits_size); + self->bits[index / bits_size] |= (uintptr_t)1 << (index % bits_size); } typedef struct _mp_opcode_t { diff --git a/py/persistentcode.h b/py/persistentcode.h index 46b474e57f707..85668e608c85b 100644 --- a/py/persistentcode.h +++ b/py/persistentcode.h @@ -50,7 +50,7 @@ // Macros to encode/decode native architecture to/from the feature byte #define MPY_FEATURE_ENCODE_ARCH(arch) ((arch) << 2) -#define MPY_FEATURE_DECODE_ARCH(feat) ((feat) >> 2) +#define MPY_FEATURE_DECODE_ARCH(feat) (((feat) >> 2) & 0x2F) // Define the host architecture #if MICROPY_EMIT_X86 @@ -90,6 +90,10 @@ #define MPY_FILE_HEADER_INT (MPY_VERSION \ | (MPY_FEATURE_ENCODE_SUB_VERSION(MPY_SUB_VERSION) | MPY_FEATURE_ENCODE_ARCH(MPY_FEATURE_ARCH)) << 8) +// Architecture-specific flags are present in the .mpy file +#define MPY_FEATURE_ARCH_FLAGS (0x40) +#define MPY_FEATURE_ARCH_FLAGS_TEST(x) (((x) & MPY_FEATURE_ARCH_FLAGS) == MPY_FEATURE_ARCH_FLAGS) + enum { MP_NATIVE_ARCH_NONE = 0, MP_NATIVE_ARCH_X86, @@ -103,6 +107,7 @@ enum { MP_NATIVE_ARCH_XTENSA, MP_NATIVE_ARCH_XTENSAWIN, MP_NATIVE_ARCH_RV32IMC, + MP_NATIVE_ARCH_RV64IMC, MP_NATIVE_ARCH_DEBUG, // this entry should always be last }; diff --git a/py/profile.c b/py/profile.c index 397d0291f9fad..b5a0c54728c97 100644 --- a/py/profile.c +++ b/py/profile.c @@ -80,7 +80,7 @@ static void frame_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t "", frame, MP_CODE_QSTR_MAP(code->context, 0), - frame->lineno, + (int)frame->lineno, MP_CODE_QSTR_MAP(code->context, prelude->qstr_block_name_idx) ); } diff --git a/py/py.cmake b/py/py.cmake index 6c180ae53e6f2..b99d2d24b5db3 100644 --- a/py/py.cmake +++ b/py/py.cmake @@ -58,6 +58,7 @@ set(MICROPY_SOURCE_PY ${MICROPY_PY_DIR}/mpz.c ${MICROPY_PY_DIR}/nativeglue.c ${MICROPY_PY_DIR}/nlr.c + ${MICROPY_PY_DIR}/nlraarch64.c ${MICROPY_PY_DIR}/nlrmips.c ${MICROPY_PY_DIR}/nlrpowerpc.c ${MICROPY_PY_DIR}/nlrrv32.c diff --git a/py/py.mk b/py/py.mk index c05327bf81d55..21be07c794fb1 100644 --- a/py/py.mk +++ b/py/py.mk @@ -191,6 +191,7 @@ PY_CORE_O_BASENAME = $(addprefix py/,\ objnamedtuple.o \ objrange.o \ objreversed.o \ + objringio.o \ objset.o \ objsingleton.o \ objslice.o \ @@ -289,7 +290,7 @@ $(HEADER_BUILD)/compressed_translations.generated.h: $(PY_SRC)/maketranslationda PY_CORE_O += $(PY_BUILD)/translations-$(TRANSLATION).o # build a list of registered modules for py/objmodule.c. -$(HEADER_BUILD)/moduledefs.h: $(HEADER_BUILD)/moduledefs.collected +$(HEADER_BUILD)/moduledefs.h: $(HEADER_BUILD)/moduledefs.collected $(PY_SRC)/makemoduledefs.py @$(ECHO) "GEN $@" $(Q)$(PYTHON) $(PY_SRC)/makemoduledefs.py $< > $@ diff --git a/py/repl.c b/py/repl.c index c9a20305c79ff..bbfdd2ea6a6e2 100644 --- a/py/repl.c +++ b/py/repl.c @@ -181,8 +181,11 @@ static bool test_qstr(mp_obj_t obj, qstr name) { return dest[0] != MP_OBJ_NULL; } else { // try builtin module - return mp_map_lookup((mp_map_t *)&mp_builtin_module_map, MP_OBJ_NEW_QSTR(name), MP_MAP_LOOKUP) || - mp_map_lookup((mp_map_t *)&mp_builtin_extensible_module_map, MP_OBJ_NEW_QSTR(name), MP_MAP_LOOKUP); + return mp_map_lookup((mp_map_t *)&mp_builtin_module_map, MP_OBJ_NEW_QSTR(name), MP_MAP_LOOKUP) + #if MICROPY_HAVE_REGISTERED_EXTENSIBLE_MODULES + || mp_map_lookup((mp_map_t *)&mp_builtin_extensible_module_map, MP_OBJ_NEW_QSTR(name), MP_MAP_LOOKUP) + #endif + ; } } @@ -237,6 +240,10 @@ static void print_completions(const mp_print_t *print, for (qstr q = q_first; q <= q_last; ++q) { size_t d_len; const char *d_str = (const char *)qstr_data(q, &d_len); + // filter out words that begin with underscore unless there's already a partial match + if (s_len == 0 && d_str[0] == '_') { + continue; + } if (s_len <= d_len && strncmp(s_start, d_str, s_len) == 0) { if (test_qstr(obj, q)) { int gap = (line_len + WORD_SLOT_LEN - 1) / WORD_SLOT_LEN * WORD_SLOT_LEN - line_len; diff --git a/py/runtime.c b/py/runtime.c index 9def98380fa45..81abc3de87570 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -134,7 +134,7 @@ void mp_init(void) { MP_STATE_VM(mp_module_builtins_override_dict) = NULL; #endif - #if MICROPY_PERSISTENT_CODE_TRACK_FUN_DATA || MICROPY_PERSISTENT_CODE_TRACK_BSS_RODATA + #if MICROPY_EMIT_MACHINE_CODE && (MICROPY_PERSISTENT_CODE_TRACK_FUN_DATA || MICROPY_PERSISTENT_CODE_TRACK_BSS_RODATA) MP_STATE_VM(persistent_code_root_pointers) = MP_OBJ_NULL; #endif @@ -446,7 +446,7 @@ mp_obj_t MICROPY_WRAP_MP_BINARY_OP(mp_binary_op)(mp_binary_op_t op, mp_obj_t lhs // Operations that can overflow: // + result always fits in mp_int_t, then handled by SMALL_INT check // - result always fits in mp_int_t, then handled by SMALL_INT check - // * checked explicitly + // * checked explicitly for fit in mp_int_t, then handled by SMALL_INT check // / if lhs=MIN and rhs=-1; result always fits in mp_int_t, then handled by SMALL_INT check // % if lhs=MIN and rhs=-1; result always fits in mp_int_t, then handled by SMALL_INT check // << checked explicitly @@ -505,30 +505,16 @@ mp_obj_t MICROPY_WRAP_MP_BINARY_OP(mp_binary_op)(mp_binary_op_t op, mp_obj_t lhs break; case MP_BINARY_OP_MULTIPLY: case MP_BINARY_OP_INPLACE_MULTIPLY: { - - // If long long type exists and is larger than mp_int_t, then - // we can use the following code to perform overflow-checked multiplication. - // Otherwise (eg in x64 case) we must use mp_small_int_mul_overflow. - #if 0 - // compute result using long long precision - long long res = (long long)lhs_val * (long long)rhs_val; - if (res > MP_SMALL_INT_MAX || res < MP_SMALL_INT_MIN) { - // result overflowed SMALL_INT, so return higher precision integer - return mp_obj_new_int_from_ll(res); - } else { - // use standard precision - lhs_val = (mp_int_t)res; - } - #endif - - if (mp_small_int_mul_overflow(lhs_val, rhs_val)) { + mp_int_t int_res; + if (mp_mul_mp_int_t_overflow(lhs_val, rhs_val, &int_res)) { // use higher precision lhs = mp_obj_new_int_from_ll(lhs_val); goto generic_binary_op; } else { // use standard precision - return MP_OBJ_NEW_SMALL_INT(lhs_val * rhs_val); + lhs_val = int_res; } + break; } case MP_BINARY_OP_FLOOR_DIVIDE: case MP_BINARY_OP_INPLACE_FLOOR_DIVIDE: @@ -568,19 +554,19 @@ mp_obj_t MICROPY_WRAP_MP_BINARY_OP(mp_binary_op)(mp_binary_op_t op, mp_obj_t lhs mp_int_t ans = 1; while (rhs_val > 0) { if (rhs_val & 1) { - if (mp_small_int_mul_overflow(ans, lhs_val)) { + if (mp_mul_mp_int_t_overflow(ans, lhs_val, &ans)) { goto power_overflow; } - ans *= lhs_val; } if (rhs_val == 1) { break; } rhs_val /= 2; - if (mp_small_int_mul_overflow(lhs_val, lhs_val)) { + mp_int_t int_res; + if (mp_mul_mp_int_t_overflow(lhs_val, lhs_val, &int_res)) { goto power_overflow; } - lhs_val *= lhs_val; + lhs_val = int_res; } lhs_val = ans; } @@ -976,7 +962,7 @@ mp_obj_t mp_call_method_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_ob // unpacked items are stored in reverse order into the array pointed to by items // CIRCUITPY-CHANGE: noline -void __attribute__((noinline, )) mp_unpack_sequence(mp_obj_t seq_in, size_t num, mp_obj_t *items) { +MP_NOINLINE void mp_unpack_sequence(mp_obj_t seq_in, size_t num, mp_obj_t *items) { size_t seq_len; if (mp_obj_is_type(seq_in, &mp_type_tuple) || mp_obj_is_type(seq_in, &mp_type_list)) { mp_obj_t *seq_items; @@ -1301,6 +1287,19 @@ void mp_load_method(mp_obj_t base, qstr attr, mp_obj_t *dest) { mp_raise_msg_varg(&mp_type_AttributeError, MP_ERROR_TEXT("type object '%q' has no attribute '%q'"), ((mp_obj_type_t *)MP_OBJ_TO_PTR(base))->name, attr); + #if MICROPY_MODULE___ALL__ && MICROPY_ERROR_REPORTING >= MICROPY_ERROR_REPORTING_DETAILED + } else if (mp_obj_is_type(base, &mp_type_module)) { + // report errors in __all__ as done by CPython + mp_obj_t dest_name[2]; + qstr module_name = MP_QSTR_; + mp_load_method_maybe(base, MP_QSTR___name__, dest_name); + if (mp_obj_is_qstr(dest_name[0])) { + module_name = mp_obj_str_get_qstr(dest_name[0]); + } + mp_raise_msg_varg(&mp_type_AttributeError, + MP_ERROR_TEXT("module '%q' has no attribute '%q'"), + module_name, attr); + #endif } else { mp_raise_msg_varg(&mp_type_AttributeError, MP_ERROR_TEXT("'%s' object has no attribute '%q'"), @@ -1619,7 +1618,7 @@ mp_obj_t mp_import_name(qstr name, mp_obj_t fromlist, mp_obj_t level) { // build args array mp_obj_t args[5]; args[0] = MP_OBJ_NEW_QSTR(name); - args[1] = mp_const_none; // TODO should be globals + args[1] = MP_OBJ_FROM_PTR(mp_globals_get()); // globals of the current context args[2] = mp_const_none; // TODO should be locals args[3] = fromlist; args[4] = level; @@ -1639,20 +1638,17 @@ mp_obj_t mp_import_name(qstr name, mp_obj_t fromlist, mp_obj_t level) { } // CIRCUITPY-CHANGE: noinline -mp_obj_t __attribute__((noinline, )) mp_import_from(mp_obj_t module, qstr name) { +MP_NOINLINE mp_obj_t mp_import_from(mp_obj_t module, qstr name) { DEBUG_printf("import from %p %s\n", module, qstr_str(name)); mp_obj_t dest[2]; mp_load_method_maybe(module, name, dest); - if (dest[1] != MP_OBJ_NULL) { - // Hopefully we can't import bound method from an object - import_error: - mp_raise_msg_varg(&mp_type_ImportError, MP_ERROR_TEXT("can't import name %q"), name); - } - - if (dest[0] != MP_OBJ_NULL) { + // Importing a bound method from a class instance. + return mp_obj_new_bound_meth(dest[0], dest[1]); + } else if (dest[0] != MP_OBJ_NULL) { + // Importing a function or attribute. return dest[0]; } @@ -1685,13 +1681,36 @@ mp_obj_t __attribute__((noinline, )) mp_import_from(mp_obj_t module, qstr name) goto import_error; #endif + +import_error: + mp_raise_msg_varg(&mp_type_ImportError, MP_ERROR_TEXT("can't import name %q"), name); } void mp_import_all(mp_obj_t module) { DEBUG_printf("import all %p\n", module); - // TODO: Support __all__ mp_map_t *map = &mp_obj_module_get_globals(module)->map; + + #if MICROPY_MODULE___ALL__ + mp_map_elem_t *elem = mp_map_lookup(map, MP_OBJ_NEW_QSTR(MP_QSTR___all__), MP_MAP_LOOKUP); + if (elem != NULL) { + // When __all__ is defined, we must explicitly load all specified + // symbols, possibly invoking the module __getattr__ function + size_t len; + mp_obj_t *items; + mp_obj_get_array(elem->value, &len, &items); + for (size_t i = 0; i < len; i++) { + qstr qname = mp_obj_str_get_qstr(items[i]); + mp_obj_t dest[2]; + mp_load_method(module, qname, dest); + mp_store_name(qname, dest[0]); + } + return; + } + #endif + + // By default, the set of public names includes all names found in the module's + // namespace which do not begin with an underscore character ('_') for (size_t i = 0; i < map->alloc; i++) { if (mp_map_slot_is_filled(map, i)) { // Entry in module global scope may be generated programmatically @@ -1747,7 +1766,7 @@ mp_obj_t mp_parse_compile_execute(mp_lexer_t *lex, mp_parse_input_kind_t parse_i #endif // MICROPY_ENABLE_COMPILER // CIRCUITPY-CHANGE: MP_COLD -NORETURN MP_COLD void m_malloc_fail(size_t num_bytes) { +MP_COLD MP_NORETURN void m_malloc_fail(size_t num_bytes) { DEBUG_printf("memory allocation failed, allocating %u bytes\n", (uint)num_bytes); #if MICROPY_ENABLE_GC if (gc_is_locked()) { @@ -1761,29 +1780,29 @@ NORETURN MP_COLD void m_malloc_fail(size_t num_bytes) { #if MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_NONE // CIRCUITPY-CHANGE: MP_COLD -NORETURN MP_COLD void mp_raise_type(const mp_obj_type_t *exc_type) { +MP_COLD MP_NORETURN void mp_raise_type(const mp_obj_type_t *exc_type) { nlr_raise(mp_obj_new_exception(exc_type)); } // CIRCUITPY-CHANGE: MP_COLD -NORETURN MP_COLD void mp_raise_ValueError_no_msg(void) { +MP_COLD MP_NORETURN void mp_raise_ValueError_no_msg(void) { mp_raise_type(&mp_type_ValueError); } // CIRCUITPY-CHANGE: MP_COLD -NORETURN MP_COLD void mp_raise_TypeError_no_msg(void) { +MP_COLD MP_NORETURN void mp_raise_TypeError_no_msg(void) { mp_raise_type(&mp_type_TypeError); } // CIRCUITPY-CHANGE: MP_COLD -NORETURN MP_COLD void mp_raise_NotImplementedError_no_msg(void) { +MP_COLD MP_NORETURN void mp_raise_NotImplementedError_no_msg(void) { mp_raise_type(&mp_type_NotImplementedError); } #else // CIRCUITPY-CHANGE: MP_COLD -NORETURN MP_COLD void mp_raise_msg(const mp_obj_type_t *exc_type, mp_rom_error_text_t msg) { +MP_COLD MP_NORETURN void mp_raise_msg(const mp_obj_type_t *exc_type, mp_rom_error_text_t msg) { if (msg == NULL) { nlr_raise(mp_obj_new_exception(exc_type)); } else { @@ -1792,24 +1811,24 @@ NORETURN MP_COLD void mp_raise_msg(const mp_obj_type_t *exc_type, mp_rom_error_t } // CIRCUITPY-CHANGE: new function for use below. -NORETURN MP_COLD void mp_raise_msg_vlist(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, va_list argptr) { +MP_COLD MP_NORETURN void mp_raise_msg_vlist(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, va_list argptr) { mp_obj_t exception = mp_obj_new_exception_msg_vlist(exc_type, fmt, argptr); nlr_raise(exception); } // CIRCUITPY-CHANGE: MP_COLD and use mp_raise_msg_vlist() -NORETURN MP_COLD void mp_raise_msg_varg(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, ...) { +MP_COLD MP_NORETURN void mp_raise_msg_varg(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, ...) { va_list argptr; va_start(argptr, fmt); mp_raise_msg_vlist(exc_type, fmt, argptr); va_end(argptr); } -NORETURN MP_COLD void mp_raise_ValueError(mp_rom_error_text_t msg) { +MP_COLD MP_NORETURN void mp_raise_ValueError(mp_rom_error_text_t msg) { mp_raise_msg(&mp_type_ValueError, msg); } -NORETURN MP_COLD void mp_raise_msg_str(const mp_obj_type_t *exc_type, const char *msg) { +MP_COLD MP_NORETURN void mp_raise_msg_str(const mp_obj_type_t *exc_type, const char *msg) { if (msg == NULL) { nlr_raise(mp_obj_new_exception(exc_type)); } else { @@ -1818,17 +1837,17 @@ NORETURN MP_COLD void mp_raise_msg_str(const mp_obj_type_t *exc_type, const char } // CIRCUITPY-CHANGE: added -NORETURN MP_COLD void mp_raise_AttributeError(mp_rom_error_text_t msg) { +MP_COLD MP_NORETURN void mp_raise_AttributeError(mp_rom_error_text_t msg) { mp_raise_msg(&mp_type_AttributeError, msg); } // CIRCUITPY-CHANGE: added -NORETURN MP_COLD void mp_raise_RuntimeError(mp_rom_error_text_t msg) { +MP_COLD MP_NORETURN void mp_raise_RuntimeError(mp_rom_error_text_t msg) { mp_raise_msg(&mp_type_RuntimeError, msg); } // CIRCUITPY-CHANGE: added -NORETURN MP_COLD void mp_raise_RuntimeError_varg(mp_rom_error_text_t fmt, ...) { +MP_COLD MP_NORETURN void mp_raise_RuntimeError_varg(mp_rom_error_text_t fmt, ...) { va_list argptr; va_start(argptr, fmt); mp_raise_msg_vlist(&mp_type_RuntimeError, fmt, argptr); @@ -1836,17 +1855,17 @@ NORETURN MP_COLD void mp_raise_RuntimeError_varg(mp_rom_error_text_t fmt, ...) { } // CIRCUITPY-CHANGE: added -NORETURN MP_COLD void mp_raise_ImportError(mp_rom_error_text_t msg) { +MP_COLD MP_NORETURN void mp_raise_ImportError(mp_rom_error_text_t msg) { mp_raise_msg(&mp_type_ImportError, msg); } // CIRCUITPY-CHANGE: added -NORETURN MP_COLD void mp_raise_IndexError(mp_rom_error_text_t msg) { +MP_COLD MP_NORETURN void mp_raise_IndexError(mp_rom_error_text_t msg) { mp_raise_msg(&mp_type_IndexError, msg); } // CIRCUITPY-CHANGE: added -NORETURN MP_COLD void mp_raise_IndexError_varg(mp_rom_error_text_t fmt, ...) { +MP_COLD MP_NORETURN void mp_raise_IndexError_varg(mp_rom_error_text_t fmt, ...) { va_list argptr; va_start(argptr, fmt); mp_raise_msg_vlist(&mp_type_IndexError, fmt, argptr); @@ -1854,7 +1873,7 @@ NORETURN MP_COLD void mp_raise_IndexError_varg(mp_rom_error_text_t fmt, ...) { } // CIRCUITPY-CHANGE: added -NORETURN MP_COLD void mp_raise_ValueError_varg(mp_rom_error_text_t fmt, ...) { +MP_COLD MP_NORETURN void mp_raise_ValueError_varg(mp_rom_error_text_t fmt, ...) { va_list argptr; va_start(argptr, fmt); mp_raise_msg_vlist(&mp_type_ValueError, fmt, argptr); @@ -1862,12 +1881,12 @@ NORETURN MP_COLD void mp_raise_ValueError_varg(mp_rom_error_text_t fmt, ...) { } // CIRCUITPY-CHANGE: MP_COLD -NORETURN MP_COLD void mp_raise_TypeError(mp_rom_error_text_t msg) { +MP_COLD MP_NORETURN void mp_raise_TypeError(mp_rom_error_text_t msg) { mp_raise_msg(&mp_type_TypeError, msg); } // CIRCUITPY-CHANGE: added -NORETURN MP_COLD void mp_raise_TypeError_varg(mp_rom_error_text_t fmt, ...) { +MP_COLD MP_NORETURN void mp_raise_TypeError_varg(mp_rom_error_text_t fmt, ...) { va_list argptr; va_start(argptr, fmt); mp_raise_msg_vlist(&mp_type_TypeError, fmt, argptr); @@ -1875,12 +1894,12 @@ NORETURN MP_COLD void mp_raise_TypeError_varg(mp_rom_error_text_t fmt, ...) { } // CIRCUITPY-CHANGE: added -NORETURN MP_COLD void mp_raise_OSError_msg(mp_rom_error_text_t msg) { +MP_COLD MP_NORETURN void mp_raise_OSError_msg(mp_rom_error_text_t msg) { mp_raise_msg(&mp_type_OSError, msg); } // CIRCUITPY-CHANGE: added -NORETURN MP_COLD void mp_raise_OSError_errno_str(int errno_, mp_obj_t str) { +MP_COLD MP_NORETURN void mp_raise_OSError_errno_str(int errno_, mp_obj_t str) { mp_obj_t args[2] = { MP_OBJ_NEW_SMALL_INT(errno_), str, @@ -1889,7 +1908,7 @@ NORETURN MP_COLD void mp_raise_OSError_errno_str(int errno_, mp_obj_t str) { } // CIRCUITPY-CHANGE: added -NORETURN MP_COLD void mp_raise_OSError_msg_varg(mp_rom_error_text_t fmt, ...) { +MP_COLD MP_NORETURN void mp_raise_OSError_msg_varg(mp_rom_error_text_t fmt, ...) { va_list argptr; va_start(argptr, fmt); mp_raise_msg_vlist(&mp_type_OSError, fmt, argptr); @@ -1897,21 +1916,23 @@ NORETURN MP_COLD void mp_raise_OSError_msg_varg(mp_rom_error_text_t fmt, ...) { } // CIRCUITPY-CHANGE: added -NORETURN MP_COLD void mp_raise_ConnectionError(mp_rom_error_text_t msg) { +MP_COLD MP_NORETURN void mp_raise_ConnectionError(mp_rom_error_text_t msg) { mp_raise_msg(&mp_type_ConnectionError, msg); } // CIRCUITPY-CHANGE: added -NORETURN MP_COLD void mp_raise_BrokenPipeError(void) { +MP_COLD MP_NORETURN void mp_raise_BrokenPipeError(void) { mp_raise_type_arg(&mp_type_BrokenPipeError, MP_OBJ_NEW_SMALL_INT(MP_EPIPE)); } -NORETURN MP_COLD void mp_raise_NotImplementedError(mp_rom_error_text_t msg) { +MP_COLD MP_NORETURN void mp_raise_NotImplementedError(mp_rom_error_text_t msg) { mp_raise_msg(&mp_type_NotImplementedError, msg); } +#endif + // CIRCUITPY-CHANGE: added -NORETURN MP_COLD void mp_raise_NotImplementedError_varg(mp_rom_error_text_t fmt, ...) { +MP_COLD MP_NORETURN void mp_raise_NotImplementedError_varg(mp_rom_error_text_t fmt, ...) { va_list argptr; va_start(argptr, fmt); mp_raise_msg_vlist(&mp_type_NotImplementedError, fmt, argptr); @@ -1919,7 +1940,7 @@ NORETURN MP_COLD void mp_raise_NotImplementedError_varg(mp_rom_error_text_t fmt, } // CIRCUITPY-CHANGE: added -NORETURN MP_COLD void mp_raise_OverflowError_varg(mp_rom_error_text_t fmt, ...) { +MP_COLD MP_NORETURN void mp_raise_OverflowError_varg(mp_rom_error_text_t fmt, ...) { va_list argptr; va_start(argptr, fmt); mp_raise_msg_vlist(&mp_type_OverflowError, fmt, argptr); @@ -1927,11 +1948,11 @@ NORETURN MP_COLD void mp_raise_OverflowError_varg(mp_rom_error_text_t fmt, ...) } // CIRCUITPY-CHANGE: MP_COLD -NORETURN MP_COLD void mp_raise_type_arg(const mp_obj_type_t *exc_type, mp_obj_t arg) { +MP_COLD MP_NORETURN void mp_raise_type_arg(const mp_obj_type_t *exc_type, mp_obj_t arg) { nlr_raise(mp_obj_new_exception_arg1(exc_type, arg)); } -NORETURN void mp_raise_StopIteration(mp_obj_t arg) { +MP_NORETURN void mp_raise_StopIteration(mp_obj_t arg) { if (arg == MP_OBJ_NULL) { mp_raise_type(&mp_type_StopIteration); } else { @@ -1939,7 +1960,7 @@ NORETURN void mp_raise_StopIteration(mp_obj_t arg) { } } -NORETURN void mp_raise_TypeError_int_conversion(mp_const_obj_t arg) { +MP_NORETURN void mp_raise_TypeError_int_conversion(mp_const_obj_t arg) { #if MICROPY_ERROR_REPORTING <= MICROPY_ERROR_REPORTING_TERSE (void)arg; mp_raise_TypeError(MP_ERROR_TEXT("can't convert to int")); @@ -1950,12 +1971,12 @@ NORETURN void mp_raise_TypeError_int_conversion(mp_const_obj_t arg) { } // CIRCUITPY-CHANGE: MP_COLD -NORETURN MP_COLD void mp_raise_OSError(int errno_) { +MP_COLD MP_NORETURN void mp_raise_OSError(int errno_) { mp_raise_type_arg(&mp_type_OSError, MP_OBJ_NEW_SMALL_INT(errno_)); } // CIRCUITPY-CHANGE: MP_COLD -NORETURN MP_COLD void mp_raise_OSError_with_filename(int errno_, const char *filename) { +MP_COLD MP_NORETURN void mp_raise_OSError_with_filename(int errno_, const char *filename) { vstr_t vstr; vstr_init(&vstr, 32); vstr_printf(&vstr, "can't open %s", filename); @@ -1965,13 +1986,12 @@ NORETURN MP_COLD void mp_raise_OSError_with_filename(int errno_, const char *fil } // CIRCUITPY-CHANGE: added -NORETURN MP_COLD void mp_raise_ZeroDivisionError(void) { +MP_COLD MP_NORETURN void mp_raise_ZeroDivisionError(void) { mp_raise_msg(&mp_type_ZeroDivisionError, MP_ERROR_TEXT("division by zero")); } #if MICROPY_STACK_CHECK || MICROPY_ENABLE_PYSTACK // CIRCUITPY-CHANGE: MP_COLD -NORETURN MP_COLD void mp_raise_recursion_depth(void) { +MP_COLD MP_NORETURN void mp_raise_recursion_depth(void) { mp_raise_RuntimeError(MP_ERROR_TEXT("maximum recursion depth exceeded")); } #endif -#endif diff --git a/py/runtime.h b/py/runtime.h index a2d69638ffc38..22ed9c3eaf9b8 100644 --- a/py/runtime.h +++ b/py/runtime.h @@ -37,11 +37,12 @@ #include "supervisor/shared/translate/translate.h" // For use with mp_call_function_1_from_nlr_jump_callback. +// Initialize an nlr_jump_callback_node_call_function_1_t struct for use with +// nlr_push_jump_callback(&ctx.callback, mp_call_function_1_from_nlr_jump_callback); #define MP_DEFINE_NLR_JUMP_CALLBACK_FUNCTION_1(ctx, f, a) \ - nlr_jump_callback_node_call_function_1_t ctx = { \ - .func = (void (*)(void *))(f), \ - .arg = (a), \ - } + nlr_jump_callback_node_call_function_1_t ctx; \ + ctx.func = (void (*)(void *))(f); \ + ctx.arg = (a) typedef enum { MP_VM_RETURN_NORMAL, @@ -58,6 +59,12 @@ typedef enum { MP_ARG_KW_ONLY = 0x200, } mp_arg_flag_t; +typedef enum { + MP_HANDLE_PENDING_CALLBACKS_ONLY, + MP_HANDLE_PENDING_CALLBACKS_AND_EXCEPTIONS, + MP_HANDLE_PENDING_CALLBACKS_AND_CLEAR_EXCEPTIONS, +} mp_handle_pending_behaviour_t; + typedef union _mp_arg_val_t { bool u_bool; mp_int_t u_int; @@ -106,7 +113,14 @@ void mp_sched_keyboard_interrupt(void); #if MICROPY_ENABLE_VM_ABORT void mp_sched_vm_abort(void); #endif -void mp_handle_pending(bool raise_exc); + +void mp_handle_pending_internal(mp_handle_pending_behaviour_t behavior); + +static inline void mp_handle_pending(bool raise_exc) { + mp_handle_pending_internal(raise_exc ? + MP_HANDLE_PENDING_CALLBACKS_AND_EXCEPTIONS : + MP_HANDLE_PENDING_CALLBACKS_AND_CLEAR_EXCEPTIONS); +} #if MICROPY_ENABLE_SCHEDULER void mp_sched_lock(void); @@ -134,7 +148,7 @@ void mp_event_wait_indefinite(void); void mp_event_wait_ms(mp_uint_t timeout_ms); // extra printing method specifically for mp_obj_t's which are integral type -int mp_print_mp_int(const mp_print_t *print, mp_obj_t x, int base, int base_char, int flags, char fill, int width, int prec); +int mp_print_mp_int(const mp_print_t *print, mp_obj_t x, unsigned base, int base_char, int flags, char fill, int width, int prec); void mp_arg_check_num_sig(size_t n_args, size_t n_kw, uint32_t sig); static inline void mp_arg_check_num(size_t n_args, size_t n_kw, size_t n_args_min, size_t n_args_max, bool takes_kw) { @@ -142,11 +156,11 @@ static inline void mp_arg_check_num(size_t n_args, size_t n_kw, size_t n_args_mi } void mp_arg_parse_all(size_t n_pos, const mp_obj_t *pos, mp_map_t *kws, size_t n_allowed, const mp_arg_t *allowed, mp_arg_val_t *out_vals); void mp_arg_parse_all_kw_array(size_t n_pos, size_t n_kw, const mp_obj_t *args, size_t n_allowed, const mp_arg_t *allowed, mp_arg_val_t *out_vals); -NORETURN void mp_arg_error_terse_mismatch(void); -NORETURN void mp_arg_error_unimpl_kw(void); +MP_NORETURN void mp_arg_error_terse_mismatch(void); +MP_NORETURN void mp_arg_error_unimpl_kw(void); // CIRCUITPY-CHANGE: arg validation routines -NORETURN void mp_arg_error_invalid(qstr arg_name); +MP_NORETURN void mp_arg_error_invalid(qstr arg_name); mp_int_t mp_arg_validate_int(mp_int_t i, mp_int_t required_i, qstr arg_name); mp_int_t mp_arg_validate_int_min(mp_int_t i, mp_int_t min, qstr arg_name); mp_int_t mp_arg_validate_int_max(mp_int_t i, mp_int_t j, qstr arg_name); @@ -194,9 +208,16 @@ static inline void mp_thread_init_state(mp_state_thread_t *ts, size_t stack_size ts->gc_lock_depth = 0; // There are no pending jump callbacks or exceptions yet + ts->nlr_top = NULL; ts->nlr_jump_callback_top = NULL; ts->mp_pending_exception = MP_OBJ_NULL; + #if MICROPY_PY_SYS_SETTRACE + ts->prof_trace_callback = MP_OBJ_NULL; + ts->prof_callback_is_executing = false; + ts->current_code_state = NULL; + #endif + // If locals/globals are not given, inherit from main thread if (locals == NULL) { locals = mp_state_ctx.thread.dict_locals; @@ -274,10 +295,10 @@ mp_obj_t mp_import_from(mp_obj_t module, qstr name); void mp_import_all(mp_obj_t module); #if MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_NONE -NORETURN void mp_raise_type(const mp_obj_type_t *exc_type); -NORETURN void mp_raise_ValueError_no_msg(void); -NORETURN void mp_raise_TypeError_no_msg(void); -NORETURN void mp_raise_NotImplementedError_no_msg(void); +MP_NORETURN void mp_raise_type(const mp_obj_type_t *exc_type); +MP_NORETURN void mp_raise_ValueError_no_msg(void); +MP_NORETURN void mp_raise_TypeError_no_msg(void); +MP_NORETURN void mp_raise_NotImplementedError_no_msg(void); #define mp_raise_msg(exc_type, msg) mp_raise_type(exc_type) #define mp_raise_msg_varg(exc_type, ...) mp_raise_type(exc_type) #define mp_raise_ValueError(msg) mp_raise_ValueError_no_msg() @@ -285,46 +306,46 @@ NORETURN void mp_raise_NotImplementedError_no_msg(void); #define mp_raise_NotImplementedError(msg) mp_raise_NotImplementedError_no_msg() #else #define mp_raise_type(exc_type) mp_raise_msg(exc_type, NULL) -NORETURN void mp_raise_msg(const mp_obj_type_t *exc_type, mp_rom_error_text_t msg); -NORETURN void mp_raise_msg_varg(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, ...); -NORETURN void mp_raise_ValueError(mp_rom_error_text_t msg); -NORETURN void mp_raise_TypeError(mp_rom_error_text_t msg); -NORETURN void mp_raise_NotImplementedError(mp_rom_error_text_t msg); +MP_NORETURN void mp_raise_msg(const mp_obj_type_t *exc_type, mp_rom_error_text_t msg); +MP_NORETURN void mp_raise_msg_varg(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, ...); +MP_NORETURN void mp_raise_ValueError(mp_rom_error_text_t msg); +MP_NORETURN void mp_raise_TypeError(mp_rom_error_text_t msg); +MP_NORETURN void mp_raise_NotImplementedError(mp_rom_error_text_t msg); #endif // CIRCUITPY-CHANGE: new mp_raise routines -NORETURN void mp_raise_type_arg(const mp_obj_type_t *exc_type, mp_obj_t arg); -NORETURN void mp_raise_msg(const mp_obj_type_t *exc_type, mp_rom_error_text_t msg); -NORETURN void mp_raise_msg_varg(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt +MP_NORETURN void mp_raise_type_arg(const mp_obj_type_t *exc_type, mp_obj_t arg); +MP_NORETURN void mp_raise_msg(const mp_obj_type_t *exc_type, mp_rom_error_text_t msg); +MP_NORETURN void mp_raise_msg_varg(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt , ...); -NORETURN void mp_raise_msg_vlist(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, va_list argptr); +MP_NORETURN void mp_raise_msg_vlist(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, va_list argptr); // Only use this string version in native mpy files. Otherwise, use the compressed string version. -NORETURN void mp_raise_msg_str(const mp_obj_type_t *exc_type, const char *msg); - -NORETURN void mp_raise_AttributeError(mp_rom_error_text_t msg); -NORETURN void mp_raise_BrokenPipeError(void); -NORETURN void mp_raise_ConnectionError(mp_rom_error_text_t msg); -NORETURN void mp_raise_ImportError(mp_rom_error_text_t msg); -NORETURN void mp_raise_IndexError(mp_rom_error_text_t msg); -NORETURN void mp_raise_IndexError_varg(mp_rom_error_text_t msg, ...); -NORETURN void mp_raise_NotImplementedError(mp_rom_error_text_t msg); -NORETURN void mp_raise_NotImplementedError_varg(mp_rom_error_text_t fmt, ...); -NORETURN void mp_raise_OSError_errno_str(int errno_, mp_obj_t str); -NORETURN void mp_raise_OSError(int errno_); -NORETURN void mp_raise_OSError_msg(mp_rom_error_text_t msg); -NORETURN void mp_raise_OSError_msg_varg(mp_rom_error_text_t fmt, ...); -NORETURN void mp_raise_OSError_with_filename(int errno_, const char *filename); -NORETURN void mp_raise_OverflowError_varg(mp_rom_error_text_t fmt, ...); -NORETURN void mp_raise_recursion_depth(void); -NORETURN void mp_raise_RuntimeError(mp_rom_error_text_t msg); -NORETURN void mp_raise_RuntimeError_varg(mp_rom_error_text_t fmt, ...); -NORETURN void mp_raise_StopIteration(mp_obj_t arg); -NORETURN void mp_raise_TypeError(mp_rom_error_text_t msg); -NORETURN void mp_raise_TypeError_varg(mp_rom_error_text_t fmt, ...); -NORETURN void mp_raise_TypeError_int_conversion(mp_const_obj_t arg); -NORETURN void mp_raise_ValueError(mp_rom_error_text_t msg); -NORETURN void mp_raise_ValueError_varg(mp_rom_error_text_t fmt, ...); -NORETURN void mp_raise_ZeroDivisionError(void); +MP_NORETURN void mp_raise_msg_str(const mp_obj_type_t *exc_type, const char *msg); + +MP_NORETURN void mp_raise_AttributeError(mp_rom_error_text_t msg); +MP_NORETURN void mp_raise_BrokenPipeError(void); +MP_NORETURN void mp_raise_ConnectionError(mp_rom_error_text_t msg); +MP_NORETURN void mp_raise_ImportError(mp_rom_error_text_t msg); +MP_NORETURN void mp_raise_IndexError(mp_rom_error_text_t msg); +MP_NORETURN void mp_raise_IndexError_varg(mp_rom_error_text_t msg, ...); +MP_NORETURN void mp_raise_NotImplementedError(mp_rom_error_text_t msg); +MP_NORETURN void mp_raise_NotImplementedError_varg(mp_rom_error_text_t fmt, ...); +MP_NORETURN void mp_raise_OSError_errno_str(int errno_, mp_obj_t str); +MP_NORETURN void mp_raise_OSError(int errno_); +MP_NORETURN void mp_raise_OSError_msg(mp_rom_error_text_t msg); +MP_NORETURN void mp_raise_OSError_msg_varg(mp_rom_error_text_t fmt, ...); +MP_NORETURN void mp_raise_OSError_with_filename(int errno_, const char *filename); +MP_NORETURN void mp_raise_OverflowError_varg(mp_rom_error_text_t fmt, ...); +MP_NORETURN void mp_raise_recursion_depth(void); +MP_NORETURN void mp_raise_RuntimeError(mp_rom_error_text_t msg); +MP_NORETURN void mp_raise_RuntimeError_varg(mp_rom_error_text_t fmt, ...); +MP_NORETURN void mp_raise_StopIteration(mp_obj_t arg); +MP_NORETURN void mp_raise_TypeError(mp_rom_error_text_t msg); +MP_NORETURN void mp_raise_TypeError_varg(mp_rom_error_text_t fmt, ...); +MP_NORETURN void mp_raise_TypeError_int_conversion(mp_const_obj_t arg); +MP_NORETURN void mp_raise_ValueError(mp_rom_error_text_t msg); +MP_NORETURN void mp_raise_ValueError_varg(mp_rom_error_text_t fmt, ...); +MP_NORETURN void mp_raise_ZeroDivisionError(void); #if MICROPY_BUILTIN_METHOD_CHECK_SELF_ARG #undef mp_check_self diff --git a/py/runtime_utils.c b/py/runtime_utils.c index 98252c3122048..33da558e30f08 100644 --- a/py/runtime_utils.c +++ b/py/runtime_utils.c @@ -53,3 +53,59 @@ mp_obj_t mp_call_function_2_protected(mp_obj_t fun, mp_obj_t arg1, mp_obj_t arg2 return MP_OBJ_NULL; } } + +#if !MICROPY_USE_GCC_MUL_OVERFLOW_INTRINSIC +bool mp_mul_ll_overflow(long long int x, long long int y, long long int *res) { + bool overflow; + + // Check for multiply overflow; see CERT INT32-C + if (x > 0) { // x is positive + if (y > 0) { // x and y are positive + overflow = (x > (LLONG_MAX / y)); + } else { // x positive, y nonpositive + overflow = (y < (LLONG_MIN / x)); + } // x positive, y nonpositive + } else { // x is nonpositive + if (y > 0) { // x is nonpositive, y is positive + overflow = (x < (LLONG_MIN / y)); + } else { // x and y are nonpositive + overflow = (x != 0 && y < (LLONG_MAX / x)); + } // End if x and y are nonpositive + } // End if x is nonpositive + + if (!overflow) { + *res = x * y; + } + + return overflow; +} + +bool mp_mul_mp_int_t_overflow(mp_int_t x, mp_int_t y, mp_int_t *res) { + // Check for multiply overflow; see CERT INT32-C + if (x > 0) { // x is positive + if (y > 0) { // x and y are positive + if (x > (MP_INT_MAX / y)) { + return true; + } + } else { // x positive, y nonpositive + if (y < (MP_INT_MIN / x)) { + return true; + } + } // x positive, y nonpositive + } else { // x is nonpositive + if (y > 0) { // x is nonpositive, y is positive + if (x < (MP_INT_MIN / y)) { + return true; + } + } else { // x and y are nonpositive + if (x != 0 && y < (MP_INT_MAX / x)) { + return true; + } + } // End if x and y are nonpositive + } // End if x is nonpositive + + // Result doesn't overflow + *res = x * y; + return false; +} +#endif diff --git a/py/scheduler.c b/py/scheduler.c index 92e5af6b31b20..ee9cd96b331a6 100644 --- a/py/scheduler.c +++ b/py/scheduler.c @@ -97,17 +97,21 @@ static inline void mp_sched_run_pending(void) { #if MICROPY_SCHEDULER_STATIC_NODES // Run all pending C callbacks. - while (MP_STATE_VM(sched_head) != NULL) { - mp_sched_node_t *node = MP_STATE_VM(sched_head); - MP_STATE_VM(sched_head) = node->next; - if (MP_STATE_VM(sched_head) == NULL) { - MP_STATE_VM(sched_tail) = NULL; - } - mp_sched_callback_t callback = node->callback; - node->callback = NULL; - MICROPY_END_ATOMIC_SECTION(atomic_state); - callback(node); - atomic_state = MICROPY_BEGIN_ATOMIC_SECTION(); + mp_sched_node_t *original_tail = MP_STATE_VM(sched_tail); + if (original_tail != NULL) { + mp_sched_node_t *node; + do { + node = MP_STATE_VM(sched_head); + MP_STATE_VM(sched_head) = node->next; + if (MP_STATE_VM(sched_head) == NULL) { + MP_STATE_VM(sched_tail) = NULL; + } + mp_sched_callback_t callback = node->callback; + node->callback = NULL; + MICROPY_END_ATOMIC_SECTION(atomic_state); + callback(node); + atomic_state = MICROPY_BEGIN_ATOMIC_SECTION(); + } while (node != original_tail); // Don't execute any callbacks scheduled during this run } #endif @@ -218,24 +222,27 @@ MP_REGISTER_ROOT_POINTER(mp_sched_item_t sched_queue[MICROPY_SCHEDULER_DEPTH]); // Called periodically from the VM or from "waiting" code (e.g. sleep) to // process background tasks and pending exceptions (e.g. KeyboardInterrupt). -void mp_handle_pending(bool raise_exc) { +void mp_handle_pending_internal(mp_handle_pending_behaviour_t behavior) { + bool handle_exceptions = (behavior != MP_HANDLE_PENDING_CALLBACKS_ONLY); + bool raise_exceptions = (behavior == MP_HANDLE_PENDING_CALLBACKS_AND_EXCEPTIONS); + // Handle pending VM abort. #if MICROPY_ENABLE_VM_ABORT - if (MP_STATE_VM(vm_abort) && mp_thread_is_main_thread()) { + if (handle_exceptions && MP_STATE_VM(vm_abort) && mp_thread_is_main_thread()) { MP_STATE_VM(vm_abort) = false; - if (raise_exc && nlr_get_abort() != NULL) { + if (raise_exceptions && nlr_get_abort() != NULL) { nlr_jump_abort(); } } #endif // Handle any pending exception. - if (MP_STATE_THREAD(mp_pending_exception) != MP_OBJ_NULL) { + if (handle_exceptions && MP_STATE_THREAD(mp_pending_exception) != MP_OBJ_NULL) { mp_uint_t atomic_state = MICROPY_BEGIN_ATOMIC_SECTION(); mp_obj_t obj = MP_STATE_THREAD(mp_pending_exception); if (obj != MP_OBJ_NULL) { MP_STATE_THREAD(mp_pending_exception) = MP_OBJ_NULL; - if (raise_exc) { + if (raise_exceptions) { MICROPY_END_ATOMIC_SECTION(atomic_state); nlr_raise(obj); } diff --git a/py/sequence.c b/py/sequence.c index ac7ad5368b91e..6bbc62f0f7fb1 100644 --- a/py/sequence.c +++ b/py/sequence.c @@ -34,18 +34,13 @@ #define SWAP(type, var1, var2) { type t = var2; var2 = var1; var1 = t; } // CIRCUITPY-CHANGE: detect sequence overflow -#if __GNUC__ < 5 -// n.b. does not actually detect overflow! -#define __builtin_mul_overflow(a, b, x) (*(x) = (a) * (b), false) -#endif - // Detect when a multiply causes an overflow. size_t mp_seq_multiply_len(size_t item_sz, size_t len) { - size_t new_len; - if (__builtin_mul_overflow(item_sz, len, &new_len)) { + mp_int_t new_len; + if (mp_mul_mp_int_t_overflow(item_sz, len, &new_len)) { mp_raise_msg(&mp_type_OverflowError, MP_ERROR_TEXT("small int overflow")); } - return new_len; + return (size_t)new_len; } // Implements backend of sequence * integer operation. Assumes elements are diff --git a/py/showbc.c b/py/showbc.c index 6913d18c1ca82..792fccd013383 100644 --- a/py/showbc.c +++ b/py/showbc.c @@ -144,17 +144,9 @@ void mp_bytecode_print(const mp_print_t *print, const mp_raw_code_t *rc, size_t mp_uint_t source_line = 1; mp_printf(print, " bc=" INT_FMT " line=" UINT_FMT "\n", bc, source_line); for (const byte *ci = code_info; ci < line_info_top;) { - if ((ci[0] & 0x80) == 0) { - // 0b0LLBBBBB encoding - bc += ci[0] & 0x1f; - source_line += ci[0] >> 5; - ci += 1; - } else { - // 0b1LLLBBBB 0bLLLLLLLL encoding (l's LSB in second byte) - bc += ci[0] & 0xf; - source_line += ((ci[0] << 4) & 0x700) | ci[1]; - ci += 2; - } + mp_code_lineinfo_t decoded = mp_bytecode_decode_lineinfo(&ci); + bc += decoded.bc_increment; + source_line += decoded.line_increment; mp_printf(print, " bc=" INT_FMT " line=" UINT_FMT "\n", bc, source_line); } } diff --git a/py/smallint.c b/py/smallint.c index aa542ca7bf29a..eb99b58667a07 100644 --- a/py/smallint.c +++ b/py/smallint.c @@ -26,32 +26,6 @@ #include "py/smallint.h" -bool mp_small_int_mul_overflow(mp_int_t x, mp_int_t y) { - // Check for multiply overflow; see CERT INT32-C - if (x > 0) { // x is positive - if (y > 0) { // x and y are positive - if (x > (MP_SMALL_INT_MAX / y)) { - return true; - } - } else { // x positive, y nonpositive - if (y < (MP_SMALL_INT_MIN / x)) { - return true; - } - } // x positive, y nonpositive - } else { // x is nonpositive - if (y > 0) { // x is nonpositive, y is positive - if (x < (MP_SMALL_INT_MIN / y)) { - return true; - } - } else { // x and y are nonpositive - if (x != 0 && y < (MP_SMALL_INT_MAX / x)) { - return true; - } - } // End if x and y are nonpositive - } // End if x is nonpositive - return false; -} - mp_int_t mp_small_int_modulo(mp_int_t dividend, mp_int_t divisor) { // Python specs require that mod has same sign as second operand dividend %= divisor; diff --git a/py/smallint.h b/py/smallint.h index 584e0018d1ba3..ec5b0af3b2867 100644 --- a/py/smallint.h +++ b/py/smallint.h @@ -68,7 +68,6 @@ // The number of bits in a MP_SMALL_INT including the sign bit. #define MP_SMALL_INT_BITS (MP_IMAX_BITS(MP_SMALL_INT_MAX) + 1) -bool mp_small_int_mul_overflow(mp_int_t x, mp_int_t y); mp_int_t mp_small_int_modulo(mp_int_t dividend, mp_int_t divisor); mp_int_t mp_small_int_floor_divide(mp_int_t num, mp_int_t denom); diff --git a/py/stream.c b/py/stream.c index fbf7fd878a133..520422cc92fcf 100644 --- a/py/stream.c +++ b/py/stream.c @@ -260,9 +260,14 @@ void mp_stream_write_adaptor(void *self, const char *buf, size_t len) { mp_stream_write(MP_OBJ_FROM_PTR(self), buf, len, MP_STREAM_RW_WRITE); } -static mp_obj_t stream_write_method(size_t n_args, const mp_obj_t *args) { +static mp_obj_t stream_readinto_write_generic(size_t n_args, const mp_obj_t *args, byte flags) { mp_buffer_info_t bufinfo; - mp_get_buffer_raise(args[1], &bufinfo, MP_BUFFER_READ); + mp_get_buffer_raise(args[1], &bufinfo, (flags & MP_STREAM_RW_WRITE) ? MP_BUFFER_READ : MP_BUFFER_WRITE); + + // CPython extension, allow optional maximum length and offset: + // - stream.operation(buf, max_len) + // - stream.operation(buf, off, max_len) + // Similar to https://docs.python.org/3/library/socket.html#socket.socket.recv_into size_t max_len = (size_t)-1; size_t off = 0; if (n_args == 3) { @@ -275,45 +280,31 @@ static mp_obj_t stream_write_method(size_t n_args, const mp_obj_t *args) { } } bufinfo.len -= off; - return mp_stream_write(args[0], (byte *)bufinfo.buf + off, MIN(bufinfo.len, max_len), MP_STREAM_RW_WRITE); + + // Perform the readinto or write operation. + return mp_stream_write(args[0], (byte *)bufinfo.buf + off, MIN(bufinfo.len, max_len), flags); +} + +static mp_obj_t stream_write_method(size_t n_args, const mp_obj_t *args) { + return stream_readinto_write_generic(n_args, args, MP_STREAM_RW_WRITE); } MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_write_obj, 2, 4, stream_write_method); -static mp_obj_t stream_write1_method(mp_obj_t self_in, mp_obj_t arg) { - mp_buffer_info_t bufinfo; - mp_get_buffer_raise(arg, &bufinfo, MP_BUFFER_READ); - return mp_stream_write(self_in, bufinfo.buf, bufinfo.len, MP_STREAM_RW_WRITE | MP_STREAM_RW_ONCE); +static mp_obj_t stream_write1_method(size_t n_args, const mp_obj_t *args) { + return stream_readinto_write_generic(n_args, args, MP_STREAM_RW_WRITE | MP_STREAM_RW_ONCE); } -MP_DEFINE_CONST_FUN_OBJ_2(mp_stream_write1_obj, stream_write1_method); +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_write1_obj, 2, 4, stream_write1_method); static mp_obj_t stream_readinto(size_t n_args, const mp_obj_t *args) { - mp_buffer_info_t bufinfo; - mp_get_buffer_raise(args[1], &bufinfo, MP_BUFFER_WRITE); - - // CPython extension: if 2nd arg is provided, that's max len to read, - // instead of full buffer. Similar to - // https://docs.python.org/3/library/socket.html#socket.socket.recv_into - mp_uint_t len = bufinfo.len; - if (n_args > 2) { - len = mp_obj_get_int(args[2]); - if (len > bufinfo.len) { - len = bufinfo.len; - } - } - - int error; - mp_uint_t out_sz = mp_stream_read_exactly(args[0], bufinfo.buf, len, &error); - if (error != 0) { - if (mp_is_nonblocking_error(error)) { - return mp_const_none; - } - mp_raise_OSError(error); - } else { - return MP_OBJ_NEW_SMALL_INT(out_sz); - } + return stream_readinto_write_generic(n_args, args, MP_STREAM_RW_READ); } MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_readinto_obj, 2, 3, stream_readinto); +static mp_obj_t stream_readinto1(size_t n_args, const mp_obj_t *args) { + return stream_readinto_write_generic(n_args, args, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE); +} +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_readinto1_obj, 2, 3, stream_readinto1); + static mp_obj_t stream_readall(mp_obj_t self_in) { const mp_stream_p_t *stream_p = mp_get_stream(self_in); diff --git a/py/stream.h b/py/stream.h index 5e1cd5d441302..6e60e849b5f48 100644 --- a/py/stream.h +++ b/py/stream.h @@ -87,10 +87,11 @@ typedef struct _mp_stream_p_t { MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_read_obj); MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_read1_obj); MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_readinto_obj); +MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_readinto1_obj); MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_unbuffered_readline_obj); MP_DECLARE_CONST_FUN_OBJ_1(mp_stream_unbuffered_readlines_obj); MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_write_obj); -MP_DECLARE_CONST_FUN_OBJ_2(mp_stream_write1_obj); +MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_write1_obj); MP_DECLARE_CONST_FUN_OBJ_1(mp_stream_close_obj); MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream___exit___obj); MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_seek_obj); diff --git a/py/vm.c b/py/vm.c index 6fd1778261971..860aa36e39792 100644 --- a/py/vm.c +++ b/py/vm.c @@ -205,6 +205,22 @@ static mp_obj_t get_active_exception(mp_exc_stack_t *exc_sp, mp_exc_stack_t *exc return MP_OBJ_NULL; } +#if MICROPY_PY_BUILTINS_SLICE +// This function is marked "no inline" so it doesn't increase the C stack usage of the main VM function. +MP_NOINLINE static mp_obj_t *build_slice_stack_allocated(byte op, mp_obj_t *sp, mp_obj_t step) { + mp_obj_t stop = sp[2]; + mp_obj_t start = sp[1]; + mp_obj_slice_t slice = { .base = { .type = &mp_type_slice }, .start = start, .stop = stop, .step = step }; + if (op == MP_BC_LOAD_SUBSCR) { + SET_TOP(mp_obj_subscr(TOP(), MP_OBJ_FROM_PTR(&slice), MP_OBJ_SENTINEL)); + } else { // MP_BC_STORE_SUBSCR + mp_obj_subscr(TOP(), MP_OBJ_FROM_PTR(&slice), sp[-1]); + sp -= 2; + } + return sp; +} +#endif + // fastn has items in reverse order (fastn[0] is local[0], fastn[-1] is local[1], etc) // sp points to bottom of stack which grows up // returns: @@ -871,9 +887,20 @@ unwind_jump:; // 3-argument slice includes step step = POP(); } - mp_obj_t stop = POP(); - mp_obj_t start = TOP(); - SET_TOP(mp_obj_new_slice(start, stop, step)); + if ((*ip == MP_BC_LOAD_SUBSCR || *ip == MP_BC_STORE_SUBSCR) + && (mp_obj_get_type(sp[-2])->flags & MP_TYPE_FLAG_SUBSCR_ALLOWS_STACK_SLICE)) { + // Fast path optimisation for when the BUILD_SLICE is immediately followed + // by a LOAD/STORE_SUBSCR for an accepting type, to avoid needing to allocate + // the slice on the heap. In some cases (e.g. a[1:3] = x) this can result + // in no allocations at all. We can't do this for instance types because + // the get/set/delattr implementation may keep a reference to the slice. + byte op = *ip++; + sp = build_slice_stack_allocated(op, sp - 2, step); + } else { + mp_obj_t stop = POP(); + mp_obj_t start = TOP(); + SET_TOP(mp_obj_new_slice(start, stop, step)); + } DISPATCH(); } #endif diff --git a/py/vstr.c b/py/vstr.c index 00972edf89796..522509d0d05cf 100644 --- a/py/vstr.c +++ b/py/vstr.c @@ -51,7 +51,7 @@ void vstr_init(vstr_t *vstr, size_t alloc) { // Init the vstr so it allocs exactly enough ram to hold a null-terminated // string of the given length, and set the length. void vstr_init_len(vstr_t *vstr, size_t len) { - // CIRCUITPY-CHANGE + // CIRCUITPY-CHANGE: check for invalid length if (len == SIZE_MAX) { m_malloc_fail(len); } diff --git a/pyproject.toml b/pyproject.toml index 7003a6af9aeae..5f84ca104ac47 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,35 +1,26 @@ -# SPDX-FileCopyrightText: 2024 Scott Shawcroft for Adafruit Industries -# -# SPDX-License-Identifier: MIT - -[tool.setuptools_scm] # can be empty if no extra settings are needed, presence enables setuptools-scm - -# Codespell settings copied from MicroPython - [tool.ruff] -target-version = "py37" - +target-version = "py312" line-length = 99 -# Exclude third-party code from linting and formatting. Ruff doesn't know how to ignore submodules. -# Exclude the following tests: -# repl_: not real python files -# viper_args: uses f(*) +# Include Python source files that don't end with .py +extend-include = [ "tools/cc1" ] +# Exclude third-party code from linting and formatting extend-exclude = [ + # CIRCUITPY-CHANGES "extmod/ulab", "frozen", "lib", "ports/analog/msdk", "ports/atmel-samd/asf4", "ports/broadcom/peripherals", + "ports/espress/tools", "ports/espressif/esp-idf", "ports/espressif/esp-protocols", "ports/raspberrypi/sdk", "ports/silabs/gecko_sdk", "ports/silabs/tools/slc_cli_linux", "ports/stm/st_driver", - "tests/*/repl_*.py", - "tests/micropython/viper_args.py", + "tests/cpydiff/syntax_assign_expr.py", # intentionally incorrect CPython code "tools/adabot", "tools/bitmap_font", "tools/cc1", @@ -40,9 +31,32 @@ extend-exclude = [ ] # Exclude third-party code, and exclude the following tests: # basics: needs careful attention before applying automatic formatting -format.exclude = [ "tests/basics/*.py" ] +# repl_: not real python files +# viper_args: uses f(*) +format.exclude = [ + "tests/*/repl_*.py", + "tests/basics/*.py", + "tests/cmdline/cmd_compile_only_error.py", + "tests/micropython/test_normalize_newlines.py", + "tests/micropython/viper_args.py", +] lint.extend-select = [ "C9", "PLC" ] -lint.ignore = [ +lint.exclude = [ + # Ruff finds Python SyntaxError in these files + "tests/cmdline/cmd_compile_only_error.py", + "tests/cmdline/repl_autocomplete.py", + "tests/cmdline/repl_autocomplete_underscore.py", + "tests/cmdline/repl_autoindent.py", + "tests/cmdline/repl_basic.py", + "tests/cmdline/repl_cont.py", + "tests/cmdline/repl_emacs_keys.py", + "tests/cmdline/repl_paste.py", + "tests/cmdline/repl_words_move.py", + "tests/feature_check/repl_emacs_check.py", + "tests/feature_check/repl_words_move_check.py", + "tests/micropython/viper_args.py", +] +lint.extend-ignore = [ "E401", "E402", "E722", @@ -51,35 +65,27 @@ lint.ignore = [ "F401", "F403", "F405", - "PLC1901", + "PLC0206", + "PLC0415", # conditional imports are common in MicroPython ] # manifest.py files are evaluated with some global names pre-defined lint.per-file-ignores."**/manifest.py" = [ "F821" ] lint.per-file-ignores."ports/**/boards/**/manifest_*.py" = [ "F821" ] -# Exclude all tests from linting (does not apply to formatting). +lint.per-file-ignores."ports/cc3200/tools/uniflash.py" = [ "E711" ] +# Exclude all tests from linting. lint.per-file-ignores."tests/**/*.py" = [ "ALL" ] +# Uses assignment expressions. +lint.per-file-ignores."tests/cpydiff/syntax_assign_expr.py" = [ "F821" ] lint.mccabe.max-complexity = 40 [tool.codespell] count = "" -ignore-regex = '\b[A-Z]{3}\b' -ignore-words-list = "ans,asend,aranges,deques,dout,emac,extint,hsi,iput,mis,notin,numer,ser,shft,synopsys,technic,ure,curren" +ignore-regex = "\\b[A-Z]{3}\\b" +# CIRCUITPY-CHANGE: aranges +ignore-words-list = "ans,aranges,asend,deques,dout,emac,extint,hsi,iput,mis,notin,numer,ser,shft,synopsys,technic,ure,curren" quiet-level = 3 -skip = """ -*/build*,\ -./.git,\ -./drivers/cc3100,\ -./lib,\ -./ports/cc3200/FreeRTOS,\ -./ports/cc3200/bootmgr/sl,\ -./ports/cc3200/hal,\ -./ports/cc3200/simplelink,\ -./ports/cc3200/telnet,\ -./ports/esp32/managed_components,\ -./ports/nrf/drivers/bluetooth/s1*,\ -./ports/stm32/usbhost,\ -./tests,\ -ACKNOWLEDGEMENTS,\ -""" - -# Ruff settings copied from MicroPython +skip = """\ + */build*,./.git,./drivers/cc3100,./lib,./ports/cc3200/FreeRTOS,./ports/cc3200/bootmgr/sl,./ports/cc3200/hal,./ports/c\ + c3200/simplelink,./ports/cc3200/telnet,./ports/esp32/managed_components,./ports/nrf/drivers/bluetooth/s1*,./ports/stm\ + 32/usbhost,./tests,ACKNOWLEDGEMENTS,\ + """ diff --git a/shared-bindings/_bleio/__init__.c b/shared-bindings/_bleio/__init__.c index 3bf8c1a2c9c56..1ec9fd96e766b 100644 --- a/shared-bindings/_bleio/__init__.c +++ b/shared-bindings/_bleio/__init__.c @@ -56,7 +56,7 @@ //| //| MP_DEFINE_BLEIO_EXCEPTION(BluetoothError, Exception) -NORETURN void mp_raise_bleio_BluetoothError(mp_rom_error_text_t fmt, ...) { +MP_NORETURN void mp_raise_bleio_BluetoothError(mp_rom_error_text_t fmt, ...) { va_list argptr; va_start(argptr, fmt); mp_obj_t exception = mp_obj_new_exception_msg_vlist(&mp_type_bleio_BluetoothError, fmt, argptr); @@ -72,7 +72,7 @@ NORETURN void mp_raise_bleio_BluetoothError(mp_rom_error_text_t fmt, ...) { //| //| MP_DEFINE_BLEIO_EXCEPTION(RoleError, bleio_BluetoothError) -NORETURN void mp_raise_bleio_RoleError(mp_rom_error_text_t msg) { +MP_NORETURN void mp_raise_bleio_RoleError(mp_rom_error_text_t msg) { mp_raise_msg(&mp_type_bleio_RoleError, msg); } @@ -83,7 +83,7 @@ NORETURN void mp_raise_bleio_RoleError(mp_rom_error_text_t msg) { //| //| MP_DEFINE_BLEIO_EXCEPTION(SecurityError, bleio_BluetoothError) -NORETURN void mp_raise_bleio_SecurityError(mp_rom_error_text_t fmt, ...) { +MP_NORETURN void mp_raise_bleio_SecurityError(mp_rom_error_text_t fmt, ...) { va_list argptr; va_start(argptr, fmt); mp_obj_t exception = mp_obj_new_exception_msg_vlist(&mp_type_bleio_SecurityError, fmt, argptr); diff --git a/shared-bindings/_bleio/__init__.h b/shared-bindings/_bleio/__init__.h index f7428d2fb2138..0c5a5fec06b29 100644 --- a/shared-bindings/_bleio/__init__.h +++ b/shared-bindings/_bleio/__init__.h @@ -45,9 +45,9 @@ void common_hal_bleio_init(void); extern mp_obj_t bleio_set_adapter(mp_obj_t adapter_obj); -NORETURN void mp_raise_bleio_BluetoothError(mp_rom_error_text_t msg, ...); -NORETURN void mp_raise_bleio_RoleError(mp_rom_error_text_t msg); -NORETURN void mp_raise_bleio_SecurityError(mp_rom_error_text_t msg, ...); +MP_NORETURN void mp_raise_bleio_BluetoothError(mp_rom_error_text_t msg, ...); +MP_NORETURN void mp_raise_bleio_RoleError(mp_rom_error_text_t msg); +MP_NORETURN void mp_raise_bleio_SecurityError(mp_rom_error_text_t msg, ...); bleio_adapter_obj_t *common_hal_bleio_allocate_adapter_or_raise(void); void common_hal_bleio_device_discover_remote_services(mp_obj_t device, mp_obj_t service_uuids_whitelist); diff --git a/shared-bindings/adafruit_pixelbuf/PixelBuf.c b/shared-bindings/adafruit_pixelbuf/PixelBuf.c index cdffbfa627aa1..4144f086b98d6 100644 --- a/shared-bindings/adafruit_pixelbuf/PixelBuf.c +++ b/shared-bindings/adafruit_pixelbuf/PixelBuf.c @@ -24,7 +24,7 @@ #include "extmod/ulab/code/ndarray.h" #endif -static NORETURN void invalid_byteorder(void) { +static MP_NORETURN void invalid_byteorder(void) { mp_arg_error_invalid(MP_QSTR_byteorder); } diff --git a/shared-bindings/alarm/__init__.h b/shared-bindings/alarm/__init__.h index af92b381137c8..37bfd9c51b7cd 100644 --- a/shared-bindings/alarm/__init__.h +++ b/shared-bindings/alarm/__init__.h @@ -24,7 +24,7 @@ extern mp_obj_t common_hal_alarm_light_sleep_until_alarms(size_t n_alarms, const // it will exit idle as if deep sleep was exited extern void common_hal_alarm_set_deep_sleep_alarms(size_t n_alarms, const mp_obj_t *alarms, size_t n_dios, digitalio_digitalinout_obj_t **preserve_dios); -extern NORETURN void common_hal_alarm_enter_deep_sleep(void); +extern MP_NORETURN void common_hal_alarm_enter_deep_sleep(void); // May be used to re-initialize peripherals like GPIO, if the VM reset returned // them to a default state diff --git a/shared-bindings/math/__init__.c b/shared-bindings/math/__init__.c index 54fe53280ca8e..49ca2958d8c96 100644 --- a/shared-bindings/math/__init__.c +++ b/shared-bindings/math/__init__.c @@ -26,7 +26,7 @@ //| """ //| -static NORETURN void math_error(void) { +static MP_NORETURN void math_error(void) { mp_raise_ValueError(MP_ERROR_TEXT("math domain error")); } diff --git a/shared-bindings/memorymonitor/__init__.c b/shared-bindings/memorymonitor/__init__.c index 64d0140ff7930..e364344b363bd 100644 --- a/shared-bindings/memorymonitor/__init__.c +++ b/shared-bindings/memorymonitor/__init__.c @@ -25,7 +25,7 @@ //| MP_DEFINE_MEMORYMONITOR_EXCEPTION(AllocationError, Exception) -NORETURN void mp_raise_memorymonitor_AllocationError(mp_rom_error_text_t fmt, ...) { +MP_NORETURN void mp_raise_memorymonitor_AllocationError(mp_rom_error_text_t fmt, ...) { va_list argptr; va_start(argptr, fmt); mp_obj_t exception = mp_obj_new_exception_msg_vlist(&mp_type_memorymonitor_AllocationError, fmt, argptr); diff --git a/shared-bindings/memorymonitor/__init__.h b/shared-bindings/memorymonitor/__init__.h index 28480bea7b52b..5f141b924d09d 100644 --- a/shared-bindings/memorymonitor/__init__.h +++ b/shared-bindings/memorymonitor/__init__.h @@ -23,4 +23,4 @@ void memorymonitor_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_pr extern const mp_obj_type_t mp_type_memorymonitor_AllocationError; -NORETURN void mp_raise_memorymonitor_AllocationError(mp_rom_error_text_t msg, ...); +MP_NORETURN void mp_raise_memorymonitor_AllocationError(mp_rom_error_text_t msg, ...); diff --git a/shared-bindings/microcontroller/Pin.c b/shared-bindings/microcontroller/Pin.c index e74d54b0770a7..1fcde50e060b6 100644 --- a/shared-bindings/microcontroller/Pin.c +++ b/shared-bindings/microcontroller/Pin.c @@ -175,14 +175,14 @@ void validate_pins(qstr what, uint8_t *pin_nos, mp_int_t max_pins, mp_obj_t seq, } } -NORETURN void raise_ValueError_invalid_pin(void) { +MP_NORETURN void raise_ValueError_invalid_pin(void) { mp_arg_error_invalid(MP_QSTR_pin); } -NORETURN void raise_ValueError_invalid_pins(void) { +MP_NORETURN void raise_ValueError_invalid_pins(void) { mp_arg_error_invalid(MP_QSTR_pins); } -NORETURN void raise_ValueError_invalid_pin_name(qstr pin_name) { +MP_NORETURN void raise_ValueError_invalid_pin_name(qstr pin_name) { mp_raise_ValueError_varg(MP_ERROR_TEXT("Invalid %q pin"), pin_name); } diff --git a/shared-bindings/microcontroller/Pin.h b/shared-bindings/microcontroller/Pin.h index 1245b5f23e4a7..8ddca71bfbbe3 100644 --- a/shared-bindings/microcontroller/Pin.h +++ b/shared-bindings/microcontroller/Pin.h @@ -21,9 +21,9 @@ void validate_no_duplicate_pins(mp_obj_t seq, qstr arg_name); void validate_no_duplicate_pins_2(mp_obj_t seq1, mp_obj_t seq2, qstr arg_name1, qstr arg_name2); void validate_list_is_free_pins(qstr what, const mcu_pin_obj_t **pins_out, mp_int_t max_pins, mp_obj_t seq, uint8_t *count_out); void validate_pins(qstr what, uint8_t *pin_nos, mp_int_t max_pins, mp_obj_t seq, uint8_t *count_out); -NORETURN void raise_ValueError_invalid_pin(void); -NORETURN void raise_ValueError_invalid_pins(void); -NORETURN void raise_ValueError_invalid_pin_name(qstr pin_name); +MP_NORETURN void raise_ValueError_invalid_pin(void); +MP_NORETURN void raise_ValueError_invalid_pins(void); +MP_NORETURN void raise_ValueError_invalid_pin_name(qstr pin_name); void assert_pin_free(const mcu_pin_obj_t *pin); diff --git a/shared-bindings/microcontroller/__init__.h b/shared-bindings/microcontroller/__init__.h index 2a4a973278162..5c5902ff965f5 100644 --- a/shared-bindings/microcontroller/__init__.h +++ b/shared-bindings/microcontroller/__init__.h @@ -20,7 +20,7 @@ extern void common_hal_mcu_disable_interrupts(void); extern void common_hal_mcu_enable_interrupts(void); extern void common_hal_mcu_on_next_reset(mcu_runmode_t runmode); -NORETURN extern void common_hal_mcu_reset(void); +MP_NORETURN extern void common_hal_mcu_reset(void); extern const mp_obj_dict_t mcu_pin_globals; diff --git a/shared-bindings/paralleldisplaybus/__init__.c b/shared-bindings/paralleldisplaybus/__init__.c index 24d968c9e3111..19a381570bada 100644 --- a/shared-bindings/paralleldisplaybus/__init__.c +++ b/shared-bindings/paralleldisplaybus/__init__.c @@ -29,6 +29,3 @@ const mp_obj_module_t paralleldisplaybus_module = { }; MP_REGISTER_MODULE(MP_QSTR_paralleldisplaybus, paralleldisplaybus_module); - -// Remove in CircuitPython 10 -MP_REGISTER_MODULE(MP_QSTR_paralleldisplay, paralleldisplaybus_module); diff --git a/shared-bindings/socketpool/SocketPool.c b/shared-bindings/socketpool/SocketPool.c index 7ae7003d0c364..1fc9e2c3335ca 100644 --- a/shared-bindings/socketpool/SocketPool.c +++ b/shared-bindings/socketpool/SocketPool.c @@ -189,7 +189,7 @@ MP_DEFINE_CONST_OBJ_TYPE( locals_dict, &socketpool_socketpool_locals_dict ); -MP_WEAK NORETURN +MP_WEAK MP_NORETURN void common_hal_socketpool_socketpool_raise_gaierror_noname(void) { vstr_t vstr; mp_print_t print; diff --git a/shared-bindings/socketpool/SocketPool.h b/shared-bindings/socketpool/SocketPool.h index 36035ed00e11a..60d58372b9131 100644 --- a/shared-bindings/socketpool/SocketPool.h +++ b/shared-bindings/socketpool/SocketPool.h @@ -25,6 +25,6 @@ bool socketpool_socket(socketpool_socketpool_obj_t *self, socketpool_socketpool_addressfamily_t family, socketpool_socketpool_sock_t type, int proto, socketpool_socket_obj_t *sock); -NORETURN void common_hal_socketpool_socketpool_raise_gaierror_noname(void); +MP_NORETURN void common_hal_socketpool_socketpool_raise_gaierror_noname(void); mp_obj_t common_hal_socketpool_getaddrinfo_raise(socketpool_socketpool_obj_t *self, const char *host, int port, int family, int type, int proto, int flags); diff --git a/shared-bindings/storage/__init__.h b/shared-bindings/storage/__init__.h index ed3e8d44e4aed..6df604262956a 100644 --- a/shared-bindings/storage/__init__.h +++ b/shared-bindings/storage/__init__.h @@ -16,7 +16,7 @@ void common_hal_storage_umount_path(const char *path); void common_hal_storage_umount_object(mp_obj_t vfs_obj); void common_hal_storage_remount(const char *path, bool readonly, bool disable_concurrent_write_protection); mp_obj_t common_hal_storage_getmount(const char *path); -NORETURN void common_hal_storage_erase_filesystem(bool extended); +MP_NORETURN void common_hal_storage_erase_filesystem(bool extended); bool common_hal_storage_disable_usb_drive(void); bool common_hal_storage_enable_usb_drive(void); diff --git a/shared-bindings/usb/core/__init__.c b/shared-bindings/usb/core/__init__.c index 3907effb8ff25..29dab285084ef 100644 --- a/shared-bindings/usb/core/__init__.c +++ b/shared-bindings/usb/core/__init__.c @@ -30,7 +30,7 @@ //| //| MP_DEFINE_USB_CORE_EXCEPTION(USBError, OSError) -NORETURN void mp_raise_usb_core_USBError(mp_rom_error_text_t fmt, ...) { +MP_NORETURN void mp_raise_usb_core_USBError(mp_rom_error_text_t fmt, ...) { mp_obj_t exception; if (fmt == NULL) { exception = mp_obj_new_exception(&mp_type_usb_core_USBError); @@ -50,7 +50,7 @@ NORETURN void mp_raise_usb_core_USBError(mp_rom_error_text_t fmt, ...) { //| //| MP_DEFINE_USB_CORE_EXCEPTION(USBTimeoutError, usb_core_USBError) -NORETURN void mp_raise_usb_core_USBTimeoutError(void) { +MP_NORETURN void mp_raise_usb_core_USBTimeoutError(void) { mp_raise_type(&mp_type_usb_core_USBTimeoutError); } diff --git a/shared-bindings/usb/core/__init__.h b/shared-bindings/usb/core/__init__.h index 0dc08fae53257..adb8430a6c382 100644 --- a/shared-bindings/usb/core/__init__.h +++ b/shared-bindings/usb/core/__init__.h @@ -25,8 +25,8 @@ void usb_core_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_print_k extern const mp_obj_type_t mp_type_usb_core_USBError; extern const mp_obj_type_t mp_type_usb_core_USBTimeoutError; -NORETURN void mp_raise_usb_core_USBError(mp_rom_error_text_t fmt, ...); -NORETURN void mp_raise_usb_core_USBTimeoutError(void); +MP_NORETURN void mp_raise_usb_core_USBError(mp_rom_error_text_t fmt, ...); +MP_NORETURN void mp_raise_usb_core_USBTimeoutError(void); // Find is all Python object oriented so we don't need a separate common-hal API // for it. It uses the device common-hal instead. diff --git a/shared-bindings/util.h b/shared-bindings/util.h index d53e5c6e8da13..11f0a00db71ea 100644 --- a/shared-bindings/util.h +++ b/shared-bindings/util.h @@ -9,7 +9,7 @@ #include "py/mpprint.h" #include "py/runtime.h" -NORETURN void raise_deinited_error(void); +MP_NORETURN void raise_deinited_error(void); void properties_print_helper(const mp_print_t *print, mp_obj_t self_in, const mp_arg_t *properties, size_t n_properties); void properties_construct_helper(mp_obj_t self_in, const mp_arg_t *args, const mp_arg_val_t *vals, size_t n_properties); bool path_exists(const char *path); diff --git a/shared-module/ssl/SSLSocket.c b/shared-module/ssl/SSLSocket.c index 8911fa2f454d8..fa1de8bb484aa 100644 --- a/shared-module/ssl/SSLSocket.c +++ b/shared-module/ssl/SSLSocket.c @@ -44,7 +44,7 @@ static void mbedtls_debug(void *ctx, int level, const char *file, int line, cons #define DEBUG_PRINT(...) do {} while (0) #endif -static NORETURN void mbedtls_raise_error(int err) { +static MP_NORETURN void mbedtls_raise_error(int err) { // _mbedtls_ssl_send and _mbedtls_ssl_recv (below) turn positive error codes from the // underlying socket into negative codes to pass them through mbedtls. Here we turn them // positive again so they get interpreted as the OSError they really are. The diff --git a/shared-module/struct/__init__.c b/shared-module/struct/__init__.c index 715b504302115..4b6b2b589efc8 100644 --- a/shared-module/struct/__init__.c +++ b/shared-module/struct/__init__.c @@ -14,7 +14,8 @@ #include "shared-bindings/struct/__init__.h" static void struct_validate_format(char fmt) { - #if MICROPY_NONSTANDARD_TYPECODES + #if MICROPY_PY_STRUCT_UNSAFE_TYPECODES + if (fmt == 'S' || fmt == 'O') { mp_raise_RuntimeError(MP_ERROR_TEXT("'S' and 'O' are not supported format types")); } diff --git a/shared/libc/abort_.c b/shared/libc/abort_.c index 3051eae81e0ce..54eab67d3fb49 100644 --- a/shared/libc/abort_.c +++ b/shared/libc/abort_.c @@ -1,7 +1,7 @@ #include -NORETURN void abort_(void); +MP_NORETURN void abort_(void); -NORETURN void abort_(void) { +MP_NORETURN void abort_(void) { mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("abort() called")); } diff --git a/shared/memzip/make-memzip.py b/shared/memzip/make-memzip.py index 92a5d6168bb1d..e406c55a43ca7 100755 --- a/shared/memzip/make-memzip.py +++ b/shared/memzip/make-memzip.py @@ -7,8 +7,6 @@ # This is somewhat like frozen modules in python, but allows arbitrary files # to be used. -from __future__ import print_function - import argparse import os import subprocess diff --git a/shared/netutils/trace.c b/shared/netutils/trace.c index a6dfb42c28f00..24af4d5ca315f 100644 --- a/shared/netutils/trace.c +++ b/shared/netutils/trace.c @@ -56,7 +56,7 @@ static const char *ethertype_str(uint16_t type) { } void netutils_ethernet_trace(const mp_print_t *print, size_t len, const uint8_t *buf, unsigned int flags) { - mp_printf(print, "[% 8d] ETH%cX len=%u", mp_hal_ticks_ms(), flags & NETUTILS_TRACE_IS_TX ? 'T' : 'R', len); + mp_printf(print, "[% 8u] ETH%cX len=%u", (unsigned)mp_hal_ticks_ms(), flags & NETUTILS_TRACE_IS_TX ? 'T' : 'R', len); mp_printf(print, " dst=%02x:%02x:%02x:%02x:%02x:%02x", buf[0], buf[1], buf[2], buf[3], buf[4], buf[5]); mp_printf(print, " src=%02x:%02x:%02x:%02x:%02x:%02x", buf[6], buf[7], buf[8], buf[9], buf[10], buf[11]); diff --git a/shared/runtime/gchelper_generic.c b/shared/runtime/gchelper_generic.c index 464aeaa9981de..230a24440050f 100644 --- a/shared/runtime/gchelper_generic.c +++ b/shared/runtime/gchelper_generic.c @@ -42,6 +42,7 @@ // stack already by the caller. #if defined(__x86_64__) +// CIRCUITPY-CHANGE: use __asm__ instead of asm static void gc_helper_get_regs(gc_helper_regs_t arr) { register long rbx __asm__ ("rbx"); register long rbp __asm__ ("rbp"); diff --git a/shared/runtime/interrupt_char.c b/shared/runtime/interrupt_char.c index 5cec1988f41b3..1f2702017190f 100644 --- a/shared/runtime/interrupt_char.c +++ b/shared/runtime/interrupt_char.c @@ -31,6 +31,7 @@ #if MICROPY_KBD_EXCEPTION +// CIRCUITPY-CHANGE #ifdef __ZEPHYR__ #include diff --git a/shared/runtime/interrupt_char.h b/shared/runtime/interrupt_char.h index c4a465456a8d1..44fd4b45a6121 100644 --- a/shared/runtime/interrupt_char.h +++ b/shared/runtime/interrupt_char.h @@ -29,6 +29,7 @@ // CIRCUITPY-CHANGE #include +// CIRCUITPY-CHANGE #ifdef __ZEPHYR__ #include diff --git a/shared/runtime/mpirq.c b/shared/runtime/mpirq.c deleted file mode 100644 index 6111b9b10cfe4..0000000000000 --- a/shared/runtime/mpirq.c +++ /dev/null @@ -1,136 +0,0 @@ -/* - * This file is part of the MicroPython project, http://micropython.org/ - * - * The MIT License (MIT) - * - * Copyright (c) 2015 Daniel Campora - * 2018 Tobias Badertscher - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include - -#include "py/runtime.h" -#include "py/gc.h" -#include "shared/runtime/mpirq.h" - -#if MICROPY_ENABLE_SCHEDULER - -/****************************************************************************** - DECLARE PUBLIC DATA - ******************************************************************************/ - -const mp_arg_t mp_irq_init_args[] = { - { MP_QSTR_handler, MP_ARG_OBJ, {.u_rom_obj = MP_ROM_NONE} }, - { MP_QSTR_trigger, MP_ARG_INT, {.u_int = 0} }, - { MP_QSTR_hard, MP_ARG_BOOL, {.u_bool = false} }, -}; - -/****************************************************************************** - DECLARE PRIVATE DATA - ******************************************************************************/ - -/****************************************************************************** - DEFINE PUBLIC FUNCTIONS - ******************************************************************************/ - -mp_irq_obj_t *mp_irq_new(const mp_irq_methods_t *methods, mp_obj_t parent) { - mp_irq_obj_t *self = m_new0(mp_irq_obj_t, 1); - mp_irq_init(self, methods, parent); - return self; -} - -void mp_irq_init(mp_irq_obj_t *self, const mp_irq_methods_t *methods, mp_obj_t parent) { - self->base.type = &mp_irq_type; - self->methods = (mp_irq_methods_t *)methods; - self->parent = parent; - self->handler = mp_const_none; - self->ishard = false; -} - -void mp_irq_handler(mp_irq_obj_t *self) { - if (self->handler != mp_const_none) { - if (self->ishard) { - // When executing code within a handler we must lock the scheduler to - // prevent any scheduled callbacks from running, and lock the GC to - // prevent any memory allocations. - mp_sched_lock(); - gc_lock(); - nlr_buf_t nlr; - if (nlr_push(&nlr) == 0) { - mp_call_function_1(self->handler, self->parent); - nlr_pop(); - } else { - // Uncaught exception; disable the callback so that it doesn't run again - self->methods->trigger(self->parent, 0); - self->handler = mp_const_none; - mp_printf(MICROPY_ERROR_PRINTER, "Uncaught exception in IRQ callback handler\n"); - mp_obj_print_exception(MICROPY_ERROR_PRINTER, MP_OBJ_FROM_PTR(nlr.ret_val)); - } - gc_unlock(); - mp_sched_unlock(); - } else { - // Schedule call to user function - mp_sched_schedule(self->handler, self->parent); - } - } -} - -/******************************************************************************/ -// MicroPython bindings - -static mp_obj_t mp_irq_flags(mp_obj_t self_in) { - mp_irq_obj_t *self = MP_OBJ_TO_PTR(self_in); - return mp_obj_new_int(self->methods->info(self->parent, MP_IRQ_INFO_FLAGS)); -} -static MP_DEFINE_CONST_FUN_OBJ_1(mp_irq_flags_obj, mp_irq_flags); - -static mp_obj_t mp_irq_trigger(size_t n_args, const mp_obj_t *args) { - mp_irq_obj_t *self = MP_OBJ_TO_PTR(args[0]); - mp_obj_t ret_obj = mp_obj_new_int(self->methods->info(self->parent, MP_IRQ_INFO_TRIGGERS)); - if (n_args == 2) { - // Set trigger - self->methods->trigger(self->parent, mp_obj_get_int(args[1])); - } - return ret_obj; -} -static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_irq_trigger_obj, 1, 2, mp_irq_trigger); - -static mp_obj_t mp_irq_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const mp_obj_t *args) { - mp_arg_check_num(n_args, n_kw, 0, 0, false); - mp_irq_handler(MP_OBJ_TO_PTR(self_in)); - return mp_const_none; -} - -static const mp_rom_map_elem_t mp_irq_locals_dict_table[] = { - { MP_ROM_QSTR(MP_QSTR_flags), MP_ROM_PTR(&mp_irq_flags_obj) }, - { MP_ROM_QSTR(MP_QSTR_trigger), MP_ROM_PTR(&mp_irq_trigger_obj) }, -}; -static MP_DEFINE_CONST_DICT(mp_irq_locals_dict, mp_irq_locals_dict_table); - -MP_DEFINE_CONST_OBJ_TYPE( - mp_irq_type, - MP_QSTR_irq, - MP_TYPE_FLAG_NONE, - call, mp_irq_call, - locals_dict, &mp_irq_locals_dict - ); - -#endif // MICROPY_ENABLE_SCHEDULER diff --git a/shared/runtime/mpirq.h b/shared/runtime/mpirq.h deleted file mode 100644 index dd423c0101137..0000000000000 --- a/shared/runtime/mpirq.h +++ /dev/null @@ -1,82 +0,0 @@ -/* - * This file is part of the MicroPython project, http://micropython.org/ - * - * The MIT License (MIT) - * - * Copyright (c) 2015 Daniel Campora - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MICROPY_INCLUDED_LIB_UTILS_MPIRQ_H -#define MICROPY_INCLUDED_LIB_UTILS_MPIRQ_H - -#include "py/runtime.h" - -/****************************************************************************** - DEFINE CONSTANTS - ******************************************************************************/ - -enum { - MP_IRQ_ARG_INIT_handler = 0, - MP_IRQ_ARG_INIT_trigger, - MP_IRQ_ARG_INIT_hard, - MP_IRQ_ARG_INIT_NUM_ARGS, -}; - -/****************************************************************************** - DEFINE TYPES - ******************************************************************************/ - -typedef mp_uint_t (*mp_irq_trigger_fun_t)(mp_obj_t self, mp_uint_t trigger); -typedef mp_uint_t (*mp_irq_info_fun_t)(mp_obj_t self, mp_uint_t info_type); - -enum { - MP_IRQ_INFO_FLAGS, - MP_IRQ_INFO_TRIGGERS, -}; - -typedef struct _mp_irq_methods_t { - mp_irq_trigger_fun_t trigger; - mp_irq_info_fun_t info; -} mp_irq_methods_t; - -typedef struct _mp_irq_obj_t { - mp_obj_base_t base; - mp_irq_methods_t *methods; - mp_obj_t parent; - mp_obj_t handler; - bool ishard; -} mp_irq_obj_t; - -/****************************************************************************** - DECLARE EXPORTED DATA - ******************************************************************************/ - -extern const mp_arg_t mp_irq_init_args[]; -extern const mp_obj_type_t mp_irq_type; - -/****************************************************************************** - DECLARE PUBLIC FUNCTIONS - ******************************************************************************/ - -mp_irq_obj_t *mp_irq_new(const mp_irq_methods_t *methods, mp_obj_t parent); -void mp_irq_init(mp_irq_obj_t *self, const mp_irq_methods_t *methods, mp_obj_t parent); -void mp_irq_handler(mp_irq_obj_t *self); - -#endif // MICROPY_INCLUDED_LIB_UTILS_MPIRQ_H diff --git a/shared/runtime/pyexec.c b/shared/runtime/pyexec.c index 06680ff2dd125..bc5f89bdc81ad 100644 --- a/shared/runtime/pyexec.c +++ b/shared/runtime/pyexec.c @@ -38,15 +38,12 @@ #include "py/gc.h" #include "py/frozenmod.h" #include "py/mphal.h" -#if MICROPY_HW_ENABLE_USB -#include "irq.h" -#include "usb.h" -#endif #include "shared/readline/readline.h" #include "shared/runtime/pyexec.h" +#include "extmod/modplatform.h" #include "genhdr/mpversion.h" -// CIRCUITPY-CHANGE: atexit support +// CIRCUITPY-CHANGE: add atexit support #if CIRCUITPY_ATEXIT #include "shared-module/atexit/__init__.h" #endif @@ -87,57 +84,71 @@ static int parse_compile_execute(const void *source, mp_parse_input_kind_t input nlr_buf_t nlr; nlr.ret_val = NULL; if (nlr_push(&nlr) == 0) { - // CIRCUITPY-CHANGE - mp_obj_t module_fun = mp_const_none; - // CIRCUITPY-CHANGE + #if MICROPY_PYEXEC_ENABLE_VM_ABORT + nlr_set_abort(&nlr); + #endif + + // CIRCUITPY-CHANGE: move declaration for easier handling of atexit #if. + // Also make it possible to determine if module_fun was set. + mp_obj_t module_fun = NULL; + + // CIRCUITPY-CHANGE: add atexit support #if CIRCUITPY_ATEXIT - if (!(exec_flags & EXEC_FLAG_SOURCE_IS_ATEXIT)) + if (exec_flags & EXEC_FLAG_SOURCE_IS_ATEXIT) { + atexit_callback_t *callback = (atexit_callback_t *)source; + mp_call_function_n_kw(callback->func, callback->n_pos, callback->n_kw, callback->args); + } else #endif - // CIRCUITPY-CHANGE: multiple code changes - { - #if MICROPY_MODULE_FROZEN_MPY - if (exec_flags & EXEC_FLAG_SOURCE_IS_RAW_CODE) { - // source is a raw_code object, create the function - const mp_frozen_module_t *frozen = source; - mp_module_context_t *ctx = m_new_obj(mp_module_context_t); - ctx->module.globals = mp_globals_get(); - ctx->constants = frozen->constants; - module_fun = mp_make_function_from_proto_fun(frozen->proto_fun, ctx, NULL); - } else - #endif - { - #if MICROPY_ENABLE_COMPILER - mp_lexer_t *lex; - if (exec_flags & EXEC_FLAG_SOURCE_IS_VSTR) { - const vstr_t *vstr = source; - lex = mp_lexer_new_from_str_len(MP_QSTR__lt_stdin_gt_, vstr->buf, vstr->len, 0); - } else if (exec_flags & EXEC_FLAG_SOURCE_IS_READER) { - lex = mp_lexer_new(MP_QSTR__lt_stdin_gt_, *(mp_reader_t *)source); - } else if (exec_flags & EXEC_FLAG_SOURCE_IS_FILENAME) { - lex = mp_lexer_new_from_file(qstr_from_str(source)); - } else { - lex = (mp_lexer_t *)source; - } - // source is a lexer, parse and compile the script - qstr source_name = lex->source_name; - // CIRCUITPY-CHANGE - #if MICROPY_PY___FILE__ - if (input_kind == MP_PARSE_FILE_INPUT) { - mp_store_global(MP_QSTR___file__, MP_OBJ_NEW_QSTR(source_name)); - } - #endif - mp_parse_tree_t parse_tree = mp_parse(lex, input_kind); - module_fun = mp_compile(&parse_tree, source_name, exec_flags & EXEC_FLAG_IS_REPL); - #else - mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("script compilation not supported")); - #endif + #if MICROPY_MODULE_FROZEN_MPY + if (exec_flags & EXEC_FLAG_SOURCE_IS_RAW_CODE) { + // source is a raw_code object, create the function + const mp_frozen_module_t *frozen = source; + mp_module_context_t *ctx = m_new_obj(mp_module_context_t); + ctx->module.globals = mp_globals_get(); + ctx->constants = frozen->constants; + module_fun = mp_make_function_from_proto_fun(frozen->proto_fun, ctx, NULL); + } else + #endif + { + #if MICROPY_ENABLE_COMPILER + mp_lexer_t *lex; + if (exec_flags & EXEC_FLAG_SOURCE_IS_VSTR) { + const vstr_t *vstr = source; + lex = mp_lexer_new_from_str_len(MP_QSTR__lt_stdin_gt_, vstr->buf, vstr->len, 0); + } else if (exec_flags & EXEC_FLAG_SOURCE_IS_READER) { + lex = mp_lexer_new(MP_QSTR__lt_stdin_gt_, *(mp_reader_t *)source); + } else if (exec_flags & EXEC_FLAG_SOURCE_IS_FILENAME) { + lex = mp_lexer_new_from_file(qstr_from_str(source)); + } else { + lex = (mp_lexer_t *)source; } - - // If the code was loaded from a file, collect any garbage before running. + // source is a lexer, parse and compile the script + qstr source_name = lex->source_name; + #if MICROPY_MODULE___FILE__ if (input_kind == MP_PARSE_FILE_INPUT) { - gc_collect(); + mp_store_global(MP_QSTR___file__, MP_OBJ_NEW_QSTR(source_name)); } + #endif + mp_parse_tree_t parse_tree = mp_parse(lex, input_kind); + #if defined(MICROPY_UNIX_COVERAGE) + // allow to print the parse tree in the coverage build + if (mp_verbose_flag >= 3) { + printf("----------------\n"); + mp_parse_node_print(&mp_plat_print, parse_tree.root, 0); + printf("----------------\n"); + } + #endif + module_fun = mp_compile(&parse_tree, source_name, exec_flags & EXEC_FLAG_IS_REPL); + #else + mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("script compilation not supported")); + #endif + } + + // CIRCUITPY-CHANGE: garbage collect after loading + // If the code was loaded from a file, collect any garbage before running. + if (input_kind == MP_PARSE_FILE_INPUT) { + gc_collect(); } // execute code @@ -147,22 +158,19 @@ static int parse_compile_execute(const void *source, mp_parse_input_kind_t input #if MICROPY_REPL_INFO start = mp_hal_ticks_ms(); #endif - // CIRCUITPY-CHANGE - #if CIRCUITPY_ATEXIT - if (exec_flags & EXEC_FLAG_SOURCE_IS_ATEXIT) { - atexit_callback_t *callback = (atexit_callback_t *)source; - mp_call_function_n_kw(callback->func, callback->n_pos, callback->n_kw, callback->args); - } else + #if MICROPY_PYEXEC_COMPILE_ONLY + if (!mp_compile_only) #endif - // CIRCUITPY-CHANGE - if (module_fun != mp_const_none) { - mp_call_function_0(module_fun); + { + // CIRCUITPY-CHANGE: if atexit function was called, there is nothing to call. + if (module_fun != NULL) { + mp_call_function_0(module_fun); + } } mp_hal_set_interrupt_char(-1); // disable interrupt mp_handle_pending(true); // handle any pending exceptions (and any callbacks) nlr_pop(); - // CIRCUITPY-CHANGE - ret = 0; + ret = PYEXEC_NORMAL_EXIT; if (exec_flags & EXEC_FLAG_PRINT_EOF) { mp_hal_stdout_tx_strn("\x04", 1); } @@ -181,36 +189,65 @@ static int parse_compile_execute(const void *source, mp_parse_input_kind_t input mp_hal_stdout_tx_strn("\x04", 1); } - // check for SystemExit - - // CIRCUITPY-CHANGE + // CIRCUITPY-CHANGE: Name and use some values. // nlr.ret_val is an exception object. mp_obj_t exception_obj = (mp_obj_t)nlr.ret_val; + const mp_obj_type_t *exception_obj_type = mp_obj_get_type(exception_obj); - if (mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(mp_obj_get_type(exception_obj)), MP_OBJ_FROM_PTR(&mp_type_SystemExit))) { - // at the moment, the value of SystemExit is unused + #if MICROPY_PYEXEC_ENABLE_VM_ABORT + if (nlr.ret_val == NULL) { // abort + ret = PYEXEC_ABORT; + } else + #endif + + if (mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(exception_obj_type), MP_OBJ_FROM_PTR(&mp_type_SystemExit))) { // system exit + #if MICROPY_PYEXEC_ENABLE_EXIT_CODE_HANDLING + // None is an exit value of 0; an int is its value; anything else is 1 + mp_obj_t val = mp_obj_exception_get_value(MP_OBJ_FROM_PTR(nlr.ret_val)); + if (val != mp_const_none) { + if (mp_obj_is_int(val)) { + ret = (int)mp_obj_int_get_truncated(val); + } else { + mp_obj_print_helper(MICROPY_ERROR_PRINTER, val, PRINT_STR); + mp_print_str(MICROPY_ERROR_PRINTER, "\n"); + ret = PYEXEC_UNHANDLED_EXCEPTION; + } + } else { + ret = PYEXEC_NORMAL_EXIT; + } + // Set PYEXEC_FORCED_EXIT flag so REPL knows to exit + ret |= PYEXEC_FORCED_EXIT; + #else ret = PYEXEC_FORCED_EXIT; - // CIRCUITPY-CHANGE + #endif + // CIRCUITPY-CHANGE: support DeepSleepRequest #if CIRCUITPY_ALARM - } else if (mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(mp_obj_get_type(exception_obj)), MP_OBJ_FROM_PTR(&mp_type_DeepSleepRequest))) { + } else if (mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(exception_obj_type), MP_OBJ_FROM_PTR(&mp_type_DeepSleepRequest))) { ret = PYEXEC_DEEP_SLEEP; #endif + // CIRCUITPY-CHANGE: support ReloadException } else if (exception_obj == MP_OBJ_FROM_PTR(&MP_STATE_VM(mp_reload_exception))) { ret = PYEXEC_RELOAD; - } else { - mp_obj_print_exception(&mp_plat_print, exception_obj); - ret = PYEXEC_EXCEPTION; + } else { // other exception + mp_obj_print_exception(&mp_plat_print, MP_OBJ_FROM_PTR(nlr.ret_val)); + ret = PYEXEC_UNHANDLED_EXCEPTION; + #if MICROPY_PYEXEC_ENABLE_EXIT_CODE_HANDLING + if (mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(((mp_obj_base_t *)nlr.ret_val)->type), MP_OBJ_FROM_PTR(&mp_type_KeyboardInterrupt))) { // keyboard interrupt + ret = PYEXEC_KEYBOARD_INTERRUPT; + } + #endif } - } + + // CIRCUITPY_CHANGE: Fill in result out argument. if (result != NULL) { result->return_code = ret; - #if CIRCUITPY_ALARM // Don't set the exception object if we exited for deep sleep. - if (ret != 0 && ret != PYEXEC_DEEP_SLEEP) { - #else - if (ret != 0) { + if (ret != 0 + #if CIRCUITPY_ALARM + && ret != PYEXEC_DEEP_SLEEP #endif + ) { mp_obj_t return_value = (mp_obj_t)nlr.ret_val; result->exception = return_value; result->exception_line = -1; @@ -227,6 +264,10 @@ static int parse_compile_execute(const void *source, mp_parse_input_kind_t input } } + #if MICROPY_PYEXEC_ENABLE_VM_ABORT + nlr_set_abort(NULL); + #endif + #if MICROPY_REPL_INFO // display debugging info if wanted if ((exec_flags & EXEC_FLAG_ALLOW_DEBUGGING) && repl_display_debugging_info) { @@ -484,7 +525,10 @@ static int pyexec_friendly_repl_process_char(int c) { // CIRCUITPY-CHANGE: print CircuitPython-style banner. mp_hal_stdout_tx_str(MICROPY_FULL_VERSION_INFO); mp_hal_stdout_tx_str("\r\n"); + #if MICROPY_PY_BUILTINS_HELP + // CIRCUITPY-CHANGE: don't print help info // mp_hal_stdout_tx_str("Type \"help()\" for more information.\r\n"); + #endif goto input_restart; } else if (ret == CHAR_CTRL_C) { // break @@ -570,10 +614,21 @@ MP_REGISTER_ROOT_POINTER(vstr_t * repl_line); #else // MICROPY_REPL_EVENT_DRIVEN +// CIRCUITPY-CHANGE: avoid warnings +#if !defined(MICROPY_HAL_HAS_STDIO_MODE_SWITCH) || !MICROPY_HAL_HAS_STDIO_MODE_SWITCH +// If the port doesn't need any stdio mode switching calls then provide trivial ones. +static inline void mp_hal_stdio_mode_raw(void) { +} +static inline void mp_hal_stdio_mode_orig(void) { +} +#endif + int pyexec_raw_repl(void) { vstr_t line; vstr_init(&line, 32); + mp_hal_stdio_mode_raw(); + raw_repl_reset: mp_hal_stdout_tx_str("raw REPL; CTRL-B to exit\r\n"); @@ -587,6 +642,7 @@ int pyexec_raw_repl(void) { if (vstr_len(&line) == 2 && vstr_str(&line)[0] == CHAR_CTRL_E) { int ret = do_reader_stdin(vstr_str(&line)[1]); if (ret & PYEXEC_FORCED_EXIT) { + mp_hal_stdio_mode_orig(); return ret; } vstr_reset(&line); @@ -599,6 +655,7 @@ int pyexec_raw_repl(void) { mp_hal_stdout_tx_str("\r\n"); vstr_clear(&line); pyexec_mode_kind = PYEXEC_MODE_FRIENDLY_REPL; + mp_hal_stdio_mode_orig(); return 0; } else if (c == CHAR_CTRL_C) { // clear line @@ -619,14 +676,18 @@ int pyexec_raw_repl(void) { // exit for a soft reset mp_hal_stdout_tx_str("\r\n"); vstr_clear(&line); + mp_hal_stdio_mode_orig(); return PYEXEC_FORCED_EXIT; } + // Switch to original terminal mode to execute code, eg to support keyboard interrupt (SIGINT). + mp_hal_stdio_mode_orig(); // CIRCUITPY-CHANGE: add last arg, handle reload int ret = parse_compile_execute(&line, MP_PARSE_FILE_INPUT, EXEC_FLAG_PRINT_EOF | EXEC_FLAG_SOURCE_IS_VSTR, NULL); if (ret & (PYEXEC_FORCED_EXIT | PYEXEC_RELOAD)) { return ret; } + mp_hal_stdio_mode_raw(); } } @@ -634,6 +695,8 @@ int pyexec_friendly_repl(void) { vstr_t line; vstr_init(&line, 32); + mp_hal_stdio_mode_raw(); + friendly_repl_reset: // CIRCUITPY-CHANGE: CircuitPython-style banner. mp_hal_stdout_tx_str("\r\n"); @@ -661,20 +724,6 @@ int pyexec_friendly_repl(void) { for (;;) { input_restart: - - #if MICROPY_HW_ENABLE_USB - if (usb_vcp_is_enabled()) { - // If the user gets to here and interrupts are disabled then - // they'll never see the prompt, traceback etc. The USB REPL needs - // interrupts to be enabled or no transfers occur. So we try to - // do the user a favor and re-enable interrupts. - if (query_irq() == IRQ_STATE_DISABLED) { - enable_irq(IRQ_STATE_ENABLED); - mp_hal_stdout_tx_str("MPY: enabling IRQs\r\n"); - } - } - #endif - // If the GC is locked at this point there is no way out except a reset, // so force the GC to be unlocked to help the user debug what went wrong. if (MP_STATE_THREAD(gc_lock_depth) != 0) { @@ -705,6 +754,7 @@ int pyexec_friendly_repl(void) { mp_hal_stdout_tx_str("\r\n"); vstr_clear(&line); pyexec_mode_kind = PYEXEC_MODE_RAW_REPL; + mp_hal_stdio_mode_orig(); return 0; } else if (ret == CHAR_CTRL_B) { // reset friendly REPL @@ -718,6 +768,7 @@ int pyexec_friendly_repl(void) { // exit for a soft reset mp_hal_stdout_tx_str("\r\n"); vstr_clear(&line); + mp_hal_stdio_mode_orig(); return PYEXEC_FORCED_EXIT; } else if (ret == CHAR_CTRL_E) { // paste mode @@ -762,11 +813,14 @@ int pyexec_friendly_repl(void) { } } + // Switch to original terminal mode to execute code, eg to support keyboard interrupt (SIGINT). + mp_hal_stdio_mode_orig(); // CIRCUITPY-CHANGE: add last arg ret = parse_compile_execute(&line, parse_input_kind, EXEC_FLAG_ALLOW_DEBUGGING | EXEC_FLAG_IS_REPL | EXEC_FLAG_SOURCE_IS_VSTR, NULL); if (ret & (PYEXEC_FORCED_EXIT | PYEXEC_RELOAD)) { return ret; } + mp_hal_stdio_mode_raw(); } } diff --git a/shared/runtime/pyexec.h b/shared/runtime/pyexec.h index 55a3003791cb2..9d1ef2a3cb029 100644 --- a/shared/runtime/pyexec.h +++ b/shared/runtime/pyexec.h @@ -51,6 +51,17 @@ extern pyexec_mode_kind_t pyexec_mode_kind; #define PYEXEC_DEEP_SLEEP (0x400) #define PYEXEC_RELOAD (0x800) +#if MICROPY_PYEXEC_ENABLE_EXIT_CODE_HANDLING +#define PYEXEC_NORMAL_EXIT (0) +#define PYEXEC_UNHANDLED_EXCEPTION (1) +#define PYEXEC_KEYBOARD_INTERRUPT (128 + 2) // same as SIG INT exit code +#define PYEXEC_ABORT (128 + 9) // same as SIG KILL exit code +#else +#define PYEXEC_NORMAL_EXIT (1) +#define PYEXEC_UNHANDLED_EXCEPTION (0) +#define PYEXEC_ABORT PYEXEC_FORCED_EXIT +#endif + int pyexec_raw_repl(void); int pyexec_friendly_repl(void); // CIRCUITPY-CHANGE: result out argument diff --git a/shared/timeutils/timeutils.c b/shared/timeutils/timeutils.c index 4282a0178dd6e..0c6916e06ddd6 100644 --- a/shared/timeutils/timeutils.c +++ b/shared/timeutils/timeutils.c @@ -29,12 +29,27 @@ #include "shared/timeutils/timeutils.h" -// LEAPOCH corresponds to 2000-03-01, which is a mod-400 year, immediately -// after Feb 29. We calculate seconds as a signed integer relative to that. +// To maintain reasonable compatibility with CPython on embedded systems, +// and avoid breaking anytime soon, timeutils functions are required to +// work properly between 1970 and 2099 on all ports. // -// Our timebase is relative to 2000-01-01. - -#define LEAPOCH ((31 + 29) * 86400) +// During that period of time, leap years occur every 4 years without +// exception, so we can keep the code short for 32 bit machines. + +// The last leap day before the required period is Feb 29, 1968. +// This is the number of days to add to get to that date. +#define PREV_LEAP_DAY ((mp_uint_t)(365 + 366 - (31 + 29))) +#define PREV_LEAP_YEAR 1968 + +// On ports where either MICROPY_TIME_SUPPORT_Y2100_AND_BEYOND or +// MICROPY_TIME_SUPPORT_Y1969_AND_BEFORE is enabled, we include extra +// code to support leap years outside of the 'easy' period. +// Computation is then made based on 1600 (a mod-400 year). +// This is the number of days between 1600 and 1968. +#define QC_BASE_DAY 134409 +#define QC_LEAP_YEAR 1600 +// This is the number of leap days between 1600 and 1970 +#define QC_LEAP_DAYS 89 #define DAYS_PER_400Y (365 * 400 + 97) #define DAYS_PER_100Y (365 * 100 + 24) @@ -42,8 +57,20 @@ static const uint16_t days_since_jan1[] = { 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365 }; +// type used internally to count small integers relative to epoch +// (using uint when possible produces smaller code on some platforms) +#if MICROPY_TIME_SUPPORT_Y1969_AND_BEFORE +typedef mp_int_t relint_t; +#else +typedef mp_uint_t relint_t; +#endif + bool timeutils_is_leap_year(mp_uint_t year) { + #if MICROPY_TIME_SUPPORT_Y2100_AND_BEYOND || MICROPY_TIME_SUPPORT_Y1969_AND_BEFORE return (year % 4 == 0 && year % 100 != 0) || year % 400 == 0; + #else + return year % 4 == 0; + #endif } // month is one based @@ -65,67 +92,67 @@ mp_uint_t timeutils_year_day(mp_uint_t year, mp_uint_t month, mp_uint_t date) { return yday; } -void timeutils_seconds_since_2000_to_struct_time(mp_uint_t t, timeutils_struct_time_t *tm) { - // The following algorithm was adapted from musl's __secs_to_tm and adapted - // for differences in MicroPython's timebase. - - mp_int_t seconds = t - LEAPOCH; +void timeutils_seconds_since_1970_to_struct_time(timeutils_timestamp_t seconds, timeutils_struct_time_t *tm) { + // The following algorithm was inspired from musl's __secs_to_tm + // and simplified to reduce code footprint in the simple case - mp_int_t days = seconds / 86400; + relint_t days = seconds / 86400; seconds %= 86400; + #if MICROPY_TIME_SUPPORT_Y1969_AND_BEFORE if (seconds < 0) { seconds += 86400; days -= 1; } + #endif tm->tm_hour = seconds / 3600; tm->tm_min = seconds / 60 % 60; tm->tm_sec = seconds % 60; - mp_int_t wday = (days + 2) % 7; // Mar 1, 2000 was a Wednesday (2) + relint_t wday = (days + 3) % 7; // Jan 1, 1970 was a Thursday (3) + #if MICROPY_TIME_SUPPORT_Y1969_AND_BEFORE if (wday < 0) { wday += 7; } + #endif tm->tm_wday = wday; - mp_int_t qc_cycles = days / DAYS_PER_400Y; + days += PREV_LEAP_DAY; + + #if MICROPY_TIME_SUPPORT_Y2100_AND_BEYOND || MICROPY_TIME_SUPPORT_Y1969_AND_BEFORE + // rebase day to the oldest supported date (=> always positive) + mp_uint_t base_year = QC_LEAP_YEAR; + days += QC_BASE_DAY; + mp_uint_t qc_cycles = days / DAYS_PER_400Y; days %= DAYS_PER_400Y; - if (days < 0) { - days += DAYS_PER_400Y; - qc_cycles--; - } - mp_int_t c_cycles = days / DAYS_PER_100Y; + mp_uint_t c_cycles = days / DAYS_PER_100Y; if (c_cycles == 4) { c_cycles--; } days -= (c_cycles * DAYS_PER_100Y); - - mp_int_t q_cycles = days / DAYS_PER_4Y; + #else + mp_uint_t base_year = PREV_LEAP_YEAR; + mp_uint_t qc_cycles = 0; + mp_uint_t c_cycles = 0; + #endif + + mp_uint_t q_cycles = days / DAYS_PER_4Y; + #if MICROPY_TIME_SUPPORT_Y2100_AND_BEYOND || MICROPY_TIME_SUPPORT_Y1969_AND_BEFORE if (q_cycles == 25) { q_cycles--; } + #endif days -= q_cycles * DAYS_PER_4Y; - mp_int_t years = days / 365; + relint_t years = days / 365; if (years == 4) { years--; } days -= (years * 365); - /* We will compute tm_yday at the very end - mp_int_t leap = !years && (q_cycles || !c_cycles); - - tm->tm_yday = days + 31 + 28 + leap; - if (tm->tm_yday >= 365 + leap) { - tm->tm_yday -= 365 + leap; - } - - tm->tm_yday++; // Make one based - */ - - tm->tm_year = 2000 + years + 4 * q_cycles + 100 * c_cycles + 400 * qc_cycles; + tm->tm_year = base_year + years + 4 * q_cycles + 100 * c_cycles + 400 * qc_cycles; // Note: days_in_month[0] corresponds to March - static const int8_t days_in_month[] = {31, 30, 31, 30, 31, 31, 30, 31, 30, 31, 31, 29}; + static const uint8_t days_in_month[] = {31, 30, 31, 30, 31, 31, 30, 31, 30, 31, 31, 29}; mp_int_t month; for (month = 0; days_in_month[month] <= days; month++) { @@ -144,21 +171,28 @@ void timeutils_seconds_since_2000_to_struct_time(mp_uint_t t, timeutils_struct_t } // returns the number of seconds, as an integer, since 2000-01-01 -mp_uint_t timeutils_seconds_since_2000(mp_uint_t year, mp_uint_t month, +timeutils_timestamp_t timeutils_seconds_since_1970(mp_uint_t year, mp_uint_t month, mp_uint_t date, mp_uint_t hour, mp_uint_t minute, mp_uint_t second) { - return - second - + minute * 60 - + hour * 3600 - + (timeutils_year_day(year, month, date) - 1 - + ((year - 2000 + 3) / 4) // add a day each 4 years starting with 2001 - - ((year - 2000 + 99) / 100) // subtract a day each 100 years starting with 2001 - + ((year - 2000 + 399) / 400) // add a day each 400 years starting with 2001 - ) * 86400 - + (year - 2000) * 31536000; + #if MICROPY_TIME_SUPPORT_Y2100_AND_BEYOND || MICROPY_TIME_SUPPORT_Y1969_AND_BEFORE + mp_uint_t ref_year = QC_LEAP_YEAR; + #else + mp_uint_t ref_year = PREV_LEAP_YEAR; + #endif + timeutils_timestamp_t res; + res = ((relint_t)year - 1970) * 365; + res += (year - (ref_year + 1)) / 4; // add a day each 4 years + #if MICROPY_TIME_SUPPORT_Y2100_AND_BEYOND || MICROPY_TIME_SUPPORT_Y1969_AND_BEFORE + res -= (year - (ref_year + 1)) / 100; // subtract a day each 100 years + res += (year - (ref_year + 1)) / 400; // add a day each 400 years + res -= QC_LEAP_DAYS; + #endif + res += timeutils_year_day(year, month, date) - 1; + res *= 86400; + res += hour * 3600 + minute * 60 + second; + return res; } -mp_uint_t timeutils_mktime_2000(mp_uint_t year, mp_int_t month, mp_int_t mday, +timeutils_timestamp_t timeutils_mktime_1970(mp_uint_t year, mp_int_t month, mp_int_t mday, mp_int_t hours, mp_int_t minutes, mp_int_t seconds) { // Normalize the tuple. This allows things like: @@ -211,12 +245,16 @@ mp_uint_t timeutils_mktime_2000(mp_uint_t year, mp_int_t month, mp_int_t mday, year++; } } - return timeutils_seconds_since_2000(year, month, mday, hours, minutes, seconds); + return timeutils_seconds_since_1970(year, month, mday, hours, minutes, seconds); } // Calculate the weekday from the date. // The result is zero based with 0 = Monday. // by Michael Keith and Tom Craver, 1990. int timeutils_calc_weekday(int y, int m, int d) { - return ((d += m < 3 ? y-- : y - 2, 23 * m / 9 + d + 4 + y / 4 - y / 100 + y / 400) + 6) % 7; + return ((d += m < 3 ? y-- : y - 2, 23 * m / 9 + d + 4 + y / 4 + #if MICROPY_TIME_SUPPORT_Y2100_AND_BEYOND || MICROPY_TIME_SUPPORT_Y1969_AND_BEFORE + - y / 100 + y / 400 + #endif + ) + 6) % 7; } diff --git a/shared/timeutils/timeutils.h b/shared/timeutils/timeutils.h index b41903d3b2777..35356b462aafc 100644 --- a/shared/timeutils/timeutils.h +++ b/shared/timeutils/timeutils.h @@ -27,12 +27,23 @@ #ifndef MICROPY_INCLUDED_LIB_TIMEUTILS_TIMEUTILS_H #define MICROPY_INCLUDED_LIB_TIMEUTILS_TIMEUTILS_H -// CIRCUITPY-CHANGE: MICROPY_EPOCH_IS_1970 -#include "mpconfigport.h" +#include "py/obj.h" +#if MICROPY_PY_BUILTINS_FLOAT && MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_DOUBLE +#include // required for trunc() +#endif + +// `timeutils_timestamp_t` is the type used internally by timeutils to +// represent timestamps, and is always referenced to 1970. +// It may not match the platform-specific `mp_timestamp_t`. +#if MICROPY_TIME_SUPPORT_Y2100_AND_BEYOND || MICROPY_TIME_SUPPORT_Y1969_AND_BEFORE +typedef long long timeutils_timestamp_t; +#else +typedef mp_uint_t timeutils_timestamp_t; +#endif // The number of seconds between 1970/1/1 and 2000/1/1 is calculated using: // time.mktime((2000,1,1,0,0,0,0,0,0)) - time.mktime((1970,1,1,0,0,0,0,0,0)) -#define TIMEUTILS_SECONDS_1970_TO_2000 (946684800ULL) +#define TIMEUTILS_SECONDS_1970_TO_2000 (946684800LL) typedef struct _timeutils_struct_time_t { uint16_t tm_year; // i.e. 2014 @@ -48,66 +59,116 @@ typedef struct _timeutils_struct_time_t { bool timeutils_is_leap_year(mp_uint_t year); mp_uint_t timeutils_days_in_month(mp_uint_t year, mp_uint_t month); mp_uint_t timeutils_year_day(mp_uint_t year, mp_uint_t month, mp_uint_t date); +int timeutils_calc_weekday(int y, int m, int d); -void timeutils_seconds_since_2000_to_struct_time(mp_uint_t t, +void timeutils_seconds_since_1970_to_struct_time(timeutils_timestamp_t t, timeutils_struct_time_t *tm); // Year is absolute, month/date are 1-based, hour/minute/second are 0-based. -mp_uint_t timeutils_seconds_since_2000(mp_uint_t year, mp_uint_t month, +timeutils_timestamp_t timeutils_seconds_since_1970(mp_uint_t year, mp_uint_t month, mp_uint_t date, mp_uint_t hour, mp_uint_t minute, mp_uint_t second); // Year is absolute, month/mday are 1-based, hours/minutes/seconds are 0-based. -mp_uint_t timeutils_mktime_2000(mp_uint_t year, mp_int_t month, mp_int_t mday, +timeutils_timestamp_t timeutils_mktime_1970(mp_uint_t year, mp_int_t month, mp_int_t mday, mp_int_t hours, mp_int_t minutes, mp_int_t seconds); +static inline mp_timestamp_t timeutils_obj_get_timestamp(mp_obj_t o_in) { + #if MICROPY_PY_BUILTINS_FLOAT && MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_DOUBLE + mp_float_t val = mp_obj_get_float(o_in); + return (mp_timestamp_t)MICROPY_FLOAT_C_FUN(trunc)(val); + #elif MICROPY_TIMESTAMP_IMPL == MICROPY_TIMESTAMP_IMPL_UINT + return mp_obj_get_uint(o_in); + #else + return mp_obj_get_ll(o_in); + #endif +} + +static inline mp_obj_t timeutils_obj_from_timestamp(mp_timestamp_t t) { + #if MICROPY_TIMESTAMP_IMPL == MICROPY_TIMESTAMP_IMPL_UINT + return mp_obj_new_int_from_uint(t); + #else + return mp_obj_new_int_from_ll(t); + #endif +} + +static inline void timeutils_seconds_since_2000_to_struct_time(mp_timestamp_t t, timeutils_struct_time_t *tm) { + timeutils_seconds_since_1970_to_struct_time((timeutils_timestamp_t)(t + TIMEUTILS_SECONDS_1970_TO_2000), tm); +} + +// Year is absolute, month/date are 1-based, hour/minute/second are 0-based. +static inline mp_timestamp_t timeutils_seconds_since_2000(mp_uint_t year, mp_uint_t month, mp_uint_t date, + mp_uint_t hour, mp_uint_t minute, mp_uint_t second) { + return (mp_timestamp_t)timeutils_seconds_since_1970(year, month, date, hour, minute, second) - TIMEUTILS_SECONDS_1970_TO_2000; +} + +// Year is absolute, month/mday are 1-based, hours/minutes/seconds are 0-based. +static inline mp_timestamp_t timeutils_mktime_2000(mp_uint_t year, mp_int_t month, mp_int_t mday, + mp_int_t hours, mp_int_t minutes, mp_int_t seconds) { + return (mp_timestamp_t)timeutils_mktime_1970(year, month, mday, hours, minutes, seconds) - TIMEUTILS_SECONDS_1970_TO_2000; +} + + // Select the Epoch used by the port. #if MICROPY_EPOCH_IS_1970 -static inline void timeutils_seconds_since_epoch_to_struct_time(uint64_t t, timeutils_struct_time_t *tm) { - // TODO this will give incorrect results for dates before 2000/1/1 - timeutils_seconds_since_2000_to_struct_time((mp_uint_t)(t - TIMEUTILS_SECONDS_1970_TO_2000), tm); +static inline void timeutils_seconds_since_epoch_to_struct_time(mp_timestamp_t t, timeutils_struct_time_t *tm) { + timeutils_seconds_since_1970_to_struct_time(t, tm); +} + +// Year is absolute, month/date are 1-based, hour/minute/second are 0-based. +static inline mp_timestamp_t timeutils_seconds_since_epoch(mp_uint_t year, mp_uint_t month, mp_uint_t date, + mp_uint_t hour, mp_uint_t minute, mp_uint_t second) { + return timeutils_seconds_since_1970(year, month, date, hour, minute, second); } // Year is absolute, month/mday are 1-based, hours/minutes/seconds are 0-based. -static inline uint64_t timeutils_mktime(mp_uint_t year, mp_int_t month, mp_int_t mday, mp_int_t hours, mp_int_t minutes, mp_int_t seconds) { - return timeutils_mktime_2000(year, month, mday, hours, minutes, seconds) + TIMEUTILS_SECONDS_1970_TO_2000; +static inline mp_timestamp_t timeutils_mktime(mp_uint_t year, mp_int_t month, mp_int_t mday, + mp_int_t hours, mp_int_t minutes, mp_int_t seconds) { + return timeutils_mktime_1970(year, month, mday, hours, minutes, seconds); } -// Year is absolute, month/date are 1-based, hour/minute/second are 0-based. -static inline uint64_t timeutils_seconds_since_epoch(mp_uint_t year, mp_uint_t month, - mp_uint_t date, mp_uint_t hour, mp_uint_t minute, mp_uint_t second) { - // TODO this will give incorrect results for dates before 2000/1/1 - return timeutils_seconds_since_2000(year, month, date, hour, minute, second) + TIMEUTILS_SECONDS_1970_TO_2000; +static inline mp_timestamp_t timeutils_seconds_since_epoch_from_nanoseconds_since_1970(int64_t ns) { + return (mp_timestamp_t)(ns / 1000000000LL); } -static inline mp_uint_t timeutils_seconds_since_epoch_from_nanoseconds_since_1970(uint64_t ns) { - return (mp_uint_t)(ns / 1000000000ULL); +static inline int64_t timeutils_seconds_since_epoch_to_nanoseconds_since_1970(mp_timestamp_t s) { + return (int64_t)s * 1000000000LL; } -static inline uint64_t timeutils_nanoseconds_since_epoch_to_nanoseconds_since_1970(uint64_t ns) { +static inline int64_t timeutils_nanoseconds_since_epoch_to_nanoseconds_since_1970(int64_t ns) { return ns; } #else // Epoch is 2000 -#define timeutils_seconds_since_epoch_to_struct_time timeutils_seconds_since_2000_to_struct_time -#define timeutils_seconds_since_epoch timeutils_seconds_since_2000 -#define timeutils_mktime timeutils_mktime_2000 +static inline void timeutils_seconds_since_epoch_to_struct_time(mp_timestamp_t t, timeutils_struct_time_t *tm) { + timeutils_seconds_since_2000_to_struct_time(t, tm); +} -static inline uint64_t timeutils_seconds_since_epoch_to_nanoseconds_since_1970(mp_uint_t s) { - return ((uint64_t)s + TIMEUTILS_SECONDS_1970_TO_2000) * 1000000000ULL; +// Year is absolute, month/date are 1-based, hour/minute/second are 0-based. +static inline mp_timestamp_t timeutils_seconds_since_epoch(mp_uint_t year, mp_uint_t month, mp_uint_t date, + mp_uint_t hour, mp_uint_t minute, mp_uint_t second) { + return timeutils_seconds_since_2000(year, month, date, hour, minute, second); } -static inline mp_uint_t timeutils_seconds_since_epoch_from_nanoseconds_since_1970(uint64_t ns) { - return ns / 1000000000ULL - TIMEUTILS_SECONDS_1970_TO_2000; +// Year is absolute, month/mday are 1-based, hours/minutes/seconds are 0-based. +static inline mp_timestamp_t timeutils_mktime(mp_uint_t year, mp_int_t month, mp_int_t mday, + mp_int_t hours, mp_int_t minutes, mp_int_t seconds) { + return timeutils_mktime_2000(year, month, mday, hours, minutes, seconds); +} + +static inline mp_timestamp_t timeutils_seconds_since_epoch_from_nanoseconds_since_1970(int64_t ns) { + return (mp_timestamp_t)(ns / 1000000000LL - TIMEUTILS_SECONDS_1970_TO_2000); +} + +static inline int64_t timeutils_seconds_since_epoch_to_nanoseconds_since_1970(mp_timestamp_t s) { + return ((int64_t)s + TIMEUTILS_SECONDS_1970_TO_2000) * 1000000000LL; } static inline int64_t timeutils_nanoseconds_since_epoch_to_nanoseconds_since_1970(int64_t ns) { - return ns + TIMEUTILS_SECONDS_1970_TO_2000 * 1000000000ULL; + return ns + TIMEUTILS_SECONDS_1970_TO_2000 * 1000000000LL; } #endif -int timeutils_calc_weekday(int y, int m, int d); - #endif // MICROPY_INCLUDED_LIB_TIMEUTILS_TIMEUTILS_H diff --git a/supervisor/port.h b/supervisor/port.h index 7e2463032f74f..f0e13e167cce1 100644 --- a/supervisor/port.h +++ b/supervisor/port.h @@ -24,13 +24,13 @@ extern uint32_t _ebss; safe_mode_t port_init(void); // Reset the microcontroller completely. -void reset_cpu(void) NORETURN; +void reset_cpu(void) MP_NORETURN; // Reset the microcontroller state. void reset_port(void); // Reset to the bootloader -void reset_to_bootloader(void) NORETURN; +void reset_to_bootloader(void) MP_NORETURN; // Get stack limit address uint32_t *port_stack_get_limit(void); diff --git a/supervisor/shared/safe_mode.h b/supervisor/shared/safe_mode.h index 8f4037cbe20c3..87f65d867ea62 100644 --- a/supervisor/shared/safe_mode.h +++ b/supervisor/shared/safe_mode.h @@ -38,6 +38,6 @@ void set_safe_mode(safe_mode_t safe_mode); safe_mode_t wait_for_safe_mode_reset(void); void safe_mode_on_next_reset(safe_mode_t reason); -void reset_into_safe_mode(safe_mode_t reason) NORETURN; +void reset_into_safe_mode(safe_mode_t reason) MP_NORETURN; void print_safe_mode_message(safe_mode_t reason); diff --git a/tests/README.md b/tests/README.md index 21e14eee5e128..534e7e0a05972 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,6 +1,7 @@ # MicroPython Test Suite -This directory contains tests for most parts of MicroPython. +This directory contains tests for most parts of MicroPython. To run it you will need +CPython 3.8.2 or newer, which is used to validate MicroPython's behaviour. To run all stable tests, run the "run-tests.py" script in this directory. By default that will run the test suite against the unix port of MicroPython. @@ -67,16 +68,14 @@ for a full list of command line options. ### Benchmarking a target -To run tests on a firmware target using `pyboard.py`, run the command line like +To run tests on a firmware target using a serial port, run the command line like this: ``` -./run-perfbench.py -p -d /dev/ttyACM0 168 100 +./run-perfbench.py -t /dev/ttyACM0 168 100 ``` -* `-p` indicates running on a remote target via pyboard.py, not the host. -* `-d PORTNAME` is the serial port, `/dev/ttyACM0` is the default if not - provided. +* `-t PORTNAME` is the serial port to use (and it supports shorthand like `a0`). * `168` is value `N`, the approximate CPU frequency in MHz (in this case Pyboard V1.1 is 168MHz). It's possible to choose other values as well: lower values like `10` will run much the tests much quicker, higher values like `1000` will @@ -136,11 +135,11 @@ Usually you want to know if something is faster or slower than a reference. To do this, copy the output of each `run-perfbench.py` run to a text file. This can be done multiple ways, but one way on Linux/macOS is with the `tee` -utility: `./run-perfbench.py -p 168 100 | tee pyb-run1.txt` +utility: `./run-perfbench.py -t a0 168 100 | tee pyb-run1.txt` Once you have two files with output from two different runs (maybe with different code or configuration), compare the runtimes with `./run-perfbench.py --t pybv-run1.txt pybv-run2.txt` or compare scores with `./run-perfbench.py -s +-m pybv-run1.txt pybv-run2.txt` or compare scores with `./run-perfbench.py -s pybv-run1.txt pybv-run2.txt`: ``` @@ -204,6 +203,18 @@ internal_bench/bytebuf: 1 tests performed (3 individual testcases) ``` +## Serial reliability and performance test + +Serial port reliability and performance can be tested using the `serial_test.py` script. +Pass the name of the port to test against, for example: + + $ ./serial_test.py -t /dev/ttyACM0 + +If no port is specified then `/dev/ttyACM0` is used as the default. + +The test will send data out to the target, and receive data from the target, in various +chunk sizes. The throughput of the serial connection will be reported for each sub-test. + ## Test key/certificates SSL/TLS tests in `multi_net` and `net_inet` use self-signed key/cert pairs diff --git a/tests/basics/annotate_var.py.exp b/tests/basics/annotate_var.py.exp deleted file mode 100644 index 9b6536e966b0e..0000000000000 --- a/tests/basics/annotate_var.py.exp +++ /dev/null @@ -1,5 +0,0 @@ -False -1 -(1, 2) -NameError -1 diff --git a/tests/basics/array_add.py b/tests/basics/array_add.py index 76ce59f761e06..e78615541c3cb 100644 --- a/tests/basics/array_add.py +++ b/tests/basics/array_add.py @@ -14,3 +14,9 @@ a1.extend(array.array('I', [5])) print(a1) + +a1.extend([6, 7]) +print(a1) + +a1.extend(i for i in (8, 9)) +print(a1) diff --git a/tests/basics/assign_expr.py.exp b/tests/basics/assign_expr.py.exp deleted file mode 100644 index 47da56b80d47b..0000000000000 --- a/tests/basics/assign_expr.py.exp +++ /dev/null @@ -1,16 +0,0 @@ -4 -True -2 -4 5 -5 -1 5 5 -5 -2 1 -1 0 -any True -8 -123 -any True -8 -[(1, 0), (2, 2), (3, 6), (4, 12)] -4 diff --git a/tests/basics/assign_expr_scope.py.exp b/tests/basics/assign_expr_scope.py.exp deleted file mode 100644 index 5c780b382215f..0000000000000 --- a/tests/basics/assign_expr_scope.py.exp +++ /dev/null @@ -1,23 +0,0 @@ -scope0 -1 -None -scope1 -[1] -1 -None -scope2 -[0, 1] -1 -1 -scope3 -[0, 1] -None -None -scope4 -[0, 1] -1 -1 -scope5 -[0, 1] -1 -None diff --git a/tests/basics/async_await.py.exp b/tests/basics/async_await.py.exp deleted file mode 100644 index b51c388a9339e..0000000000000 --- a/tests/basics/async_await.py.exp +++ /dev/null @@ -1,32 +0,0 @@ -4 -3 -2 -1 -0 -0 -1 -0 -0 -2 -1 -0 -0 -1 -0 -0 -3 -2 -1 -0 -0 -1 -0 -0 -2 -1 -0 -0 -1 -0 -0 -finished diff --git a/tests/basics/async_await2.py.exp b/tests/basics/async_await2.py.exp deleted file mode 100644 index fc9ff0aa535fc..0000000000000 --- a/tests/basics/async_await2.py.exp +++ /dev/null @@ -1,5 +0,0 @@ -wait value: 1 -return from send: message from wait(1) -wait got back: message from main -x = 100 -got StopIteration diff --git a/tests/basics/async_def.py.exp b/tests/basics/async_def.py.exp deleted file mode 100644 index f555ace99ab20..0000000000000 --- a/tests/basics/async_def.py.exp +++ /dev/null @@ -1,3 +0,0 @@ -decorator -foo -StopIteration diff --git a/tests/basics/async_for.py.exp b/tests/basics/async_for.py.exp deleted file mode 100644 index 6f59979c065de..0000000000000 --- a/tests/basics/async_for.py.exp +++ /dev/null @@ -1,51 +0,0 @@ -== start == -init -aiter -init -anext -a -anext -b -anext -c -anext -== finish == -== start == -init -aiter -init -anext -d -anext -e -anext -f -anext -AsyncIteratorWrapper-def -== finish == -init -== start == -aiter -init -anext -g -anext -h -anext -i -anext -AsyncIteratorWrapper-ghi -== finish == -init -== start == -aiter -init -anext -j -anext -k -anext -l -anext -AsyncIteratorWrapper-jkl -== finish == diff --git a/tests/basics/async_for2.py b/tests/basics/async_for2.py index bbdb02c49b209..82232d52fc2a1 100644 --- a/tests/basics/async_for2.py +++ b/tests/basics/async_for2.py @@ -1,7 +1,7 @@ # test waiting within "async for" __anext__ function # CIRCUITPY-CHANGE -# uPy allows normal generators to be awaitables. +# MicroPython allows normal generators to be awaitables. # CircuitPython does not. # In CircuitPython you need to have an __await__ method on an awaitable like in CPython; # and like in CPython, generators do not have __await__. diff --git a/tests/basics/async_for2.py.exp b/tests/basics/async_for2.py.exp deleted file mode 100644 index 52bbe90c85376..0000000000000 --- a/tests/basics/async_for2.py.exp +++ /dev/null @@ -1,32 +0,0 @@ -init -aiter -anext -f start: 20 -coro yielded: 21 -coro yielded: 22 -f returned: 23 -x 0 -anext -f start: 20 -coro yielded: 21 -coro yielded: 22 -f returned: 23 -x 1 -anext -f start: 20 -coro yielded: 21 -coro yielded: 22 -f returned: 23 -x 2 -anext -f start: 20 -coro yielded: 21 -coro yielded: 22 -f returned: 23 -x 3 -anext -f start: 20 -coro yielded: 21 -coro yielded: 22 -f returned: 23 -finished diff --git a/tests/basics/async_syntaxerror.py.exp b/tests/basics/async_syntaxerror.py.exp deleted file mode 100644 index 5275689b41383..0000000000000 --- a/tests/basics/async_syntaxerror.py.exp +++ /dev/null @@ -1,2 +0,0 @@ -SyntaxError -SyntaxError diff --git a/tests/basics/async_with.py.exp b/tests/basics/async_with.py.exp deleted file mode 100644 index 6bbf84cb4b52c..0000000000000 --- a/tests/basics/async_with.py.exp +++ /dev/null @@ -1,11 +0,0 @@ -enter -body -exit None None -finished -enter -1 -exit error -ValueError -enter -exit -BaseException diff --git a/tests/basics/async_with2.py b/tests/basics/async_with2.py index 5bac38236fe4a..b18fa6bdee106 100644 --- a/tests/basics/async_with2.py +++ b/tests/basics/async_with2.py @@ -1,7 +1,7 @@ # test waiting within async with enter/exit functions # CIRCUITPY-CHANGE -# uPy allows normal generators to be awaitables. +# MicroPython allows normal generators to be awaitables. # CircuitPython does not. # In CircuitPython you need to have an __await__ method on an awaitable like in CPython; # and like in CPython, generators do not have __await__. diff --git a/tests/basics/async_with2.py.exp b/tests/basics/async_with2.py.exp deleted file mode 100644 index 76b173b4c24a7..0000000000000 --- a/tests/basics/async_with2.py.exp +++ /dev/null @@ -1,17 +0,0 @@ -enter -f start: 10 -coro yielded: 11 -coro yielded: 12 -f returned: 13 -body start -f start: 30 -coro yielded: 31 -coro yielded: 32 -body f returned: 33 -body end -exit None None -f start: 20 -coro yielded: 21 -coro yielded: 22 -f returned: 23 -finished diff --git a/tests/basics/async_with_break.py.exp b/tests/basics/async_with_break.py.exp deleted file mode 100644 index d077a88fad0e4..0000000000000 --- a/tests/basics/async_with_break.py.exp +++ /dev/null @@ -1,15 +0,0 @@ -enter -body -exit None None -finished -enter -body -exit None None -finally -finished -enter -body -exit None None -finally inner -finally outer -finished diff --git a/tests/basics/async_with_return.py.exp b/tests/basics/async_with_return.py.exp deleted file mode 100644 index d077a88fad0e4..0000000000000 --- a/tests/basics/async_with_return.py.exp +++ /dev/null @@ -1,15 +0,0 @@ -enter -body -exit None None -finished -enter -body -exit None None -finally -finished -enter -body -exit None None -finally inner -finally outer -finished diff --git a/tests/basics/attrtuple2.py b/tests/basics/attrtuple2.py new file mode 100644 index 0000000000000..081d24b6ae92c --- /dev/null +++ b/tests/basics/attrtuple2.py @@ -0,0 +1,25 @@ +# test os.uname() attrtuple, if available +try: + import os +except ImportError: + print("SKIP") + raise SystemExit + +try: + u = os.uname() +except AttributeError: + print("SKIP") + raise SystemExit + +# test printing of attrtuple +print(str(u).find("machine=") > 0) + +# test read attr +print(isinstance(u.machine, str)) + +# test str modulo operator for attrtuple +impl_str = ("%s " * len(u)) % u +test_str = "" +for val in u: + test_str += val + " " +print(impl_str == test_str) diff --git a/tests/basics/boundmeth1.py b/tests/basics/boundmeth1.py index fe1b454688d64..9f08d9bbb259b 100644 --- a/tests/basics/boundmeth1.py +++ b/tests/basics/boundmeth1.py @@ -1,6 +1,6 @@ # tests basics of bound methods -# uPy and CPython differ when printing a bound method, so just print the type +# MicroPython and CPython differ when printing a bound method, so just print the type print(type(repr([].append))) diff --git a/tests/basics/builtin_help.py b/tests/basics/builtin_help.py index 6ec39653fe993..3d4e4f26bd2bb 100644 --- a/tests/basics/builtin_help.py +++ b/tests/basics/builtin_help.py @@ -14,4 +14,10 @@ help(micropython) # help for a module help('modules') # list available modules +class A: + x = 1 + y = 2 + del x +help(A) + print('done') # so last bit of output is predictable diff --git a/tests/basics/builtin_pow3_intbig.py b/tests/basics/builtin_pow3_intbig.py index bedc8b36b7edd..41d2acbc0cc75 100644 --- a/tests/basics/builtin_pow3_intbig.py +++ b/tests/basics/builtin_pow3_intbig.py @@ -20,3 +20,8 @@ print(hex(pow(y, x-1, x))) # Should be 1, since x is prime print(hex(pow(y, y-1, x))) # Should be a 'big value' print(hex(pow(y, y-1, y))) # Should be a 'big value' + +try: + print(pow(1, 2, 0)) +except ValueError: + print("ValueError") diff --git a/tests/basics/builtin_range.py b/tests/basics/builtin_range.py index 66226bad16a6d..9608a8163107e 100644 --- a/tests/basics/builtin_range.py +++ b/tests/basics/builtin_range.py @@ -32,13 +32,29 @@ print(range(1, 4)[0:]) print(range(1, 4)[1:]) print(range(1, 4)[:-1]) +print(range(4, 1)[:]) +print(range(4, 1)[0:]) +print(range(4, 1)[1:]) +print(range(4, 1)[:-1]) print(range(7, -2, -4)[:]) print(range(1, 100, 5)[5:15:3]) print(range(1, 100, 5)[15:5:-3]) print(range(100, 1, -5)[5:15:3]) print(range(100, 1, -5)[15:5:-3]) +print(range(1, 100, 5)[5:15:-3]) +print(range(1, 100, 5)[15:5:3]) +print(range(100, 1, -5)[5:15:-3]) +print(range(100, 1, -5)[15:5:3]) +print(range(1, 100, 5)[5:15:3]) +print(range(1, 100, 5)[15:5:-3]) +print(range(1, 100, -5)[5:15:3]) +print(range(1, 100, -5)[15:5:-3]) +print(range(1, 100, 5)[5:15:-3]) +print(range(1, 100, 5)[15:5:3]) +print(range(1, 100, -5)[5:15:-3]) +print(range(1, 100, -5)[15:5:3]) -# for this case uPy gives a different stop value but the listed elements are still correct +# for this case MicroPython gives a different stop value but the listed elements are still correct print(list(range(7, -2, -4)[2:-2:])) # zero step diff --git a/tests/basics/builtin_range_maxsize.py b/tests/basics/builtin_range_maxsize.py new file mode 100644 index 0000000000000..b0f3a5e5129f1 --- /dev/null +++ b/tests/basics/builtin_range_maxsize.py @@ -0,0 +1,38 @@ +try: + from sys import maxsize +except ImportError: + print("SKIP") + raise SystemExit + +# Test the range builtin at extreme values. (https://github.com/micropython/micropython/issues/17684) +# +# This is written using asserts instead of prints because the value of `maxsize` differs. +# +# Numbers & counts right up against the max of mp_int_t also cause overflows, such as +# objrange.c:115:14: runtime error: signed integer overflow: 9223372036854775807 + 1 cannot be represented in type 'long int' +# so just avoid them for the purposes of this test. + +r = range(-maxsize + 1, -1) +assert r.start == -maxsize + 1 +assert r.stop == -1 +assert r[0] == -maxsize + 1 +assert r[1] == -maxsize + 2 +assert r[-1] == -2 +assert r[-2] == -3 + +ir = iter(r) +assert next(ir) == -maxsize + 1 +assert next(ir) == -maxsize + 2 + +r = range(0, maxsize - 1) +assert len(r) == maxsize - 1 +assert r.stop == maxsize - 1 + +r = range(maxsize, 0, -1) +assert len(r) == maxsize +assert r.start == maxsize +assert r[0] == maxsize +assert r[1] == maxsize - 1 +ir = iter(r) +assert next(ir) == maxsize +assert next(ir) == maxsize - 1 diff --git a/tests/basics/builtin_setattr.py b/tests/basics/builtin_setattr.py index e4acb14cac354..2c9a452c32774 100644 --- a/tests/basics/builtin_setattr.py +++ b/tests/basics/builtin_setattr.py @@ -21,5 +21,5 @@ def __init__(self): try: setattr(int, 'to_bytes', 1) except (AttributeError, TypeError): - # uPy raises AttributeError, CPython raises TypeError + # MicroPython raises AttributeError, CPython raises TypeError print('AttributeError/TypeError') diff --git a/tests/basics/builtin_slice.py b/tests/basics/builtin_slice.py index 3ff3c3dd04d43..cbce3d14ed699 100644 --- a/tests/basics/builtin_slice.py +++ b/tests/basics/builtin_slice.py @@ -1,11 +1,44 @@ # test builtin slice +# ensures that slices passed to user types are heap-allocated and can be +# safely stored as well as not overridden by subsequent slices. + # print slice class A: def __getitem__(self, idx): - print(idx) + print("get", idx) + print("abc"[1:]) + print("get", idx) + return idx + + def __setitem__(self, idx, value): + print("set", idx) + print("abc"[1:]) + print("set", idx) + self.saved_idx = idx + return idx + + def __delitem__(self, idx): + print("del", idx) + print("abc"[1:]) + print("del", idx) return idx -s = A()[1:2:3] + + +a = A() +s = a[1:2:3] +a[4:5:6] = s +del a[7:8:9] + +print(a.saved_idx) + +# nested slicing +print(A()[1 : A()[A()[2:3:4] : 5]]) + +# tuple slicing +a[1:2, 4:5, 7:8] +a[1, 4:5, 7:8, 2] +a[1:2, a[3:4], 5:6] # check type print(type(s) is slice) diff --git a/tests/basics/bytes_format_modulo.py.exp b/tests/basics/bytes_format_modulo.py.exp deleted file mode 100644 index 782b7f91fcbfe..0000000000000 --- a/tests/basics/bytes_format_modulo.py.exp +++ /dev/null @@ -1,5 +0,0 @@ -b'%' -b'=1=' -b'=1=2=' -b'=str=' -b"=b'str'=" diff --git a/tests/basics/class_bind_self.py b/tests/basics/class_bind_self.py index 813f876446ef2..8bece902ca254 100644 --- a/tests/basics/class_bind_self.py +++ b/tests/basics/class_bind_self.py @@ -52,7 +52,7 @@ def f4(self, arg): # generator print(C.f8(13)) print(C.f9(14)) -# not working in uPy +# not working in MicroPython #class C(list): # # this acts like a method and binds self # f1 = list.extend diff --git a/tests/basics/class_descriptor.py b/tests/basics/class_descriptor.py index 83d31674301d5..feaed2fbb2a43 100644 --- a/tests/basics/class_descriptor.py +++ b/tests/basics/class_descriptor.py @@ -1,22 +1,28 @@ class Descriptor: def __get__(self, obj, cls): - print('get') + print("get") print(type(obj) is Main) print(cls is Main) - return 'result' + return "result" def __set__(self, obj, val): - print('set') + print("set") print(type(obj) is Main) print(val) def __delete__(self, obj): - print('delete') + print("delete") print(type(obj) is Main) + def __set_name__(self, owner, name): + print("set_name", name) + print(owner.__name__ == "Main") + + class Main: Forward = Descriptor() + m = Main() try: m.__class__ @@ -26,15 +32,15 @@ class Main: raise SystemExit r = m.Forward -if 'Descriptor' in repr(r.__class__): +if "Descriptor" in repr(r.__class__): # Target doesn't support descriptors. - print('SKIP') + print("SKIP") raise SystemExit # Test assignment and deletion. print(r) -m.Forward = 'a' +m.Forward = "a" del m.Forward # Test that lookup of descriptors like __get__ are not passed into __getattr__. diff --git a/tests/basics/class_inplace_op2.py.exp b/tests/basics/class_inplace_op2.py.exp deleted file mode 100644 index 8c323b5178ef4..0000000000000 --- a/tests/basics/class_inplace_op2.py.exp +++ /dev/null @@ -1,12 +0,0 @@ -__imul__ -__imatmul__ -__ifloordiv__ -__itruediv__ -__imod__ -__ipow__ -__ior__ -__ixor__ -__iand__ -__ilshift__ -__irshift__ -TypeError diff --git a/tests/basics/class_ordereddict.py.exp b/tests/basics/class_ordereddict.py.exp deleted file mode 100644 index b723e327515c7..0000000000000 --- a/tests/basics/class_ordereddict.py.exp +++ /dev/null @@ -1 +0,0 @@ -['a', 'b', 'c', 'd', 'e'] diff --git a/tests/basics/class_setname_hazard.py b/tests/basics/class_setname_hazard.py new file mode 100644 index 0000000000000..77c0409346282 --- /dev/null +++ b/tests/basics/class_setname_hazard.py @@ -0,0 +1,182 @@ +# Test that __set_name__ can access and mutate its owner argument. + + +def skip_if_no_descriptors(): + class Descriptor: + def __get__(self, obj, cls): + return + + class TestClass: + Forward = Descriptor() + + a = TestClass() + try: + a.__class__ + except AttributeError: + # Target doesn't support __class__. + print("SKIP") + raise SystemExit + + b = a.Forward + if "Descriptor" in repr(b.__class__): + # Target doesn't support descriptors. + print("SKIP") + raise SystemExit + + +skip_if_no_descriptors() + + +# Test basic accesses and mutations. + + +class GetSibling: + def __set_name__(self, owner, name): + print(getattr(owner, name + "_sib")) + + +class GetSiblingTest: + desc = GetSibling() + desc_sib = 111 + + +t110 = GetSiblingTest() + + +class SetSibling: + def __set_name__(self, owner, name): + setattr(owner, name + "_sib", 121) + + +class SetSiblingTest: + desc = SetSibling() + + +t120 = SetSiblingTest() + +print(t120.desc_sib) + + +class DelSibling: + def __set_name__(self, owner, name): + delattr(owner, name + "_sib") + + +class DelSiblingTest: + desc = DelSibling() + desc_sib = 131 + + +t130 = DelSiblingTest() + +try: + print(t130.desc_sib) +except AttributeError: + print("AttributeError") + + +class GetSelf: + x = 211 + + def __set_name__(self, owner, name): + print(getattr(owner, name).x) + + +class GetSelfTest: + desc = GetSelf() + + +t210 = GetSelfTest() + + +class SetSelf: + def __set_name__(self, owner, name): + setattr(owner, name, 221) + + +class SetSelfTest: + desc = SetSelf() + + +t220 = SetSelfTest() + +print(t220.desc) + + +class DelSelf: + def __set_name__(self, owner, name): + delattr(owner, name) + + +class DelSelfTest: + desc = DelSelf() + + +t230 = DelSelfTest() + +try: + print(t230.desc) +except AttributeError: + print("AttributeError") + + +# Test exception behavior. + + +class Raise: + def __set_name__(self, owner, name): + raise Exception() + + +try: + + class RaiseTest: + desc = Raise() +except Exception as e: # CPython raises RuntimeError, MicroPython propagates the original exception + print("Exception") + + +# Ensure removed/overwritten class members still get __set_name__ called. + + +class SetSpecific: + def __init__(self, sib_name, sib_replace): + self.sib_name = sib_name + self.sib_replace = sib_replace + + def __set_name__(self, owner, name): + setattr(owner, self.sib_name, self.sib_replace) + + +class SetReplaceTest: + a = SetSpecific("b", 312) # one of these is changed first + b = SetSpecific("a", 311) + + +t310 = SetReplaceTest() +print(t310.a) +print(t310.b) + + +class DelSpecific: + def __init__(self, sib_name): + self.sib_name = sib_name + + def __set_name__(self, owner, name): + delattr(owner, self.sib_name) + + +class DelReplaceTest: + a = DelSpecific("b") # one of these is removed first + b = DelSpecific("a") + + +t320 = DelReplaceTest() +try: + print(t320.a) +except AttributeError: + print("AttributeError") +try: + print(t320.b) +except AttributeError: + print("AttributeError") diff --git a/tests/basics/class_setname_hazard_rand.py b/tests/basics/class_setname_hazard_rand.py new file mode 100644 index 0000000000000..4c9934c3bf068 --- /dev/null +++ b/tests/basics/class_setname_hazard_rand.py @@ -0,0 +1,111 @@ +# Test to make sure there's no sequence hazard even when a __set_name__ implementation +# mutates and reorders the namespace of its owner class. +# VERY hard bug to prove out except via a stochastic test. + + +try: + from random import choice + import re +except ImportError: + print("SKIP") + raise SystemExit + + +def skip_if_no_descriptors(): + class Descriptor: + def __get__(self, obj, cls): + return + + class TestClass: + Forward = Descriptor() + + a = TestClass() + try: + a.__class__ + except AttributeError: + # Target doesn't support __class__. + print("SKIP") + raise SystemExit + + b = a.Forward + if "Descriptor" in repr(b.__class__): + # Target doesn't support descriptors. + print("SKIP") + raise SystemExit + + +skip_if_no_descriptors() + +letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + +# Would be r"[A-Z]{5}", but not all ports support the {n} quantifier. +junk_re = re.compile(r"[A-Z][A-Z][A-Z][A-Z][A-Z]") + + +def junk_fill(obj, n=10): # Add randomly-generated attributes to an object. + for i in range(n): + name = "".join(choice(letters) for j in range(5)) + setattr(obj, name, object()) + + +def junk_clear(obj): # Remove attributes added by junk_fill. + to_del = [name for name in dir(obj) if junk_re.match(name)] + for name in to_del: + delattr(obj, name) + + +def junk_sequencer(): + global runs + try: + while True: + owner, name = yield + runs[name] = runs.get(name, 0) + 1 + junk_fill(owner) + finally: + junk_clear(owner) + + +class JunkMaker: + def __set_name__(self, owner, name): + global seq + seq.send((owner, name)) + + +runs = {} +seq = junk_sequencer() +next(seq) + + +class Main: + a = JunkMaker() + b = JunkMaker() + c = JunkMaker() + d = JunkMaker() + e = JunkMaker() + f = JunkMaker() + g = JunkMaker() + h = JunkMaker() + i = JunkMaker() + j = JunkMaker() + k = JunkMaker() + l = JunkMaker() + m = JunkMaker() + n = JunkMaker() + o = JunkMaker() + p = JunkMaker() + q = JunkMaker() + r = JunkMaker() + s = JunkMaker() + t = JunkMaker() + u = JunkMaker() + v = JunkMaker() + w = JunkMaker() + x = JunkMaker() + y = JunkMaker() + z = JunkMaker() + + +seq.close() + +for k in letters.lower(): + print(k, runs.get(k, 0)) diff --git a/tests/basics/del_attr.py b/tests/basics/del_attr.py index 8487b97e311d0..90a8a5a6f833d 100644 --- a/tests/basics/del_attr.py +++ b/tests/basics/del_attr.py @@ -35,5 +35,5 @@ def f(): try: del int.to_bytes except (AttributeError, TypeError): - # uPy raises AttributeError, CPython raises TypeError + # MicroPython raises AttributeError, CPython raises TypeError print('AttributeError/TypeError') diff --git a/tests/basics/del_global.py b/tests/basics/del_global.py index d740357b0332d..e1aa074b97a43 100644 --- a/tests/basics/del_global.py +++ b/tests/basics/del_global.py @@ -14,7 +14,7 @@ def do_del(): try: do_del() except: # NameError: - # FIXME uPy returns KeyError for this + # FIXME MicroPython returns KeyError for this print("NameError") # delete globals using a list diff --git a/tests/basics/del_name.py b/tests/basics/del_name.py index c92be54d3b655..8393b2df92a72 100644 --- a/tests/basics/del_name.py +++ b/tests/basics/del_name.py @@ -10,7 +10,7 @@ try: del x except: # NameError: - # FIXME uPy returns KeyError for this + # FIXME MicroPython returns KeyError for this print("NameError") class C: diff --git a/tests/basics/dict_views.py b/tests/basics/dict_views.py index a82f47b6be26f..45f3238f1c624 100644 --- a/tests/basics/dict_views.py +++ b/tests/basics/dict_views.py @@ -6,6 +6,11 @@ # print a view with more than one item print({1:1, 2:1}.values()) +# `bool` and `len` unary ops +for d in ({}, {1: 2}, {1: 2, 3: 4}): + for op in (bool, len): + print(op(d.keys()), op(d.values()), op(d.items())) + # unsupported binary op on a dict values view try: {1:1}.values() + 1 diff --git a/tests/basics/exception_chain.py b/tests/basics/exception_chain.py index 579756c85fa20..96c06c847ab37 100644 --- a/tests/basics/exception_chain.py +++ b/tests/basics/exception_chain.py @@ -1,5 +1,12 @@ # CIRCUITPY-CHANGE: exception chaining is supported +import sys + +# The unix minimal build doesn't enable MICROPY_WARNINGS (required for this test). +if getattr(sys.implementation, "_build", None) == "minimal": + print("SKIP") + raise SystemExit + try: Exception().__cause__ except AttributeError: diff --git a/tests/basics/frozenset_set.py b/tests/basics/frozenset_set.py index 3bf456acfd70f..2b3c683260989 100644 --- a/tests/basics/frozenset_set.py +++ b/tests/basics/frozenset_set.py @@ -8,5 +8,5 @@ # "Instances of set are compared to instances of frozenset based on their # members. For example:" print(set('abc') == frozenset('abc')) -# This doesn't work in uPy +# This doesn't work in MicroPython #print(set('abc') in set([frozenset('abc')])) diff --git a/tests/basics/fun_code_colines.py b/tests/basics/fun_code_colines.py new file mode 100644 index 0000000000000..a8867770eddf1 --- /dev/null +++ b/tests/basics/fun_code_colines.py @@ -0,0 +1,81 @@ +# Check that we have sensical bytecode offsets in function.__code__.co_lines + +def f1(x, y, obj, obj2, obj3): + a = x + y # line 4: bc+4 line+4 + b = x - y # line 5: bc+4 line+1 + # line 6 + # line 7 + # line 8 + # line 9 + # line 10 + # line 11 + # line 12 + # line 13 + # line 14 + # line 15 + # line 16 + # line 17 + # line 18 + # line 19 + c = a * b # line 20: bc+4 line+15 + obj.a.b.c.d.e.f.g.h.i.j.k.l.m.n.o.p.q.r.s.t.u.v.w.x.y.z.fun() # line 21: bc+31 line+1; bc+27 line+0 + # line 22 + # line 23 + # line 24: bc+0 line+3 + # line 25 + # line 26 + # line 27: bc+0 line+3 + # line 28 + # line 29 + obj2.a.b.c.d.e.f.g.h.i.j.k.l.m.n.o.p.q.r.s.t.u.v.w.x.y.z.fun() # line 30: bc+31 line+3; bc+27 line+0 + # line 31 + # line 32 + # line 33: bc+0 line+3 + # line 34 + # line 35 + # line 36 + # line 37 + # line 38 + # line 39 + # line 40 + # line 41 + # line 42 + # line 43 + # line 44 + # line 45 + # line 46 + # line 47 + # line 48 + # line 49 + # line 50 + # line 51 + # line 52 + # line 53 + # line 54 + # line 55 + # line 56 + # line 57 + # line 58 + # line 59 + return obj3.a.b.c.d.e.f.g.h.i.j.k.l.m.n.o.p.q.r.s.t.u.v.w.x.y.z.fun() # line 60: bc+31 line+27; bc+27 line+0 + +def f2(x, y): + a = x + y # line 63 + b = x - y # line 64 + return a * b # line 65 + +try: + f1.__code__.co_lines +except AttributeError: + print("SKIP") + raise SystemExit + +print("f1") +for start, end, line_no in f1.__code__.co_lines(): + print("line {} start: {}".format(line_no, start)) + print("line {} end: {}".format(line_no, end)) + +print("f2") +for start, end, line_no in f2.__code__.co_lines(): + print("line {} start: {}".format(line_no, start)) + print("line {} end: {}".format(line_no, end)) diff --git a/tests/basics/fun_code_colines.py.exp b/tests/basics/fun_code_colines.py.exp new file mode 100644 index 0000000000000..19bd4ef6e2a5a --- /dev/null +++ b/tests/basics/fun_code_colines.py.exp @@ -0,0 +1,20 @@ +f1 +line 4 start: 0 +line 4 end: 4 +line 5 start: 4 +line 5 end: 8 +line 20 start: 8 +line 20 end: 12 +line 21 start: 12 +line 21 end: 70 +line 30 start: 70 +line 30 end: 128 +line 60 start: 128 +line 60 end: 186 +f2 +line 63 start: 0 +line 63 end: 4 +line 64 start: 4 +line 64 end: 8 +line 65 start: 8 +line 65 end: 12 diff --git a/tests/basics/fun_code_full.py b/tests/basics/fun_code_full.py new file mode 100644 index 0000000000000..538863631a6e0 --- /dev/null +++ b/tests/basics/fun_code_full.py @@ -0,0 +1,38 @@ +# Test function.__code__ attributes not available with MICROPY_PY_BUILTINS_CODE <= MICROPY_PY_BUILTINS_CODE_BASIC + +try: + (lambda: 0).__code__.co_code +except AttributeError: + print("SKIP") + raise SystemExit + +def f(x, y): + a = x + y + b = x - y + return a * b + +code = f.__code__ + +print(type(code.co_code)) # both bytes (but mpy and cpy have different instruction sets) +print(code.co_consts) # (not necessarily the same set, but in this function they are) +print(code.co_filename.rsplit('/')[-1]) # same terminal filename but might be different paths on other ports +print(type(code.co_firstlineno)) # both ints (but mpy points to first line inside, cpy points to declaration) +print(code.co_name) +print(iter(code.co_names) is not None) # both iterable (but mpy returns dict with names as keys, cpy only the names; and not necessarily the same set) + +co_lines = code.co_lines() + +l = list(co_lines) +first_start = l[0][0] +last_end = l[-1][1] +print(first_start) # co_lines should start at the start of the bytecode +print(len(code.co_code) - last_end) # and end at the end of the bytecode + +prev_end = 0 +for start, end, line_no in l: + if end != prev_end: + print("non-contiguous") + break # the offset ranges should be contiguous + prev_end = end +else: + print("contiguous") diff --git a/tests/basics/fun_code_full.py.exp b/tests/basics/fun_code_full.py.exp new file mode 100644 index 0000000000000..830effadfc695 --- /dev/null +++ b/tests/basics/fun_code_full.py.exp @@ -0,0 +1,9 @@ + +(None,) +fun_code_full.py + +f +True +0 +0 +non-contiguous diff --git a/tests/basics/fun_code_lnotab.py b/tests/basics/fun_code_lnotab.py new file mode 100644 index 0000000000000..9223e5730f0ea --- /dev/null +++ b/tests/basics/fun_code_lnotab.py @@ -0,0 +1,34 @@ +# Test deprecation of co_lnotab + +try: + (lambda: 0).__code__.co_code +except AttributeError: + print("SKIP") + raise SystemExit + + +import unittest +import sys + + +mpy_is_v2 = getattr(sys.implementation, '_v2', False) + + +def f(): + pass + + +class Test(unittest.TestCase): + + @unittest.skipIf(mpy_is_v2, "Removed in MicroPython v2 and later.") + def test_co_lnotab_exists(self): + self.assertIsInstance(f.__code__.co_lnotab, bytes) + + @unittest.skipUnless(mpy_is_v2, "Not removed before MicroPython v2.") + def test_co_lnotab_removed(self): + with self.assertRaises(AttributeError): + f.__code__.co_lnotab + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/basics/fun_name.py b/tests/basics/fun_name.py index b2642280a2a65..8aac35eeafa25 100644 --- a/tests/basics/fun_name.py +++ b/tests/basics/fun_name.py @@ -16,7 +16,7 @@ def Fun(self): print('SKIP') raise SystemExit -# __name__ of a bound native method is not implemented in uPy +# __name__ of a bound native method is not implemented in MicroPython # the test here is to make sure it doesn't crash try: str((1).to_bytes.__name__) diff --git a/tests/basics/gc1.py b/tests/basics/gc1.py index 332bf9744c072..dff9538b14b5c 100644 --- a/tests/basics/gc1.py +++ b/tests/basics/gc1.py @@ -15,13 +15,13 @@ gc.collect() if hasattr(gc, 'mem_free'): - # uPy has these extra functions + # MicroPython has these extra functions # just test they execute and return an int assert type(gc.mem_free()) is int assert type(gc.mem_alloc()) is int if hasattr(gc, 'threshold'): - # uPy has this extra function + # MicroPython has this extra function # check execution and returns assert(gc.threshold(1) is None) assert(gc.threshold() == 0) diff --git a/tests/basics/import_instance_method.py b/tests/basics/import_instance_method.py new file mode 100644 index 0000000000000..d25b70ac5f0bf --- /dev/null +++ b/tests/basics/import_instance_method.py @@ -0,0 +1,38 @@ +# Test importing a method from a class instance. +# This is not a common thing to do, but ensures MicroPython has the same semantics as CPython. + +import sys + +if not hasattr(sys, "modules"): + print("SKIP") + raise SystemExit + + +class A: + def __init__(self, value): + self.value = value + + def meth(self): + return self.value + + def meth_with_arg(self, a): + return [self.value, a] + + +# Register a class instance as the module "mod". +sys.modules["mod"] = A(1) + +# Try importing it as a module. +import mod + +print(mod.meth()) +print(mod.meth_with_arg(2)) + +# Change the module. +sys.modules["mod"] = A(3) + +# Try importing it using "from ... import". +from mod import meth, meth_with_arg + +print(meth()) +print(meth_with_arg(4)) diff --git a/tests/basics/int_64_basics.py b/tests/basics/int_64_basics.py new file mode 100644 index 0000000000000..ef76793317ec8 --- /dev/null +++ b/tests/basics/int_64_basics.py @@ -0,0 +1,161 @@ +# test support for 64-bit long integers +# (some ports don't support arbitrary precision but do support these) + +# this test is adapted from int_big1.py with numbers kept within 64-bit signed range + +# to test arbitrary precision integers + +x = 1000000000000000000 +xn = -1000000000000000000 +y = 2000000000000000000 + +# printing +print(x) +print(y) +print('%#X' % (x - x)) # print prefix +print('{:#,}'.format(x)) # print with commas + +# construction +print(int(x)) + +# addition +print(x + 1) +print(x + y) +print(x + xn == 0) +print(bool(x + xn)) + +# subtraction +print(x - 1) +print(x - y) +print(y - x) +print(x - x == 0) +print(bool(x - x)) + +# multiplication +print(x * 2) +print(1090511627776 * 1048500) + +# integer division +print(x // 2) +print(y // x) + +# bit inversion +print(~x) +print(~(-x)) + +# left shift +print("left shift positive") +x = 0x40000000 +for i in range(32): + x = x << 1 + print(x) + +# right shift +print("right shift positive") +x = 0x2000000000000000 # TODO: why can't second-tip bit be set? +for i in range(64): + x = x >> 1 + print(x) + +# left shift of a negative number +print("left shift negative") +for i in range(8): + print(-10000000000000000 << i) + print(-10000000000000001 << i) + print(-10000000000000002 << i) + print(-10000000000000003 << i) + print(-10000000000000004 << i) + + +# right shift of a negative number +print("right shift negative") +for i in range(8): + print(-1000000000000000000 >> i) + print(-1000000000000000001 >> i) + print(-1000000000000000002 >> i) + print(-1000000000000000003 >> i) + print(-1000000000000000004 >> i) + +# conversion from string +print(int("1234567890123456789")) +print(int("-1234567890123456789")) +print(int("1234567890abcdef", 16)) +print(int("1234567890ABCDEF", 16)) +print(int("-1234567890ABCDEF", 16)) +print(int("ijklmnopqrsz", 36)) + +# numbers close to 64-bit limits +print(int("-9111222333444555666")) +print(int("9111222333444555666")) + +# numbers with preceding 0s +print(int("-00000000000000000000009111222333444555666")) +print(int("0000000000000000000000009111222333444555666")) + +# invalid characters in string +try: + print(int("1234567890abcdef")) +except ValueError: + print('ValueError'); +try: + print(int("123456789\x01")) +except ValueError: + print('ValueError'); + +# test parsing ints just on threshold of small to big +# for 32 bit archs +x = 1073741823 # small +x = -1073741823 # small +x = 1073741824 # big +x = -1073741824 # big +# for 64 bit archs +x = 4611686018427387903 # small +x = -4611686018427387903 # small +x = 4611686018427387904 # big +x = -4611686018427387904 # big + +# sys.maxsize is a constant bigint, so test it's compatible with dynamic ones +import sys +if hasattr(sys, "maxsize"): + print(sys.maxsize - 1 + 1 == sys.maxsize) +else: + print(True) # No maxsize property in this config + +# test extraction of big int value via mp_obj_get_int_maybe +x = 1 << 62 +print('a' * (x + 4 - x)) + +# test overflow check in mp_obj_get_int_maybe +x = 1 << 32 +r = None +try: + r = range(0, x) +except OverflowError: + # 32-bit target, correctly handled the overflow of x + print("ok") +if r is not None: + if len(r) == x: + # 64-bit target, everything is just a small-int + print("ok") + else: + # 32-bit target that did not handle the overflow of x + print("unhandled overflow") + +# negative shifts are invalid +try: + print((1 << 48) >> -4) +except ValueError as e: + print(e) + +try: + print((1 << 48) << -6) +except ValueError as e: + print(e) + +# Test that the most extreme 64 bit integer values all parse with int() +print(int("-9223372036854775807")) +print(int("-9223372036854775808")) +print(int("9223372036854775807")) + +# Test that the most negative 64 bit integer can be formed via arithmetic +print(-9223372036854775807-1) diff --git a/tests/basics/int_big_to_small.py b/tests/basics/int_big_to_small.py index 64280d0c635f5..c92d263fc2837 100644 --- a/tests/basics/int_big_to_small.py +++ b/tests/basics/int_big_to_small.py @@ -5,6 +5,17 @@ print("SKIP") raise SystemExit +# Skip this test on "REPR B" where (1<<29 + 1) is not a small int. +k = 1 << 29 +micropython.heap_lock() +try: + k = k + 1 +except MemoryError: + print("SKIP") + raise SystemExit +finally: + micropython.heap_unlock() + # All less than small int max. for d in (0, 27, 1<<29, -1861, -(1<<29)): i = 1<<70 diff --git a/tests/basics/int_big_to_small_int29.py b/tests/basics/int_big_to_small_int29.py new file mode 100644 index 0000000000000..438a74d2bc6a4 --- /dev/null +++ b/tests/basics/int_big_to_small_int29.py @@ -0,0 +1,22 @@ +try: + import micropython + micropython.heap_lock +except: + print("SKIP") + raise SystemExit + +# All less than small int max. +for d in (0, 27, 1<<28, -1861, -(1<<28)): + i = 1<<70 + print(i) + j = (1<<70) + d + print(j) + # k should now be a small int. + k = j - i + print(k) + + # Now verify that working with k doesn't allocate (i.e. it's a small int). + micropython.heap_lock() + print(k + 20) + print(k // 20) + micropython.heap_unlock() diff --git a/tests/basics/int_big_to_small_int29.py.exp b/tests/basics/int_big_to_small_int29.py.exp new file mode 100644 index 0000000000000..0920101924aa8 --- /dev/null +++ b/tests/basics/int_big_to_small_int29.py.exp @@ -0,0 +1,25 @@ +1180591620717411303424 +1180591620717411303424 +0 +20 +0 +1180591620717411303424 +1180591620717411303451 +27 +47 +1 +1180591620717411303424 +1180591620717679738880 +268435456 +268435476 +13421772 +1180591620717411303424 +1180591620717411301563 +-1861 +-1841 +-94 +1180591620717411303424 +1180591620717142867968 +-268435456 +-268435436 +-13421773 diff --git a/tests/basics/io_buffered_writer.py b/tests/basics/io_buffered_writer.py index 11df0324e2467..3cfee0103f777 100644 --- a/tests/basics/io_buffered_writer.py +++ b/tests/basics/io_buffered_writer.py @@ -1,10 +1,10 @@ -import io - try: + import io + io.BytesIO io.BufferedWriter -except AttributeError: - print("SKIP") +except (AttributeError, ImportError): + print('SKIP') raise SystemExit bts = io.BytesIO() @@ -28,3 +28,27 @@ # hashing a BufferedWriter print(type(hash(buf))) + +# Test failing flush() +class MyIO(io.IOBase): + def __init__(self): + self.count = 0 + + def write(self, buf): + self.count += 1 + if self.count < 3: + return None + print("writing", buf) + return len(buf) + + +buf = io.BufferedWriter(MyIO(), 8) + +buf.write(b"foobar") + +for _ in range(4): + try: + buf.flush() + print("flushed") + except OSError: + print("OSError") diff --git a/tests/basics/io_buffered_writer.py.exp b/tests/basics/io_buffered_writer.py.exp index 2209348f5af45..d61eb148b453c 100644 --- a/tests/basics/io_buffered_writer.py.exp +++ b/tests/basics/io_buffered_writer.py.exp @@ -4,3 +4,8 @@ b'foobarfoobar' b'foobarfoobar' b'foo' +OSError +OSError +writing bytearray(b'foobar') +flushed +flushed diff --git a/tests/basics/io_bytesio_cow.py b/tests/basics/io_bytesio_cow.py index 2edb7136a9691..543c12ad42ab2 100644 --- a/tests/basics/io_bytesio_cow.py +++ b/tests/basics/io_bytesio_cow.py @@ -1,6 +1,12 @@ # Make sure that write operations on io.BytesIO don't # change original object it was constructed from. -import io + +try: + import io +except ImportError: + print("SKIP") + raise SystemExit + b = b"foobar" a = io.BytesIO(b) diff --git a/tests/basics/io_bytesio_ext.py b/tests/basics/io_bytesio_ext.py index 4d4c60c1363b7..92e715178116c 100644 --- a/tests/basics/io_bytesio_ext.py +++ b/tests/basics/io_bytesio_ext.py @@ -1,5 +1,11 @@ # Extended stream operations on io.BytesIO -import io + +try: + import io +except ImportError: + print("SKIP") + raise SystemExit + a = io.BytesIO(b"foobar") a.seek(10) print(a.read(10)) diff --git a/tests/basics/io_bytesio_ext2.py b/tests/basics/io_bytesio_ext2.py index 414ac90a3b083..f60a6a9a6041e 100644 --- a/tests/basics/io_bytesio_ext2.py +++ b/tests/basics/io_bytesio_ext2.py @@ -1,4 +1,9 @@ -import io +try: + import io +except ImportError: + print("SKIP") + raise SystemExit + a = io.BytesIO(b"foobar") try: a.seek(-10) diff --git a/tests/basics/io_iobase.py b/tests/basics/io_iobase.py index b2ee5cd63a619..31f2616310e79 100644 --- a/tests/basics/io_iobase.py +++ b/tests/basics/io_iobase.py @@ -1,15 +1,17 @@ import io try: + import io + io.IOBase -except AttributeError: +except (AttributeError, ImportError): print("SKIP") raise SystemExit class MyIO(io.IOBase): def write(self, buf): - # CPython and uPy pass in different types for buf (str vs bytearray) - print("write", len(buf)) + # CPython and MicroPython pass in different types for buf (str vs bytearray) + print('write', len(buf)) return len(buf) -print("test", file=MyIO()) +print('test', file=MyIO()) diff --git a/tests/basics/io_stringio1.py b/tests/basics/io_stringio1.py index 7d355930f5a29..889e3ce697377 100644 --- a/tests/basics/io_stringio1.py +++ b/tests/basics/io_stringio1.py @@ -1,4 +1,9 @@ -import io +try: + import io +except ImportError: + print("SKIP") + raise SystemExit + a = io.StringIO() print('io.StringIO' in repr(a)) print(a.getvalue()) diff --git a/tests/basics/io_stringio_base.py b/tests/basics/io_stringio_base.py index 0f65fb3fabc3b..c8890dab73ab7 100644 --- a/tests/basics/io_stringio_base.py +++ b/tests/basics/io_stringio_base.py @@ -1,7 +1,11 @@ # Checks that an instance type inheriting from a native base that uses # MP_TYPE_FLAG_ITER_IS_STREAM will still have a getiter. -import io +try: + import io +except ImportError: + print("SKIP") + raise SystemExit a = io.StringIO() a.write("hello\nworld\nmicro\npython\n") diff --git a/tests/basics/io_stringio_with.py b/tests/basics/io_stringio_with.py index a3aa6ec84e066..0155ad5382dcd 100644 --- a/tests/basics/io_stringio_with.py +++ b/tests/basics/io_stringio_with.py @@ -1,4 +1,9 @@ -import io +try: + import io +except ImportError: + print("SKIP") + raise SystemExit + # test __enter__/__exit__ with io.StringIO() as b: b.write("foo") diff --git a/tests/basics/io_write_ext.py b/tests/basics/io_write_ext.py index 79f7450954e4c..5af1de7a6c3aa 100644 --- a/tests/basics/io_write_ext.py +++ b/tests/basics/io_write_ext.py @@ -1,11 +1,12 @@ # This tests extended (MicroPython-specific) form of write: # write(buf, len) and write(buf, offset, len) -import io try: + import io + io.BytesIO -except AttributeError: - print("SKIP") +except (AttributeError, ImportError): + print('SKIP') raise SystemExit buf = io.BytesIO() diff --git a/tests/basics/list_pop.py b/tests/basics/list_pop.py index 87ed456f85174..58bcdd91e414c 100644 --- a/tests/basics/list_pop.py +++ b/tests/basics/list_pop.py @@ -10,7 +10,7 @@ else: raise AssertionError("No IndexError raised") -# popping such that list storage shrinks (tests implementation detail of uPy) +# popping such that list storage shrinks (tests implementation detail of MicroPython) l = list(range(20)) for i in range(len(l)): l.pop() diff --git a/tests/basics/module2.py b/tests/basics/module2.py index a135601579cd7..d1dc18930f00d 100644 --- a/tests/basics/module2.py +++ b/tests/basics/module2.py @@ -1,4 +1,4 @@ -# uPy behaviour only: builtin modules are read-only +# MicroPython behaviour only: builtin modules are read-only import sys try: sys.x = 1 diff --git a/tests/basics/namedtuple1.py b/tests/basics/namedtuple1.py index 362c60583ec32..b9689b58423b8 100644 --- a/tests/basics/namedtuple1.py +++ b/tests/basics/namedtuple1.py @@ -21,6 +21,9 @@ print(isinstance(t, tuple)) + # a NamedTuple can be used as a tuple + print("(%d, %d)" % t) + # Check tuple can compare equal to namedtuple with same elements print(t == (t[0], t[1]), (t[0], t[1]) == t) diff --git a/tests/basics/parser.py b/tests/basics/parser.py index 626b67ad7a941..6ae5f05ba458e 100644 --- a/tests/basics/parser.py +++ b/tests/basics/parser.py @@ -7,7 +7,7 @@ raise SystemExit # completely empty string -# uPy and CPy differ for this case +# MPy and CPy differ for this case #try: # compile("", "stdin", "single") #except SyntaxError: diff --git a/tests/basics/python34.py b/tests/basics/python34.py index 922234d22db93..3071b55eff8bf 100644 --- a/tests/basics/python34.py +++ b/tests/basics/python34.py @@ -28,7 +28,7 @@ def test_syntax(code): test_syntax("del ()") # can't delete empty tuple (in 3.6 we can) # from basics/sys1.py -# uPy prints version 3.4 +# MicroPython prints version 3.4 import sys print(sys.version[:3]) print(sys.version_info[0], sys.version_info[1]) diff --git a/tests/basics/python36.py.exp b/tests/basics/python36.py.exp deleted file mode 100644 index 4b65daafa18fe..0000000000000 --- a/tests/basics/python36.py.exp +++ /dev/null @@ -1,5 +0,0 @@ -100000 -165 -65535 -123 -83 diff --git a/tests/basics/self_type_check.py b/tests/basics/self_type_check.py index 3bd5b3eed6058..5f6f6f2ca6585 100644 --- a/tests/basics/self_type_check.py +++ b/tests/basics/self_type_check.py @@ -3,6 +3,14 @@ import skip_if skip_if.board_in("gemma_m0", "trinket_m0") +import sys + +# Minimal builds usually don't enable MICROPY_BUILTIN_METHOD_CHECK_SELF_ARG, +# which is required for this test. +if getattr(sys.implementation, "_build", None) == "minimal": + print("SKIP") + raise SystemExit + list.append try: diff --git a/tests/basics/slice_optimise.py b/tests/basics/slice_optimise.py new file mode 100644 index 0000000000000..f663e16b8c2f9 --- /dev/null +++ b/tests/basics/slice_optimise.py @@ -0,0 +1,23 @@ +# Test that the slice-on-stack optimisation does not break various uses of slice +# (see MP_TYPE_FLAG_SUBSCR_ALLOWS_STACK_SLICE type option). +# +# Note: this test has a corresponding .py.exp file because hashing slice objects +# was not allowed in CPython until 3.12. + +try: + from collections import OrderedDict +except ImportError: + print("SKIP") + raise SystemExit + +# Attempt to index with a slice, error should contain the slice (failed key). +try: + dict()[:] +except KeyError as e: + print("KeyError", e.args) + +# Put a slice and another object into an OrderedDict, and retrieve them. +x = OrderedDict() +x[:"a"] = 1 +x["b"] = 2 +print(list(x.keys()), list(x.values())) diff --git a/tests/basics/slice_optimise.py.exp b/tests/basics/slice_optimise.py.exp new file mode 100644 index 0000000000000..3fa59aae15ae6 --- /dev/null +++ b/tests/basics/slice_optimise.py.exp @@ -0,0 +1,2 @@ +KeyError (slice(None, None, None),) +[slice(None, 'a', None), 'b'] [1, 2] diff --git a/tests/basics/special_methods2.py.exp b/tests/basics/special_methods2.py.exp deleted file mode 100644 index a9ae75be55c95..0000000000000 --- a/tests/basics/special_methods2.py.exp +++ /dev/null @@ -1,19 +0,0 @@ -__pos__ called -__pos__ called -__neg__ called -__invert__ called -__mul__ called -__matmul__ called -__truediv__ called -__floordiv__ called -__iadd__ called -__isub__ called -__mod__ called -__pow__ called -__or__ called -__and__ called -__xor__ called -__lshift__ called -__rshift__ called -['a', 'b', 'c'] -False diff --git a/tests/basics/string_format.py b/tests/basics/string_format.py index e8600f5836139..11e7836a73ece 100644 --- a/tests/basics/string_format.py +++ b/tests/basics/string_format.py @@ -22,7 +22,17 @@ def test(fmt, *args): test("{:4x}", 123) test("{:4X}", 123) +test("{:4,d}", 1) +test("{:4_d}", 1) +test("{:4_o}", 1) +test("{:4_b}", 1) +test("{:4_x}", 1) + test("{:4,d}", 12345678) +test("{:4_d}", 12345678) +test("{:4_o}", 12345678) +test("{:4_b}", 12345678) +test("{:4_x}", 12345678) test("{:#4b}", 10) test("{:#4o}", 123) diff --git a/tests/basics/string_format_sep.py b/tests/basics/string_format_sep.py new file mode 100644 index 0000000000000..de131fdaf3383 --- /dev/null +++ b/tests/basics/string_format_sep.py @@ -0,0 +1,9 @@ +try: + "%d" % 1 +except TypeError: + print("SKIP") + raise SystemExit + +for v in (0, 0x10, 0x1000, -0x10, -0x1000): + for sz in range(1, 12): print(("{:0%d,d}" % sz).format(v)) + for sz in range(1, 12): print(("{:0%d_x}" % sz).format(v)) diff --git a/tests/basics/string_fstring_debug.py.exp b/tests/basics/string_fstring_debug.py.exp deleted file mode 100644 index f0309e1c98aa9..0000000000000 --- a/tests/basics/string_fstring_debug.py.exp +++ /dev/null @@ -1,9 +0,0 @@ -x=1 -x=00000001 -a x=1 b 2 c -a x=00000001 b 2 c -a f() + g("foo") + h()=15 b -a f() + g("foo") + h()=0000000f b -a 1,=(1,) b -a x,y,=(1, 2) b -a x,1=(1, 1) b diff --git a/tests/basics/subclass_native_init.py b/tests/basics/subclass_native_init.py index 64167fa037e0c..102befd551fad 100644 --- a/tests/basics/subclass_native_init.py +++ b/tests/basics/subclass_native_init.py @@ -1,5 +1,9 @@ # test subclassing a native type and overriding __init__ +if not hasattr(object, "__init__"): + print("SKIP") + raise SystemExit + # overriding list.__init__() class L(list): def __init__(self, a, b): diff --git a/tests/basics/syntaxerror.py b/tests/basics/syntaxerror.py index c0702cb24525e..7b6ef9b7550ea 100644 --- a/tests/basics/syntaxerror.py +++ b/tests/basics/syntaxerror.py @@ -86,7 +86,7 @@ def test_syntax(code): test_syntax("nonlocal a") test_syntax("await 1") -# error on uPy, warning on CPy +# error on MPy, warning on CPy #test_syntax("def f():\n a = 1\n global a") # default except must be last @@ -98,7 +98,7 @@ def test_syntax(code): # non-keyword after keyword test_syntax("f(a=1, 2)") -# doesn't error on uPy but should +# doesn't error on MPy but should #test_syntax("f(1, i for i in i)") # all elements of dict/set must be pairs or singles diff --git a/tests/basics/sys1.py b/tests/basics/sys1.py index 7f261aa96459d..02946abc220be 100644 --- a/tests/basics/sys1.py +++ b/tests/basics/sys1.py @@ -31,6 +31,12 @@ # Effectively skip subtests print(str) +if hasattr(sys.implementation, '_thread'): + print(sys.implementation._thread in ("GIL", "unsafe")) +else: + # Effectively skip subtests + print(True) + try: print(sys.intern('micropython') == 'micropython') has_intern = True diff --git a/tests/basics/sys_tracebacklimit.py b/tests/basics/sys_tracebacklimit.py index 3ae372b8d524e..6f575fde65404 100644 --- a/tests/basics/sys_tracebacklimit.py +++ b/tests/basics/sys_tracebacklimit.py @@ -30,7 +30,7 @@ def print_exc(e): if l.startswith(" File "): l = l.split('"') print(l[0], l[2]) - # uPy and CPy tracebacks differ in that CPy prints a source line for + # MPy and CPy tracebacks differ in that CPy prints a source line for # each traceback entry. In this case, we know that offending line # has 4-space indent, so filter it out. elif not l.startswith(" "): diff --git a/tests/basics/sys_tracebacklimit.py.native.exp b/tests/basics/sys_tracebacklimit.py.native.exp new file mode 100644 index 0000000000000..f9d30c058564b --- /dev/null +++ b/tests/basics/sys_tracebacklimit.py.native.exp @@ -0,0 +1,22 @@ +ValueError: value + +limit 4 +ValueError: value + +limit 3 +ValueError: value + +limit 2 +ValueError: value + +limit 1 +ValueError: value + +limit 0 +ValueError: value + +limit -1 +ValueError: value + +True +False diff --git a/tests/basics/try_finally_continue.py.exp b/tests/basics/try_finally_continue.py.exp deleted file mode 100644 index 2901997b1aa18..0000000000000 --- a/tests/basics/try_finally_continue.py.exp +++ /dev/null @@ -1,9 +0,0 @@ -4 0 -continue -4 1 -continue -4 2 -continue -4 3 -continue -None diff --git a/tests/basics/tuple1.py b/tests/basics/tuple1.py index 72bb3f01bff4a..70ae071898e5b 100644 --- a/tests/basics/tuple1.py +++ b/tests/basics/tuple1.py @@ -17,7 +17,7 @@ x += (10, 11, 12) print(x) -# construction of tuple from large iterator (tests implementation detail of uPy) +# construction of tuple from large iterator (tests implementation detail of MicroPython) print(tuple(range(20))) # unsupported unary operation diff --git a/tests/cmdline/cmd_compile_only.py b/tests/cmdline/cmd_compile_only.py new file mode 100644 index 0000000000000..89964c1b5bdf2 --- /dev/null +++ b/tests/cmdline/cmd_compile_only.py @@ -0,0 +1,13 @@ +# cmdline: -X compile-only +# test compile-only functionality +print("This should not be printed") +x = 1 + 2 + + +def hello(): + return "world" + + +class TestClass: + def __init__(self): + self.value = 42 diff --git a/tests/cmdline/cmd_compile_only.py.exp b/tests/cmdline/cmd_compile_only.py.exp new file mode 100644 index 0000000000000..8b137891791fe --- /dev/null +++ b/tests/cmdline/cmd_compile_only.py.exp @@ -0,0 +1 @@ + diff --git a/tests/cmdline/cmd_compile_only_error.py b/tests/cmdline/cmd_compile_only_error.py new file mode 100644 index 0000000000000..326937a5c07ef --- /dev/null +++ b/tests/cmdline/cmd_compile_only_error.py @@ -0,0 +1,6 @@ +# cmdline: -X compile-only +# test compile-only with syntax error +print("This should not be printed") +def broken_syntax( + # Missing closing parenthesis + return "error" diff --git a/tests/cmdline/cmd_compile_only_error.py.exp b/tests/cmdline/cmd_compile_only_error.py.exp new file mode 100644 index 0000000000000..3911f71d6244d --- /dev/null +++ b/tests/cmdline/cmd_compile_only_error.py.exp @@ -0,0 +1 @@ +CRASH \ No newline at end of file diff --git a/tests/cmdline/cmd_file_variable.py b/tests/cmdline/cmd_file_variable.py new file mode 100644 index 0000000000000..6cac6744d904e --- /dev/null +++ b/tests/cmdline/cmd_file_variable.py @@ -0,0 +1,5 @@ +# Test that __file__ is set correctly for script execution +try: + print("__file__ =", __file__) +except NameError: + print("__file__ not defined") diff --git a/tests/cmdline/cmd_file_variable.py.exp b/tests/cmdline/cmd_file_variable.py.exp new file mode 100644 index 0000000000000..6807569f662b5 --- /dev/null +++ b/tests/cmdline/cmd_file_variable.py.exp @@ -0,0 +1 @@ +__file__ = cmdline/cmd_file_variable.py diff --git a/tests/cmdline/cmd_module_atexit.py b/tests/cmdline/cmd_module_atexit.py new file mode 100644 index 0000000000000..100bc112777e1 --- /dev/null +++ b/tests/cmdline/cmd_module_atexit.py @@ -0,0 +1,16 @@ +# cmdline: -m cmdline.cmd_module_atexit +# +# Test running as a module and using sys.atexit. + +import sys + +if not hasattr(sys, "atexit"): + print("SKIP") + raise SystemExit + +# Verify we ran as a module. +print(sys.argv) + +sys.atexit(lambda: print("done")) + +print("start") diff --git a/tests/cmdline/cmd_module_atexit.py.exp b/tests/cmdline/cmd_module_atexit.py.exp new file mode 100644 index 0000000000000..2a0f756b1e799 --- /dev/null +++ b/tests/cmdline/cmd_module_atexit.py.exp @@ -0,0 +1,3 @@ +['cmdline.cmd_module_atexit', 'cmdline/cmd_module_atexit.py'] +start +done diff --git a/tests/cmdline/cmd_module_atexit_exc.py b/tests/cmdline/cmd_module_atexit_exc.py new file mode 100644 index 0000000000000..88940a7741fb9 --- /dev/null +++ b/tests/cmdline/cmd_module_atexit_exc.py @@ -0,0 +1,19 @@ +# cmdline: -m cmdline.cmd_module_atexit_exc +# +# Test running as a module and using sys.atexit, with script completion via sys.exit. + +import sys + +if not hasattr(sys, "atexit"): + print("SKIP") + raise SystemExit + +# Verify we ran as a module. +print(sys.argv) + +sys.atexit(lambda: print("done")) + +print("start") + +# This will raise SystemExit to finish the script, and atexit should still be run. +sys.exit(0) diff --git a/tests/cmdline/cmd_module_atexit_exc.py.exp b/tests/cmdline/cmd_module_atexit_exc.py.exp new file mode 100644 index 0000000000000..6320d9d2d3011 --- /dev/null +++ b/tests/cmdline/cmd_module_atexit_exc.py.exp @@ -0,0 +1,3 @@ +['cmdline.cmd_module_atexit_exc', 'cmdline/cmd_module_atexit_exc.py'] +start +done diff --git a/tests/cmdline/cmd_sys_exit_0.py b/tests/cmdline/cmd_sys_exit_0.py new file mode 100644 index 0000000000000..1294b739e8ff1 --- /dev/null +++ b/tests/cmdline/cmd_sys_exit_0.py @@ -0,0 +1,5 @@ +# cmdline: +# test sys.exit(0) - success exit code +import sys + +sys.exit(0) diff --git a/tests/cmdline/cmd_sys_exit_0.py.exp b/tests/cmdline/cmd_sys_exit_0.py.exp new file mode 100644 index 0000000000000..8b137891791fe --- /dev/null +++ b/tests/cmdline/cmd_sys_exit_0.py.exp @@ -0,0 +1 @@ + diff --git a/tests/cmdline/cmd_sys_exit_error.py b/tests/cmdline/cmd_sys_exit_error.py new file mode 100644 index 0000000000000..ecac15e94f1bf --- /dev/null +++ b/tests/cmdline/cmd_sys_exit_error.py @@ -0,0 +1,5 @@ +# cmdline: +# test sys.exit() functionality and exit codes +import sys + +sys.exit(123) diff --git a/tests/cmdline/cmd_sys_exit_error.py.exp b/tests/cmdline/cmd_sys_exit_error.py.exp new file mode 100644 index 0000000000000..3911f71d6244d --- /dev/null +++ b/tests/cmdline/cmd_sys_exit_error.py.exp @@ -0,0 +1 @@ +CRASH \ No newline at end of file diff --git a/tests/cmdline/cmd_sys_exit_none.py b/tests/cmdline/cmd_sys_exit_none.py new file mode 100644 index 0000000000000..66e19666589ed --- /dev/null +++ b/tests/cmdline/cmd_sys_exit_none.py @@ -0,0 +1,5 @@ +# cmdline: +# test sys.exit(None) - should exit with code 0 +import sys + +sys.exit(None) diff --git a/tests/cmdline/cmd_sys_exit_none.py.exp b/tests/cmdline/cmd_sys_exit_none.py.exp new file mode 100644 index 0000000000000..8b137891791fe --- /dev/null +++ b/tests/cmdline/cmd_sys_exit_none.py.exp @@ -0,0 +1 @@ + diff --git a/tests/cmdline/repl_autocomplete.py.exp b/tests/cmdline/repl_autocomplete.py.exp index 2e2397bb028d3..7840e52e8a4a2 100644 --- a/tests/cmdline/repl_autocomplete.py.exp +++ b/tests/cmdline/repl_autocomplete.py.exp @@ -1,5 +1,5 @@ -CircuitPython \.\+ version -Use \.\+ + +Adafruit CircuitPython \.\+ version >>> # tests for autocompletion >>> import sys >>> not_exist. @@ -11,4 +11,4 @@ Use \.\+ >>> i.lower('ABC') 'abc' >>> None. ->>> +>>> \$ diff --git a/tests/cmdline/repl_autocomplete_underscore.py b/tests/cmdline/repl_autocomplete_underscore.py new file mode 100644 index 0000000000000..e685a7fe7ff02 --- /dev/null +++ b/tests/cmdline/repl_autocomplete_underscore.py @@ -0,0 +1,33 @@ +# Test REPL autocompletion filtering of underscore attributes + +# Start paste mode +{\x05} +class TestClass: + def __init__(self): + self.public_attr = 1 + self._private_attr = 2 + self.__very_private = 3 + + def public_method(self): + pass + + def _private_method(self): + pass + + @property + def public_property(self): + return 42 + + @property + def _private_property(self): + return 99 + +{\x04} +# Paste executed + +# Create an instance +obj = TestClass() + +# Test tab completion on the instance +# The tab character after `obj.` and 'a' below triggers the completions +obj.{\x09}{\x09}a{\x09} diff --git a/tests/cmdline/repl_autocomplete_underscore.py.exp b/tests/cmdline/repl_autocomplete_underscore.py.exp new file mode 100644 index 0000000000000..98e6c2aeb05a3 --- /dev/null +++ b/tests/cmdline/repl_autocomplete_underscore.py.exp @@ -0,0 +1,41 @@ + +Adafruit CircuitPython \.\+ version +>>> # Test REPL autocompletion filtering of underscore attributes +>>> \$ +>>> # Start paste mode +>>> \$ +paste mode; Ctrl-C to cancel, Ctrl-D to finish +=== \$ +=== class TestClass: +=== def __init__(self): +=== self.public_attr = 1 +=== self._private_attr = 2 +=== self.__very_private = 3 +=== \$ +=== def public_method(self): +=== pass +=== \$ +=== def _private_method(self): +=== pass +=== \$ +=== @property +=== def public_property(self): +=== return 42 +=== \$ +=== @property +=== def _private_property(self): +=== return 99 +=== \$ +=== \$ +>>> # Paste executed +>>> \$ +>>> # Create an instance +>>> obj = TestClass() +>>> \$ +>>> # Test tab completion on the instance +>>> # The tab character after `obj.` and 'a' below triggers the completions +>>> obj.public_ +public_attr public_method public_property +>>> obj.public_attr +1 +>>> \$ diff --git a/tests/cmdline/repl_autoindent.py.exp b/tests/cmdline/repl_autoindent.py.exp index 9ff83a92870e8..04f00686b208d 100644 --- a/tests/cmdline/repl_autoindent.py.exp +++ b/tests/cmdline/repl_autoindent.py.exp @@ -1,22 +1,22 @@ -CircuitPython \.\+ version -Use \.\+ + +Adafruit CircuitPython \.\+ version >>> # tests for autoindent >>> if 1: ... print(1) -... -... -... +... \$ +... \$ +... \$ 1 >>> if 0: ...  print(2) ... else: ... print(3) -... +... \$ 3 >>> if 0: ... print(4) ... else: ... print(5) -... +... \$ 5 ->>> +>>> \$ diff --git a/tests/cmdline/repl_basic.py.exp b/tests/cmdline/repl_basic.py.exp index 26442b6445589..5bdcc9d6d3246 100644 --- a/tests/cmdline/repl_basic.py.exp +++ b/tests/cmdline/repl_basic.py.exp @@ -1,5 +1,5 @@ -CircuitPython \.\+ version -Use \.\+ + +Adafruit CircuitPython \.\+ version >>> # basic REPL tests >>> print(1) 1 @@ -7,4 +7,4 @@ Use \.\+ 1 >>> 2 2 ->>> +>>> \$ diff --git a/tests/cmdline/repl_cont.py.exp b/tests/cmdline/repl_cont.py.exp index 6eed4a3e02c47..41f2436ac18a0 100644 --- a/tests/cmdline/repl_cont.py.exp +++ b/tests/cmdline/repl_cont.py.exp @@ -1,5 +1,5 @@ -CircuitPython \.\+ version -Use \.\+ + +Adafruit CircuitPython \.\+ version >>> # check REPL allows to continue input >>> 1 \\\\ ... + 2 @@ -54,4 +54,4 @@ two >>> if1 = 2 >>> print(if1) 2 ->>> +>>> \$ diff --git a/tests/cmdline/repl_emacs_keys.py.exp b/tests/cmdline/repl_emacs_keys.py.exp index 2e8667a8e6ca9..4979f8bbfc3ab 100644 --- a/tests/cmdline/repl_emacs_keys.py.exp +++ b/tests/cmdline/repl_emacs_keys.py.exp @@ -1,5 +1,5 @@ -CircuitPython \.\+ version -Use \.\+ + +Adafruit CircuitPython \.\+ version >>> # REPL tests of GNU-ish readline navigation >>> # history buffer navigation >>> 1 @@ -16,4 +16,4 @@ Use \.\+ >>> t = 121 >>> \.\+ 'foobar' ->>> +>>> \$ diff --git a/tests/cmdline/repl_inspect.py.exp b/tests/cmdline/repl_inspect.py.exp index 8ece5ffc37f5c..30257f31aab93 100644 --- a/tests/cmdline/repl_inspect.py.exp +++ b/tests/cmdline/repl_inspect.py.exp @@ -1,6 +1,6 @@ test -CircuitPython \.\+ version -Use \.\+ + +Adafruit CircuitPython \.\+ version >>> # cmdline: -i -c print("test") >>> # -c option combined with -i option results in REPL ->>> +>>> \$ diff --git a/tests/cmdline/repl_lock.py b/tests/cmdline/repl_lock.py new file mode 100644 index 0000000000000..77e2e9327401a --- /dev/null +++ b/tests/cmdline/repl_lock.py @@ -0,0 +1,7 @@ +import micropython +micropython.heap_lock() +1+1 +micropython.heap_lock() +None # Cause the repl's line storage to be enlarged ---------------- +micropython.heap_lock() +raise SystemExit diff --git a/tests/cmdline/repl_lock.py.exp b/tests/cmdline/repl_lock.py.exp new file mode 100644 index 0000000000000..e6d63fcf20044 --- /dev/null +++ b/tests/cmdline/repl_lock.py.exp @@ -0,0 +1,10 @@ + +Adafruit CircuitPython \.\+ version +>>> import micropython +>>> micropython.heap_lock() +>>> 1+1 +2 +>>> micropython.heap_lock() +>>> None # Cause the repl's line storage to be enlarged ---------------- +>>> micropython.heap_lock() +>>> raise SystemExit diff --git a/tests/cmdline/repl_micropyinspect.py.exp b/tests/cmdline/repl_micropyinspect.py.exp index 3c9cbc030c0b5..36bef37c62c6b 100644 --- a/tests/cmdline/repl_micropyinspect.py.exp +++ b/tests/cmdline/repl_micropyinspect.py.exp @@ -1,5 +1,5 @@ -CircuitPython \.\+ version -Use \.\+ + +Adafruit CircuitPython \.\+ version >>> # cmdline: cmdline/repl_micropyinspect >>> # setting MICROPYINSPECT environment variable before program exit triggers REPL ->>> +>>> \$ diff --git a/tests/cmdline/repl_paste.py b/tests/cmdline/repl_paste.py new file mode 100644 index 0000000000000..7cec450fce7bf --- /dev/null +++ b/tests/cmdline/repl_paste.py @@ -0,0 +1,90 @@ +# Test REPL paste mode functionality + +# Basic paste mode with a simple function +{\x05} +def hello(): + print('Hello from paste mode!') +hello() +{\x04} + +# Paste mode with multiple indentation levels +{\x05} +def calculate(n): + if n > 0: + for i in range(n): + if i % 2 == 0: + print(f'Even: {i}') + else: + print(f'Odd: {i}') + else: + print('n must be positive') + +calculate(5) +{\x04} + +# Paste mode with blank lines +{\x05} +def function_with_blanks(): + print('First line') + + print('After blank line') + + + print('After two blank lines') + +function_with_blanks() +{\x04} + +# Paste mode with class definition and multiple methods +{\x05} +class TestClass: + def __init__(self, value): + self.value = value + + def display(self): + print(f'Value is: {self.value}') + + def double(self): + self.value *= 2 + return self.value + +obj = TestClass(21) +obj.display() +print(f'Doubled: {obj.double()}') +obj.display() +{\x04} + +# Paste mode with exception handling +{\x05} +try: + x = 1 / 0 +except ZeroDivisionError: + print('Caught division by zero') +finally: + print('Finally block executed') +{\x04} + +# Cancel paste mode with Ctrl-C +{\x05} +print('This should not execute') +{\x03} + +# Normal REPL still works after cancelled paste +print('Back to normal REPL') + +# Paste mode with syntax error +{\x05} +def bad_syntax(: + print('Missing parameter') +{\x04} + +# Paste mode with runtime error +{\x05} +def will_error(): + undefined_variable + +will_error() +{\x04} + +# Final test to show REPL is still functioning +1 + 2 + 3 diff --git a/tests/cmdline/repl_paste.py.exp b/tests/cmdline/repl_paste.py.exp new file mode 100644 index 0000000000000..cc5ac2f08007d --- /dev/null +++ b/tests/cmdline/repl_paste.py.exp @@ -0,0 +1,133 @@ + +Adafruit CircuitPython \.\+ version +>>> # Test REPL paste mode functionality +>>> \$ +>>> # Basic paste mode with a simple function +>>> \$ +paste mode; Ctrl-C to cancel, Ctrl-D to finish +=== \$ +=== def hello(): +=== print('Hello from paste mode!') +=== hello() +=== \$ +Hello from paste mode! +>>> \$ +>>> # Paste mode with multiple indentation levels +>>> \$ +paste mode; Ctrl-C to cancel, Ctrl-D to finish +=== \$ +=== def calculate(n): +=== if n > 0: +=== for i in range(n): +=== if i % 2 == 0: +=== print(f'Even: {i}') +=== else: +=== print(f'Odd: {i}') +=== else: +=== print('n must be positive') +=== \$ +=== calculate(5) +=== \$ +Even: 0 +Odd: 1 +Even: 2 +Odd: 3 +Even: 4 +>>> \$ +>>> # Paste mode with blank lines +>>> \$ +paste mode; Ctrl-C to cancel, Ctrl-D to finish +=== \$ +=== def function_with_blanks(): +=== print('First line') +=== \$ +=== print('After blank line') +=== \$ +=== \$ +=== print('After two blank lines') +=== \$ +=== function_with_blanks() +=== \$ +First line +After blank line +After two blank lines +>>> \$ +>>> # Paste mode with class definition and multiple methods +>>> \$ +paste mode; Ctrl-C to cancel, Ctrl-D to finish +=== \$ +=== class TestClass: +=== def __init__(self, value): +=== self.value = value +=== \$ +=== def display(self): +=== print(f'Value is: {self.value}') +=== \$ +=== def double(self): +=== self.value *= 2 +=== return self.value +=== \$ +=== obj = TestClass(21) +=== obj.display() +=== print(f'Doubled: {obj.double()}') +=== obj.display() +=== \$ +Value is: 21 +Doubled: 42 +Value is: 42 +>>> \$ +>>> # Paste mode with exception handling +>>> \$ +paste mode; Ctrl-C to cancel, Ctrl-D to finish +=== \$ +=== try: +=== x = 1 / 0 +=== except ZeroDivisionError: +=== print('Caught division by zero') +=== finally: +=== print('Finally block executed') +=== \$ +Caught division by zero +Finally block executed +>>> \$ +>>> # Cancel paste mode with Ctrl-C +>>> \$ +paste mode; Ctrl-C to cancel, Ctrl-D to finish +=== \$ +=== print('This should not execute') +=== \$ +>>> \$ +>>> \$ +>>> # Normal REPL still works after cancelled paste +>>> print('Back to normal REPL') +Back to normal REPL +>>> \$ +>>> # Paste mode with syntax error +>>> \$ +paste mode; Ctrl-C to cancel, Ctrl-D to finish +=== \$ +=== def bad_syntax(: +=== print('Missing parameter') +=== \$ +Traceback (most recent call last): + File "", line 2 +SyntaxError: invalid syntax +>>> \$ +>>> # Paste mode with runtime error +>>> \$ +paste mode; Ctrl-C to cancel, Ctrl-D to finish +=== \$ +=== def will_error(): +=== undefined_variable +=== \$ +=== will_error() +=== \$ +Traceback (most recent call last): + File "", line 5, in + File "", line 3, in will_error +NameError: name 'undefined_variable' isn't defined +>>> \$ +>>> # Final test to show REPL is still functioning +>>> 1 + 2 + 3 +6 +>>> \$ diff --git a/tests/cmdline/repl_sys_ps1_ps2.py.exp b/tests/cmdline/repl_sys_ps1_ps2.py.exp index 452a54fe5ae9b..d4bcf7a44d592 100644 --- a/tests/cmdline/repl_sys_ps1_ps2.py.exp +++ b/tests/cmdline/repl_sys_ps1_ps2.py.exp @@ -1,5 +1,5 @@ -CircuitPython \.\+ version -Use \.\+ + +Adafruit CircuitPython \.\+ version >>> # test changing ps1/ps2 >>> import sys >>> sys.ps1 = "PS1" diff --git a/tests/cmdline/repl_words_move.py.exp b/tests/cmdline/repl_words_move.py.exp index ba5c3648cff98..41ad3d8e73351 100644 --- a/tests/cmdline/repl_words_move.py.exp +++ b/tests/cmdline/repl_words_move.py.exp @@ -1,5 +1,5 @@ -CircuitPython \.\+ version -Use \.\+ + +Adafruit CircuitPython \.\+ version >>> # word movement >>> # backward-word, start in word >>> \.\+ @@ -19,7 +19,7 @@ Use \.\+ >>> # forward-word on eol. if cursor is moved, this will result in a SyntaxError >>> \.\+ 6 ->>> +>>> \$ >>> # kill word >>> # backward-kill-word, start in word >>> \.\+ @@ -33,7 +33,7 @@ Use \.\+ >>> # forward-kill-word, don't start in word >>> \.\+ 3 ->>> +>>> \$ >>> # extra move/kill shortcuts >>> # ctrl-left >>> \.\+ @@ -44,4 +44,4 @@ Use \.\+ >>> # ctrl-w >>> \.\+ 1 ->>> +>>> \$ diff --git a/tests/cpydiff/core_class_initsubclass.py b/tests/cpydiff/core_class_initsubclass.py new file mode 100644 index 0000000000000..8683271dcb0f2 --- /dev/null +++ b/tests/cpydiff/core_class_initsubclass.py @@ -0,0 +1,21 @@ +""" +categories: Core,Classes +description: ``__init_subclass__`` isn't automatically called. +cause: MicroPython does not currently implement PEP 487. +workaround: Manually call ``__init_subclass__`` after class creation if needed. e.g.:: + + class A(Base): + pass + A.__init_subclass__() + +""" + + +class Base: + @classmethod + def __init_subclass__(cls): + print(f"Base.__init_subclass__({cls.__name__})") + + +class A(Base): + pass diff --git a/tests/cpydiff/core_class_initsubclass_autoclassmethod.py b/tests/cpydiff/core_class_initsubclass_autoclassmethod.py new file mode 100644 index 0000000000000..b2f7628976c92 --- /dev/null +++ b/tests/cpydiff/core_class_initsubclass_autoclassmethod.py @@ -0,0 +1,31 @@ +""" +categories: Core,Classes +description: ``__init_subclass__`` isn't an implicit classmethod. +cause: MicroPython does not currently implement PEP 487. ``__init_subclass__`` is not currently in the list of special-cased class/static methods. +workaround: Decorate declarations of ``__init_subclass__`` with ``@classmethod``. +""" + + +def regularize_spelling(text, prefix="bound_"): + # for regularizing across the CPython "method" vs MicroPython "bound_method" spelling for the type of a bound classmethod + if text.startswith(prefix): + return text[len(prefix) :] + return text + + +class A: + def __init_subclass__(cls): + pass + + @classmethod + def manual_decorated(cls): + pass + + +a = type(A.__init_subclass__).__name__ +b = type(A.manual_decorated).__name__ + +print(regularize_spelling(a)) +print(regularize_spelling(b)) +if a != b: + print("FAIL") diff --git a/tests/cpydiff/core_class_initsubclass_kwargs.py b/tests/cpydiff/core_class_initsubclass_kwargs.py new file mode 100644 index 0000000000000..ed5157afeaede --- /dev/null +++ b/tests/cpydiff/core_class_initsubclass_kwargs.py @@ -0,0 +1,22 @@ +""" +categories: Core,Classes +description: MicroPython doesn't support parameterized ``__init_subclass__`` class customization. +cause: MicroPython does not currently implement PEP 487. The MicroPython syntax tree does not include a kwargs node after the class inheritance list. +workaround: Use class variables or another mechanism to specify base-class customizations. +""" + + +class Base: + @classmethod + def __init_subclass__(cls, arg=None, **kwargs): + cls.init_subclass_was_called = True + print(f"Base.__init_subclass__({cls.__name__}, {arg=!r}, {kwargs=!r})") + + +class A(Base, arg="arg"): + pass + + +# Regularize across MicroPython not automatically calling __init_subclass__ either. +if not getattr(A, "init_subclass_was_called", False): + A.__init_subclass__() diff --git a/tests/cpydiff/core_class_initsubclass_super.py b/tests/cpydiff/core_class_initsubclass_super.py new file mode 100644 index 0000000000000..d18671a74bf8e --- /dev/null +++ b/tests/cpydiff/core_class_initsubclass_super.py @@ -0,0 +1,22 @@ +""" +categories: Core,Classes +description: ``__init_subclass__`` can't be defined a cooperatively-recursive way. +cause: MicroPython does not currently implement PEP 487. The base object type does not have an ``__init_subclass__`` implementation. +workaround: Omit the recursive ``__init_subclass__`` call unless it's known that the grandparent also defines it. +""" + + +class Base: + @classmethod + def __init_subclass__(cls, **kwargs): + cls.init_subclass_was_called = True + super().__init_subclass__(**kwargs) + + +class A(Base): + pass + + +# Regularize across MicroPython not automatically calling __init_subclass__ either. +if not getattr(A, "init_subclass_was_called", False): + A.__init_subclass__() diff --git a/tests/cpydiff/core_fstring_concat.py b/tests/cpydiff/core_fstring_concat.py index 3daa13d75360e..2fbe1b961a1e6 100644 --- a/tests/cpydiff/core_fstring_concat.py +++ b/tests/cpydiff/core_fstring_concat.py @@ -1,5 +1,5 @@ """ -categories: Core +categories: Core,f-strings description: f-strings don't support concatenation with adjacent literals if the adjacent literals contain braces cause: MicroPython is optimised for code space. workaround: Use the + operator between literal strings when they are not both f-strings diff --git a/tests/cpydiff/core_fstring_parser.py b/tests/cpydiff/core_fstring_parser.py index 87cf1e63ed8c2..4964b9707aceb 100644 --- a/tests/cpydiff/core_fstring_parser.py +++ b/tests/cpydiff/core_fstring_parser.py @@ -1,5 +1,5 @@ """ -categories: Core +categories: Core,f-strings description: f-strings cannot support expressions that require parsing to resolve unbalanced nested braces and brackets cause: MicroPython is optimised for code space. workaround: Always use balanced braces and brackets in expressions inside f-strings diff --git a/tests/cpydiff/core_fstring_repr.py b/tests/cpydiff/core_fstring_repr.py index d37fb48db758c..2589a34b7e3b9 100644 --- a/tests/cpydiff/core_fstring_repr.py +++ b/tests/cpydiff/core_fstring_repr.py @@ -1,5 +1,5 @@ """ -categories: Core +categories: Core,f-strings description: f-strings don't support !a conversions cause: MicropPython does not implement ascii() workaround: None diff --git a/tests/cpydiff/core_import_all.py b/tests/cpydiff/core_import_all.py deleted file mode 100644 index 0fbe9d4d4ecd6..0000000000000 --- a/tests/cpydiff/core_import_all.py +++ /dev/null @@ -1,10 +0,0 @@ -""" -categories: Core,import -description: __all__ is unsupported in __init__.py in MicroPython. -cause: Not implemented. -workaround: Manually import the sub-modules directly in __init__.py using ``from . import foo, bar``. -""" - -from modules3 import * - -foo.hello() diff --git a/tests/cpydiff/modules3/__init__.py b/tests/cpydiff/modules3/__init__.py deleted file mode 100644 index 27a2bf2ad9044..0000000000000 --- a/tests/cpydiff/modules3/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__all__ = ["foo"] diff --git a/tests/cpydiff/modules3/foo.py b/tests/cpydiff/modules3/foo.py deleted file mode 100644 index dd9b9d4ddd4c4..0000000000000 --- a/tests/cpydiff/modules3/foo.py +++ /dev/null @@ -1,2 +0,0 @@ -def hello(): - print("hello") diff --git a/tests/cpydiff/modules_errno_enotsup.py b/tests/cpydiff/modules_errno_enotsup.py new file mode 100644 index 0000000000000..80e5ad9d03220 --- /dev/null +++ b/tests/cpydiff/modules_errno_enotsup.py @@ -0,0 +1,10 @@ +""" +categories: Modules,errno +description: MicroPython does not include ``ENOTSUP`` as a name for errno 95. +cause: MicroPython does not implement the ``ENOTSUP`` canonical alias for ``EOPNOTSUPP`` added in CPython 3.2. +workaround: Use ``errno.EOPNOTSUPP`` in place of ``errno.ENOTSUP``. +""" + +import errno + +print(f"{errno.errorcode[errno.EOPNOTSUPP]=!s}") diff --git a/tests/cpydiff/modules_struct_fewargs.py b/tests/cpydiff/modules_struct_fewargs.py index 49b2a3213c898..f6346a67938ed 100644 --- a/tests/cpydiff/modules_struct_fewargs.py +++ b/tests/cpydiff/modules_struct_fewargs.py @@ -1,6 +1,6 @@ """ categories: Modules,struct -description: Struct pack with too few args, not checked by uPy +description: Struct pack with too few args, not checked by MicroPython cause: Unknown workaround: Unknown """ diff --git a/tests/cpydiff/modules_struct_manyargs.py b/tests/cpydiff/modules_struct_manyargs.py index e3b78930f219f..b2ea93b6c9330 100644 --- a/tests/cpydiff/modules_struct_manyargs.py +++ b/tests/cpydiff/modules_struct_manyargs.py @@ -1,6 +1,6 @@ """ categories: Modules,struct -description: Struct pack with too many args, not checked by uPy +description: Struct pack with too many args, not checked by MicroPython cause: Unknown workaround: Unknown """ diff --git a/tests/cpydiff/modules_struct_whitespace_in_format.py b/tests/cpydiff/modules_struct_whitespace_in_format.py index a7a1d2facdfff..8b609425eb01c 100644 --- a/tests/cpydiff/modules_struct_whitespace_in_format.py +++ b/tests/cpydiff/modules_struct_whitespace_in_format.py @@ -1,6 +1,6 @@ """ categories: Modules,struct -description: Struct pack with whitespace in format, whitespace ignored by CPython, error on uPy +description: Struct pack with whitespace in format, whitespace ignored by CPython, error on MicroPython cause: MicroPython is optimised for code size. workaround: Don't use spaces in format strings. """ diff --git a/tests/cpydiff/syntax_arg_unpacking.py b/tests/cpydiff/syntax_arg_unpacking.py index e54832ddb9165..7133a8a28272c 100644 --- a/tests/cpydiff/syntax_arg_unpacking.py +++ b/tests/cpydiff/syntax_arg_unpacking.py @@ -1,5 +1,5 @@ """ -categories: Syntax +categories: Syntax,Unpacking description: Argument unpacking does not work if the argument being unpacked is the nth or greater argument where n is the number of bits in an MP_SMALL_INT. cause: The implementation uses an MP_SMALL_INT to flag args that need to be unpacked. workaround: Use fewer arguments. diff --git a/tests/cpydiff/syntax_literal_underscore.py b/tests/cpydiff/syntax_literal_underscore.py new file mode 100644 index 0000000000000..4b1406e9f3f71 --- /dev/null +++ b/tests/cpydiff/syntax_literal_underscore.py @@ -0,0 +1,19 @@ +""" +categories: Syntax,Literals +description: MicroPython accepts underscores in numeric literals where CPython doesn't +cause: Different parser implementation + +MicroPython's tokenizer ignores underscores in numeric literals, while CPython +rejects multiple consecutive underscores and underscores after the last digit. + +workaround: Remove the underscores not accepted by CPython. +""" + +try: + print(eval("1__1")) +except SyntaxError: + print("Should not work") +try: + print(eval("1_")) +except SyntaxError: + print("Should not work") diff --git a/tests/cpydiff/syntax_spaces.py b/tests/cpydiff/syntax_spaces.py index 03d25d56199d1..670cefdeac23e 100644 --- a/tests/cpydiff/syntax_spaces.py +++ b/tests/cpydiff/syntax_spaces.py @@ -1,8 +1,15 @@ """ -categories: Syntax,Spaces -description: uPy requires spaces between literal numbers and keywords, CPy doesn't -cause: Unknown -workaround: Unknown +categories: Syntax,Literals +description: MicroPython requires spaces between literal numbers and keywords or ".", CPython doesn't +cause: Different parser implementation + +MicroPython's tokenizer treats a sequence like ``1and`` as a single token, while CPython treats it as two tokens. + +Since CPython 3.11, when the literal number is followed by a token, this syntax causes a ``SyntaxWarning`` for an "invalid literal". When a literal number is followed by a "." denoting attribute access, CPython does not warn. + +workaround: Add a space between the integer literal and the intended next token. + +This also fixes the ``SyntaxWarning`` in CPython. """ try: @@ -17,3 +24,7 @@ print(eval("1if 1else 0")) except SyntaxError: print("Should have worked") +try: + print(eval("0x1.to_bytes(1)")) +except SyntaxError: + print("Should have worked") diff --git a/tests/cpydiff/types_complex_parser.py b/tests/cpydiff/types_complex_parser.py new file mode 100644 index 0000000000000..4a012987d9e48 --- /dev/null +++ b/tests/cpydiff/types_complex_parser.py @@ -0,0 +1,14 @@ +""" +categories: Types,complex +description: MicroPython's complex() accepts certain incorrect values that CPython rejects +cause: MicroPython is highly optimized for memory usage. +workaround: Do not use non-standard complex literals as argument to complex() + +MicroPython's ``complex()`` function accepts literals that contain a space and +no sign between the real and imaginary parts, and interprets it as a plus. +""" + +try: + print(complex("1 1j")) +except ValueError: + print("ValueError") diff --git a/tests/cpydiff/types_float_implicit_conversion.py b/tests/cpydiff/types_float_implicit_conversion.py index 3726839fac6d1..764c9e4e6ed7a 100644 --- a/tests/cpydiff/types_float_implicit_conversion.py +++ b/tests/cpydiff/types_float_implicit_conversion.py @@ -1,6 +1,6 @@ """ categories: Types,float -description: uPy allows implicit conversion of objects in maths operations while CPython does not. +description: MicroPython allows implicit conversion of objects in maths operations while CPython does not. cause: Unknown workaround: Objects should be wrapped in ``float(obj)`` for compatibility with CPython. """ diff --git a/tests/cpydiff/types_float_rounding.py b/tests/cpydiff/types_float_rounding.py deleted file mode 100644 index 206e359ed9be7..0000000000000 --- a/tests/cpydiff/types_float_rounding.py +++ /dev/null @@ -1,8 +0,0 @@ -""" -categories: Types,float -description: uPy and CPython outputs formats may differ -cause: Unknown -workaround: Unknown -""" - -print("%.1g" % -9.9) diff --git a/tests/cpydiff/types_oserror_errnomap.py b/tests/cpydiff/types_oserror_errnomap.py new file mode 100644 index 0000000000000..6627bd2af4ab7 --- /dev/null +++ b/tests/cpydiff/types_oserror_errnomap.py @@ -0,0 +1,48 @@ +""" +categories: Types,OSError +description: OSError constructor returns a plain OSError for all errno values, rather than a relevant subtype. +cause: MicroPython does not include the CPython-standard OSError subclasses. +workaround: Catch OSError and use its errno attribute to discriminate the cause. +""" + +import errno + +errno_list = [ # i.e. the set implemented by micropython + errno.EPERM, + errno.ENOENT, + errno.EIO, + errno.EBADF, + errno.EAGAIN, + errno.ENOMEM, + errno.EACCES, + errno.EEXIST, + errno.ENODEV, + errno.EISDIR, + errno.EINVAL, + errno.EOPNOTSUPP, + errno.EADDRINUSE, + errno.ECONNABORTED, + errno.ECONNRESET, + errno.ENOBUFS, + errno.ENOTCONN, + errno.ETIMEDOUT, + errno.ECONNREFUSED, + errno.EHOSTUNREACH, + errno.EALREADY, + errno.EINPROGRESS, +] + + +def errno_output_type(n): + try: + raise OSError(n, "") + except OSError as e: + return f"{type(e).__name__}" + except Exception as e: + return f"non-OSError {type(e).__name__}" + else: + return "no error" + + +for n in errno_list: + print(errno.errorcode[n], "=", errno_output_type(n)) diff --git a/tests/cpydiff/types_range_limits.py b/tests/cpydiff/types_range_limits.py new file mode 100644 index 0000000000000..e53d5fd4088f3 --- /dev/null +++ b/tests/cpydiff/types_range_limits.py @@ -0,0 +1,26 @@ +""" +categories: Types,range +description: Range objects with large start or stop arguments misbehave. +cause: Intermediate calculations overflow the C mp_int_t type +workaround: Avoid using such ranges +""" + +from sys import maxsize + +# A range including `maxsize-1` cannot be created +try: + print(range(-maxsize - 1, 0)) +except OverflowError: + print("OverflowError") + +# A range with `stop-start` exceeding sys.maxsize has incorrect len(), while CPython cannot calculate len(). +try: + print(len(range(-maxsize, maxsize))) +except OverflowError: + print("OverflowError") + +# A range with `stop-start` exceeding sys.maxsize has incorrect len() +try: + print(len(range(-maxsize, maxsize, maxsize))) +except OverflowError: + print("OverflowError") diff --git a/tests/cpydiff/types_str_formatsep.py b/tests/cpydiff/types_str_formatsep.py new file mode 100644 index 0000000000000..05d0b8d3d2cf4 --- /dev/null +++ b/tests/cpydiff/types_str_formatsep.py @@ -0,0 +1,19 @@ +""" +categories: Types,str +description: MicroPython accepts the "," grouping option with any radix, unlike CPython +cause: To reduce code size, MicroPython does not issue an error for this combination +workaround: Do not use a format string like ``{:,b}`` if CPython compatibility is required. +""" + +try: + print("{:,b}".format(99)) +except ValueError: + print("ValueError") +try: + print("{:,x}".format(99)) +except ValueError: + print("ValueError") +try: + print("{:,o}".format(99)) +except ValueError: + print("ValueError") diff --git a/tests/cpydiff/types_str_formatsep_float.py b/tests/cpydiff/types_str_formatsep_float.py new file mode 100644 index 0000000000000..b487cd3758e73 --- /dev/null +++ b/tests/cpydiff/types_str_formatsep_float.py @@ -0,0 +1,11 @@ +""" +categories: Types,str +description: MicroPython accepts but does not properly implement the "," or "_" grouping character for float values +cause: To reduce code size, MicroPython does not implement this combination. Grouping characters will not appear in the number's significant digits and will appear at incorrect locations in leading zeros. +workaround: Do not use a format string like ``{:,f}`` if exact CPython compatibility is required. +""" + +print("{:,f}".format(3141.159)) +print("{:_f}".format(3141.159)) +print("{:011,.2f}".format(3141.159)) +print("{:011_.2f}".format(3141.159)) diff --git a/tests/extmod/asyncio_basic.py.exp b/tests/extmod/asyncio_basic.py.exp deleted file mode 100644 index 478e22abc8ff6..0000000000000 --- a/tests/extmod/asyncio_basic.py.exp +++ /dev/null @@ -1,6 +0,0 @@ -start -after sleep -short -long -negative -took 200 400 0 diff --git a/tests/extmod/asyncio_event_queue.py b/tests/extmod/asyncio_event_queue.py new file mode 100644 index 0000000000000..e0125b1aefe13 --- /dev/null +++ b/tests/extmod/asyncio_event_queue.py @@ -0,0 +1,64 @@ +# Ensure that an asyncio task can wait on an Event when the +# _task_queue is empty +# https://github.com/micropython/micropython/issues/16569 + +try: + import asyncio +except ImportError: + print("SKIP") + raise SystemExit + +# This test requires checking that the asyncio scheduler +# remains active "indefinitely" when the task queue is empty. +# +# To check this, we need another independent scheduler that +# can wait for a certain amount of time. So we have to +# create one using micropython.schedule() and time.ticks_ms() +# +# Technically, this code breaks the rules, as it is clearly +# documented that Event.set() should _NOT_ be called from a +# schedule (soft IRQ) because in some cases, a race condition +# can occur, resulting in a crash. However: +# - since the risk of a race condition in that specific +# case has been analysed and excluded +# - given that there is no other simple alternative to +# write this test case, +# an exception to the rule was deemed acceptable. See +# https://github.com/micropython/micropython/pull/16772 + +import micropython, time + +try: + micropython.schedule +except AttributeError: + print("SKIP") + raise SystemExit + + +evt = asyncio.Event() + + +def schedule_watchdog(end_ticks): + if time.ticks_diff(end_ticks, time.ticks_ms()) <= 0: + print("asyncio still pending, unlocking event") + # Caution: about to call Event.set() from a schedule + # (see the note in the comment above) + evt.set() + return + micropython.schedule(schedule_watchdog, end_ticks) + + +async def foo(): + print("foo waiting") + schedule_watchdog(time.ticks_add(time.ticks_ms(), 100)) + await evt.wait() + print("foo done") + + +async def main(): + print("main started") + await foo() + print("main done") + + +asyncio.run(main()) diff --git a/tests/extmod/asyncio_event_queue.py.exp b/tests/extmod/asyncio_event_queue.py.exp new file mode 100644 index 0000000000000..ee42c96d83ed8 --- /dev/null +++ b/tests/extmod/asyncio_event_queue.py.exp @@ -0,0 +1,5 @@ +main started +foo waiting +asyncio still pending, unlocking event +foo done +main done diff --git a/tests/extmod/asyncio_heaplock.py b/tests/extmod/asyncio_heaplock.py index 8326443f0e6c5..9e9908de1cb6a 100644 --- a/tests/extmod/asyncio_heaplock.py +++ b/tests/extmod/asyncio_heaplock.py @@ -4,7 +4,11 @@ # - StreamWriter.write, stream is blocked and data to write is a bytes object # - StreamWriter.write, when stream is not blocked -import micropython +try: + import asyncio, micropython +except ImportError: + print("SKIP") + raise SystemExit # strict stackless builds can't call functions without allocating a frame on the heap try: @@ -24,12 +28,6 @@ def f(x): print("SKIP") raise SystemExit -try: - import asyncio -except ImportError: - print("SKIP") - raise SystemExit - class TestStream: def __init__(self, blocked): diff --git a/tests/extmod/asyncio_iterator_event.py b/tests/extmod/asyncio_iterator_event.py new file mode 100644 index 0000000000000..f61fefcf051be --- /dev/null +++ b/tests/extmod/asyncio_iterator_event.py @@ -0,0 +1,86 @@ +# Ensure that an asyncio task can wait on an Event when the +# _task_queue is empty, in the context of an async iterator +# https://github.com/micropython/micropython/issues/16318 + +try: + import asyncio +except ImportError: + print("SKIP") + raise SystemExit + +# This test requires checking that the asyncio scheduler +# remains active "indefinitely" when the task queue is empty. +# +# To check this, we need another independent scheduler that +# can wait for a certain amount of time. So we have to +# create one using micropython.schedule() and time.ticks_ms() +# +# Technically, this code breaks the rules, as it is clearly +# documented that Event.set() should _NOT_ be called from a +# schedule (soft IRQ) because in some cases, a race condition +# can occur, resulting in a crash. However: +# - since the risk of a race condition in that specific +# case has been analysed and excluded +# - given that there is no other simple alternative to +# write this test case, +# an exception to the rule was deemed acceptable. See +# https://github.com/micropython/micropython/pull/16772 + +import micropython, time + +try: + micropython.schedule +except AttributeError: + print("SKIP") + raise SystemExit + +ai = None + + +def schedule_watchdog(end_ticks): + if time.ticks_diff(end_ticks, time.ticks_ms()) <= 0: + print("good: asyncio iterator is still pending, exiting") + # Caution: ai.fetch_data() will invoke Event.set() + # (see the note in the comment above) + ai.fetch_data(None) + return + micropython.schedule(schedule_watchdog, end_ticks) + + +async def test(ai): + for x in range(3): + await asyncio.sleep(0.1) + ai.fetch_data("bar {}".format(x)) + + +class AsyncIterable: + def __init__(self): + self.message = None + self.evt = asyncio.Event() + + def __aiter__(self): + return self + + async def __anext__(self): + await self.evt.wait() + self.evt.clear() + if self.message is None: + raise StopAsyncIteration + return self.message + + def fetch_data(self, message): + self.message = message + self.evt.set() + + +async def main(): + global ai + ai = AsyncIterable() + asyncio.create_task(test(ai)) + schedule_watchdog(time.ticks_add(time.ticks_ms(), 500)) + async for message in ai: + print(message) + print("end main") + + +asyncio.run(main()) diff --git a/tests/extmod/asyncio_iterator_event.py.exp b/tests/extmod/asyncio_iterator_event.py.exp new file mode 100644 index 0000000000000..a1893197d02ef --- /dev/null +++ b/tests/extmod/asyncio_iterator_event.py.exp @@ -0,0 +1,5 @@ +bar 0 +bar 1 +bar 2 +good: asyncio iterator is still pending, exiting +end main diff --git a/tests/extmod/asyncio_lock.py.exp b/tests/extmod/asyncio_lock.py.exp deleted file mode 100644 index a37dfcbd2e519..0000000000000 --- a/tests/extmod/asyncio_lock.py.exp +++ /dev/null @@ -1,41 +0,0 @@ -False -True -False -have lock ----- -task start 1 -task start 2 -task start 3 -task have 1 0 -task have 2 0 -task have 3 0 -task have 1 1 -task have 2 1 -task have 3 1 -task have 1 2 -task end 1 -task have 2 2 -task end 2 -task have 3 2 -task end 3 ----- -task have True -task release False -task have True -task release False -task have again -task have again ----- -task got 0 -task release 0 -task cancel 1 -task got 2 -task release 2 -False ----- -task got 0 -task cancel 1 -task release 0 -task got 2 -task cancel 2 -False diff --git a/tests/extmod/asyncio_set_exception_handler.py b/tests/extmod/asyncio_set_exception_handler.py index 5935f0f4ebeaa..0ac4a624224ce 100644 --- a/tests/extmod/asyncio_set_exception_handler.py +++ b/tests/extmod/asyncio_set_exception_handler.py @@ -12,7 +12,7 @@ def custom_handler(loop, context): async def task(i): - # Raise with 2 args so exception prints the same in uPy and CPython + # Raise with 2 args so exception prints the same in MicroPython and CPython raise ValueError(i, i + 1) diff --git a/tests/extmod/asyncio_wait_for_linked_task.py b/tests/extmod/asyncio_wait_for_linked_task.py new file mode 100644 index 0000000000000..4dda62d5476c7 --- /dev/null +++ b/tests/extmod/asyncio_wait_for_linked_task.py @@ -0,0 +1,66 @@ +# Test asyncio.wait_for, with dependent tasks +# https://github.com/micropython/micropython/issues/16759 + +try: + import asyncio +except ImportError: + print("SKIP") + raise SystemExit + + +# CPython 3.12 deprecated calling get_event_loop() when there is no current event +# loop, so to make this test run on CPython requires setting the event loop. +if hasattr(asyncio, "set_event_loop"): + asyncio.set_event_loop(asyncio.new_event_loop()) + + +class Worker: + def __init__(self): + self._eventLoop = None + self._tasks = [] + + def launchTask(self, asyncJob): + if self._eventLoop is None: + self._eventLoop = asyncio.get_event_loop() + return self._eventLoop.create_task(asyncJob) + + async def job(self, prerequisite, taskName): + if prerequisite: + await prerequisite + await asyncio.sleep(0.1) + print(taskName, "work completed") + + def planTasks(self): + self._tasks.append(self.launchTask(self.job(None, "task0"))) + self._tasks.append(self.launchTask(self.job(self._tasks[0], "task1"))) + self._tasks.append(self.launchTask(self.job(self._tasks[1], "task2"))) + + async def waitForTask(self, taskIdx): + return await self._tasks[taskIdx] + + def syncWaitForTask(self, taskIdx): + return self._eventLoop.run_until_complete(self._tasks[taskIdx]) + + +async def async_test(): + print("--- async test") + worker = Worker() + worker.planTasks() + await worker.waitForTask(0) + print("-> task0 done") + await worker.waitForTask(2) + print("-> task2 done") + + +def sync_test(): + print("--- sync test") + worker = Worker() + worker.planTasks() + worker.syncWaitForTask(0) + print("-> task0 done") + worker.syncWaitForTask(2) + print("-> task2 done") + + +asyncio.get_event_loop().run_until_complete(async_test()) +sync_test() diff --git a/tests/extmod/asyncio_wait_task.py.exp b/tests/extmod/asyncio_wait_task.py.exp deleted file mode 100644 index 514a434223315..0000000000000 --- a/tests/extmod/asyncio_wait_task.py.exp +++ /dev/null @@ -1,12 +0,0 @@ -start -task 1 -task 2 ----- -start -hello -world -took 200 200 -task_raise -ValueError -task_raise -ValueError diff --git a/tests/extmod/binascii_hexlify.py b/tests/extmod/binascii_hexlify.py index d06029aabaffb..ae90b67336586 100644 --- a/tests/extmod/binascii_hexlify.py +++ b/tests/extmod/binascii_hexlify.py @@ -1,5 +1,5 @@ try: - import binascii + from binascii import hexlify except ImportError: print("SKIP") raise SystemExit @@ -10,10 +10,10 @@ b"\x7f\x80\xff", b"1234ABCDabcd", ): - print(binascii.hexlify(x)) + print(hexlify(x)) # Two-argument version (now supported in CPython) -print(binascii.hexlify(b"123", ":")) +print(hexlify(b"123", ":")) # zero length buffer -print(binascii.hexlify(b"", b":")) +print(hexlify(b"", b":")) diff --git a/tests/extmod/binascii_unhexlify.py b/tests/extmod/binascii_unhexlify.py index 731b6a2bd4d10..b08704cba65b2 100644 --- a/tests/extmod/binascii_unhexlify.py +++ b/tests/extmod/binascii_unhexlify.py @@ -1,5 +1,5 @@ try: - import binascii + from binascii import unhexlify except ImportError: print("SKIP") raise SystemExit @@ -10,18 +10,18 @@ b"7f80ff", b"313233344142434461626364", ): - print(binascii.unhexlify(x)) + print(unhexlify(x)) # CIRCUITPY-CHANGE # Unicode strings can be decoded -print(binascii.unhexlify("313233344142434461626364")) +print(unhexlify("313233344142434461626364")) try: - a = binascii.unhexlify(b"0") # odd buffer length + a = unhexlify(b"0") # odd buffer length except ValueError: print("ValueError") try: - a = binascii.unhexlify(b"gg") # digit not hex + a = unhexlify(b"gg") # digit not hex except ValueError: print("ValueError") diff --git a/tests/extmod/framebuf_blit.py b/tests/extmod/framebuf_blit.py new file mode 100644 index 0000000000000..b1d98b330a838 --- /dev/null +++ b/tests/extmod/framebuf_blit.py @@ -0,0 +1,68 @@ +# Test FrameBuffer.blit method. + +try: + import framebuf +except ImportError: + print("SKIP") + raise SystemExit + + +def printbuf(): + print("--8<--") + for y in range(h): + for x in range(w): + print("%02x" % buf[(x + y * w)], end="") + print() + print("-->8--") + + +w = 5 +h = 4 +buf = bytearray(w * h) +fbuf = framebuf.FrameBuffer(buf, w, h, framebuf.GS8) + +fbuf2 = framebuf.FrameBuffer(bytearray(4), 2, 2, framebuf.GS8) +fbuf2.fill(0xFF) + +# Blit another FrameBuffer, at various locations. +for x, y in ((-1, -1), (0, 0), (1, 1), (4, 3)): + fbuf.fill(0) + fbuf.blit(fbuf2, x, y) + printbuf() + +# Blit a bytes object. +fbuf.fill(0) +image = (b"\x10\x11\x12\x13", 2, 2, framebuf.GS8) +fbuf.blit(image, 1, 1) +printbuf() + +# Blit a bytes object that has a stride. +fbuf.fill(0) +image = (b"\x20\x21\xff\x22\x23\xff", 2, 2, framebuf.GS8, 3) +fbuf.blit(image, 1, 1) +printbuf() + +# Blit a bytes object with a bytes palette. +fbuf.fill(0) +image = (b"\x00\x01\x01\x00", 2, 2, framebuf.GS8) +palette = (b"\xa1\xa2", 2, 1, framebuf.GS8) +fbuf.blit(image, 1, 1, -1, palette) +printbuf() + +# Not enough elements in the tuple. +try: + fbuf.blit((0, 0, 0), 0, 0) +except ValueError: + print("ValueError") + +# Too many elements in the tuple. +try: + fbuf.blit((0, 0, 0, 0, 0, 0), 0, 0) +except ValueError: + print("ValueError") + +# Bytes too small. +try: + fbuf.blit((b"", 1, 1, framebuf.GS8), 0, 0) +except ValueError: + print("ValueError") diff --git a/tests/extmod/framebuf_blit.py.exp b/tests/extmod/framebuf_blit.py.exp new file mode 100644 index 0000000000000..e340f1990c783 --- /dev/null +++ b/tests/extmod/framebuf_blit.py.exp @@ -0,0 +1,45 @@ +--8<-- +ff00000000 +0000000000 +0000000000 +0000000000 +-->8-- +--8<-- +ffff000000 +ffff000000 +0000000000 +0000000000 +-->8-- +--8<-- +0000000000 +00ffff0000 +00ffff0000 +0000000000 +-->8-- +--8<-- +0000000000 +0000000000 +0000000000 +00000000ff +-->8-- +--8<-- +0000000000 +0010110000 +0012130000 +0000000000 +-->8-- +--8<-- +0000000000 +0020210000 +0022230000 +0000000000 +-->8-- +--8<-- +0000000000 +00a1a20000 +00a2a10000 +0000000000 +-->8-- +ValueError +ValueError +ValueError diff --git a/tests/extmod/hashlib_md5.py b/tests/extmod/hashlib_md5.py index 5f925fc97d02e..ebbe9155e0493 100644 --- a/tests/extmod/hashlib_md5.py +++ b/tests/extmod/hashlib_md5.py @@ -1,8 +1,7 @@ try: import hashlib except ImportError: - # This is neither uPy, nor cPy, so must be uPy with - # hashlib module disabled. + # MicroPython with hashlib module disabled. print("SKIP") raise SystemExit diff --git a/tests/extmod/hashlib_sha1.py b/tests/extmod/hashlib_sha1.py index af23033a591f2..46ffb73fcbe01 100644 --- a/tests/extmod/hashlib_sha1.py +++ b/tests/extmod/hashlib_sha1.py @@ -1,8 +1,7 @@ try: import hashlib except ImportError: - # This is neither uPy, nor cPy, so must be uPy with - # hashlib module disabled. + # MicroPython with hashlib module disabled. print("SKIP") raise SystemExit diff --git a/tests/extmod/hashlib_sha256.py b/tests/extmod/hashlib_sha256.py index 95cd301d160d0..209fcb3987792 100644 --- a/tests/extmod/hashlib_sha256.py +++ b/tests/extmod/hashlib_sha256.py @@ -1,8 +1,7 @@ try: import hashlib except ImportError: - # This is neither uPy, nor cPy, so must be uPy with - # hashlib module disabled. + # MicroPython with hashlib module disabled. print("SKIP") raise SystemExit diff --git a/tests/extmod/json_dump.py b/tests/extmod/json_dump.py index 897d33cc81253..0beb4f5f85698 100644 --- a/tests/extmod/json_dump.py +++ b/tests/extmod/json_dump.py @@ -16,11 +16,11 @@ # dump to a small-int not allowed try: json.dump(123, 1) -except (AttributeError, OSError): # CPython and uPy have different errors +except (AttributeError, OSError): # CPython and MicroPython have different errors print("Exception") # dump to an object not allowed try: json.dump(123, {}) -except (AttributeError, OSError): # CPython and uPy have different errors +except (AttributeError, OSError): # CPython and MicroPython have different errors print("Exception") diff --git a/tests/extmod/json_dump_iobase.py b/tests/extmod/json_dump_iobase.py index 94d317b87968f..81105e36dccec 100644 --- a/tests/extmod/json_dump_iobase.py +++ b/tests/extmod/json_dump_iobase.py @@ -18,7 +18,7 @@ def __init__(self): def write(self, buf): if type(buf) == bytearray: - # uPy passes a bytearray, CPython passes a str + # MicroPython passes a bytearray, CPython passes a str buf = str(buf, "ascii") self.buf += buf return len(buf) diff --git a/tests/extmod/json_dump_separators.py b/tests/extmod/json_dump_separators.py index 4f8e56dceb536..ce39294820fa9 100644 --- a/tests/extmod/json_dump_separators.py +++ b/tests/extmod/json_dump_separators.py @@ -25,20 +25,20 @@ # dump to a small-int not allowed try: json.dump(123, 1, separators=sep) - except (AttributeError, OSError): # CPython and uPy have different errors + except (AttributeError, OSError): # CPython and MicroPython have different errors print("Exception") # dump to an object not allowed try: json.dump(123, {}, separators=sep) - except (AttributeError, OSError): # CPython and uPy have different errors + except (AttributeError, OSError): # CPython and MicroPython have different errors print("Exception") try: s = StringIO() json.dump(False, s, separators={"a": 1}) -except (TypeError, ValueError): # CPython and uPy have different errors +except (TypeError, ValueError): # CPython and MicroPython have different errors print("Exception") # invalid separator types diff --git a/tests/extmod/json_dumps_extra.py b/tests/extmod/json_dumps_extra.py index a410b0ee0ef63..70efc86645115 100644 --- a/tests/extmod/json_dumps_extra.py +++ b/tests/extmod/json_dumps_extra.py @@ -1,4 +1,4 @@ -# test uPy json behaviour that's not valid in CPy +# test MicroPython json behaviour that's not valid in CPy # CIRCUITPY-CHANGE: This behavior matches CPython print("SKIP") raise SystemExit diff --git a/tests/extmod/json_dumps_separators.py b/tests/extmod/json_dumps_separators.py index a3a9ec308f09d..0a95f489a08ec 100644 --- a/tests/extmod/json_dumps_separators.py +++ b/tests/extmod/json_dumps_separators.py @@ -39,7 +39,7 @@ try: json.dumps(False, separators={"a": 1}) -except (TypeError, ValueError): # CPython and uPy have different errors +except (TypeError, ValueError): # CPython and MicroPython have different errors print("Exception") # invalid separator types diff --git a/tests/extmod/json_loads.py b/tests/extmod/json_loads.py index f9073c121e2ef..092402d715d69 100644 --- a/tests/extmod/json_loads.py +++ b/tests/extmod/json_loads.py @@ -71,3 +71,27 @@ def my_print(o): my_print(json.loads("[null] a")) except ValueError: print("ValueError") + +# incomplete object declaration +try: + my_print(json.loads('{"a":0,')) +except ValueError: + print("ValueError") + +# incomplete nested array declaration +try: + my_print(json.loads('{"a":0, [')) +except ValueError: + print("ValueError") + +# incomplete array declaration +try: + my_print(json.loads("[0,")) +except ValueError: + print("ValueError") + +# incomplete nested object declaration +try: + my_print(json.loads('[0, {"a":0, ')) +except ValueError: + print("ValueError") diff --git a/tests/extmod/json_loads_bytes.py.exp b/tests/extmod/json_loads_bytes.py.exp deleted file mode 100644 index c2735a99052d2..0000000000000 --- a/tests/extmod/json_loads_bytes.py.exp +++ /dev/null @@ -1,2 +0,0 @@ -[1, 2] -[None] diff --git a/tests/extmod/json_loads_int_64.py b/tests/extmod/json_loads_int_64.py new file mode 100644 index 0000000000000..f6236f1904a87 --- /dev/null +++ b/tests/extmod/json_loads_int_64.py @@ -0,0 +1,16 @@ +# Parse 64-bit integers from JSON payloads. +# +# This also exercises parsing integers from strings +# where the value may not be null terminated (last line) +try: + import json +except ImportError: + print("SKIP") + raise SystemExit + + +print(json.loads("9111222333444555666")) +print(json.loads("-9111222333444555666")) +print(json.loads("9111222333444555666")) +print(json.loads("-9111222333444555666")) +print(json.loads('["9111222333444555666777",9111222333444555666]')) diff --git a/tests/extmod/machine_hard_timer.py b/tests/extmod/machine_hard_timer.py new file mode 100644 index 0000000000000..8fe42ea850842 --- /dev/null +++ b/tests/extmod/machine_hard_timer.py @@ -0,0 +1,45 @@ +import sys + +try: + from machine import Timer + from time import sleep_ms +except: + print("SKIP") + raise SystemExit + +if sys.platform == "esp8266": + timer = Timer(0) +else: + # Hardware timers are not implemented. + print("SKIP") + raise SystemExit + +# Test both hard and soft IRQ handlers and both one-shot and periodic +# timers. We adjust period in tests/extmod/machine_soft_timer.py, so try +# adjusting freq here instead. The heap should be locked in hard callbacks +# and unlocked in soft callbacks. + + +def callback(t): + print("callback", mode[1], kind[1], freq, end=" ") + try: + allocate = bytearray(1) + print("unlocked") + except MemoryError: + print("locked") + + +modes = [(Timer.ONE_SHOT, "one-shot"), (Timer.PERIODIC, "periodic")] +kinds = [(False, "soft"), (True, "hard")] + +for mode in modes: + for kind in kinds: + for freq in 50, 25: + timer.init( + mode=mode[0], + freq=freq, + hard=kind[0], + callback=callback, + ) + sleep_ms(90) + timer.deinit() diff --git a/tests/extmod/machine_hard_timer.py.exp b/tests/extmod/machine_hard_timer.py.exp new file mode 100644 index 0000000000000..26cdc644fdd08 --- /dev/null +++ b/tests/extmod/machine_hard_timer.py.exp @@ -0,0 +1,16 @@ +callback one-shot soft 50 unlocked +callback one-shot soft 25 unlocked +callback one-shot hard 50 locked +callback one-shot hard 25 locked +callback periodic soft 50 unlocked +callback periodic soft 50 unlocked +callback periodic soft 50 unlocked +callback periodic soft 50 unlocked +callback periodic soft 25 unlocked +callback periodic soft 25 unlocked +callback periodic hard 50 locked +callback periodic hard 50 locked +callback periodic hard 50 locked +callback periodic hard 50 locked +callback periodic hard 25 locked +callback periodic hard 25 locked diff --git a/tests/extmod/machine_timer.py b/tests/extmod/machine_timer.py new file mode 100644 index 0000000000000..ef97ea4e94955 --- /dev/null +++ b/tests/extmod/machine_timer.py @@ -0,0 +1,48 @@ +import sys + +try: + from machine import Timer + from time import sleep_ms +except: + print("SKIP") + raise SystemExit + +if sys.platform in ("esp32", "esp8266", "nrf"): + # Software timers aren't implemented on the esp32 and esp8266 ports. + # The nrf port doesn't support selection of hard and soft callbacks, + # and only allows Timer(period=N), not Timer(freq=N). + print("SKIP") + raise SystemExit +else: + timer_id = -1 + +# Test both hard and soft IRQ handlers and both one-shot and periodic +# timers. We adjust period in tests/extmod/machine_soft_timer.py, so try +# adjusting freq here instead. The heap should be locked in hard callbacks +# and unlocked in soft callbacks. + + +def callback(t): + print("callback", mode[1], kind[1], freq, end=" ") + try: + allocate = bytearray(1) + print("unlocked") + except MemoryError: + print("locked") + + +modes = [(Timer.ONE_SHOT, "one-shot"), (Timer.PERIODIC, "periodic")] +kinds = [(False, "soft"), (True, "hard")] + +for mode in modes: + for kind in kinds: + for freq in 50, 25: + timer = Timer( + timer_id, + mode=mode[0], + freq=freq, + hard=kind[0], + callback=callback, + ) + sleep_ms(90) + timer.deinit() diff --git a/tests/extmod/machine_timer.py.exp b/tests/extmod/machine_timer.py.exp new file mode 100644 index 0000000000000..26cdc644fdd08 --- /dev/null +++ b/tests/extmod/machine_timer.py.exp @@ -0,0 +1,16 @@ +callback one-shot soft 50 unlocked +callback one-shot soft 25 unlocked +callback one-shot hard 50 locked +callback one-shot hard 25 locked +callback periodic soft 50 unlocked +callback periodic soft 50 unlocked +callback periodic soft 50 unlocked +callback periodic soft 50 unlocked +callback periodic soft 25 unlocked +callback periodic soft 25 unlocked +callback periodic hard 50 locked +callback periodic hard 50 locked +callback periodic hard 50 locked +callback periodic hard 50 locked +callback periodic hard 25 locked +callback periodic hard 25 locked diff --git a/tests/extmod/platform_basic.py b/tests/extmod/platform_basic.py new file mode 100644 index 0000000000000..eb6f2be13c17a --- /dev/null +++ b/tests/extmod/platform_basic.py @@ -0,0 +1,8 @@ +try: + import platform +except ImportError: + print("SKIP") + raise SystemExit + +print(type(platform.python_compiler())) +print(type(platform.libc_ver())) diff --git a/tests/extmod/random_extra_float.py b/tests/extmod/random_extra_float.py index 3b37ed8dcef2f..03973c583492d 100644 --- a/tests/extmod/random_extra_float.py +++ b/tests/extmod/random_extra_float.py @@ -1,12 +1,8 @@ try: import random -except ImportError: - print("SKIP") - raise SystemExit -try: - random.randint -except AttributeError: + random.random +except (ImportError, AttributeError): print("SKIP") raise SystemExit diff --git a/tests/extmod/re_error.py b/tests/extmod/re_error.py index f61d0913289e1..bd678c9d25156 100644 --- a/tests/extmod/re_error.py +++ b/tests/extmod/re_error.py @@ -11,7 +11,7 @@ def test_re(r): try: re.compile(r) print("OK") - except: # uPy and CPy use different errors, so just ignore the type + except: # MPy and CPy use different errors, so just ignore the type print("Error") diff --git a/tests/extmod/re_start_end_pos.py b/tests/extmod/re_start_end_pos.py new file mode 100644 index 0000000000000..bd16584374b89 --- /dev/null +++ b/tests/extmod/re_start_end_pos.py @@ -0,0 +1,78 @@ +# test start and end pos specification + +try: + import re +except ImportError: + print("SKIP") + raise SystemExit + + +def print_groups(match): + print("----") + try: + if match is not None: + i = 0 + while True: + print(match.group(i)) + i += 1 + except IndexError: + pass + + +p = re.compile(r"o") +m = p.match("dog") +print_groups(m) + +m = p.match("dog", 1) +print_groups(m) + +m = p.match("dog", 2) +print_groups(m) + +# No match past end of input +m = p.match("dog", 5) +print_groups(m) + +m = p.match("dog", 0, 1) +print_groups(m) + +# Caret only matches the actual beginning +p = re.compile(r"^o") +m = p.match("dog", 1) +print_groups(m) + +# End at beginning means searching empty string +p = re.compile(r"o") +m = p.match("dog", 1, 1) +print_groups(m) + +# End before the beginning doesn't match anything +m = p.match("dog", 2, 1) +print_groups(m) + +# Negative starting values don't crash +m = p.search("dog", -2) +print_groups(m) + +m = p.search("dog", -2, -5) +print_groups(m) + +# Search also works +print("--search") + +p = re.compile(r"o") +m = p.search("dog") +print_groups(m) + +m = p.search("dog", 1) +print_groups(m) + +m = p.search("dog", 2) +print_groups(m) + +# Negative starting values don't crash +m = p.search("dog", -2) +print_groups(m) + +m = p.search("dog", -2, -5) +print_groups(m) diff --git a/tests/extmod/re_sub.py b/tests/extmod/re_sub.py index 3959949724d98..bb9aa111287e6 100644 --- a/tests/extmod/re_sub.py +++ b/tests/extmod/re_sub.py @@ -62,7 +62,7 @@ def A(): except: print("invalid group") -# invalid group with very large number (to test overflow in uPy) +# invalid group with very large number (to test overflow in MicroPython) try: re.sub("(a)", "b\\199999999999999999999999999999999999999", "a") except: diff --git a/tests/extmod/re_sub_unmatched.py.exp b/tests/extmod/re_sub_unmatched.py.exp deleted file mode 100644 index 1e5f0fda0554d..0000000000000 --- a/tests/extmod/re_sub_unmatched.py.exp +++ /dev/null @@ -1 +0,0 @@ -1-a2 diff --git a/tests/extmod/socket_badconstructor.py b/tests/extmod/socket_badconstructor.py new file mode 100644 index 0000000000000..4a9d2668c7f1a --- /dev/null +++ b/tests/extmod/socket_badconstructor.py @@ -0,0 +1,22 @@ +# Test passing in bad values to socket.socket constructor. + +try: + import socket +except: + print("SKIP") + raise SystemExit + +try: + s = socket.socket(None) +except TypeError: + print("TypeError") + +try: + s = socket.socket(socket.AF_INET, None) +except TypeError: + print("TypeError") + +try: + s = socket.socket(socket.AF_INET, socket.SOCK_RAW, None) +except TypeError: + print("TypeError") diff --git a/tests/extmod/socket_fileno.py b/tests/extmod/socket_fileno.py new file mode 100644 index 0000000000000..da15825e3d56b --- /dev/null +++ b/tests/extmod/socket_fileno.py @@ -0,0 +1,17 @@ +# Test socket.fileno() functionality + +try: + import socket +except ImportError: + print("SKIP") + raise SystemExit + +if not hasattr(socket.socket, "fileno"): + print("SKIP") + raise SystemExit + +s = socket.socket() +print(s.fileno() >= 0) + +s.close() +print(s.fileno()) # should print -1 diff --git a/tests/extmod/time_mktime.py b/tests/extmod/time_mktime.py new file mode 100644 index 0000000000000..7fc643dc3cb8c --- /dev/null +++ b/tests/extmod/time_mktime.py @@ -0,0 +1,120 @@ +# test conversion from date tuple to timestamp and back + +try: + import time + + time.localtime +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit + +# Range of date expected to work on all MicroPython platforms +MIN_YEAR = 1970 +MAX_YEAR = 2099 +# CPython properly supported date range: +# - on Windows: year 1970 to 3000+ +# - on Unix: year 1583 to 3000+ + +# Start test from Jan 1, 2001 13:00 (Feb 2000 might already be broken) +SAFE_DATE = (2001, 1, 1, 13, 0, 0, 0, 0, -1) + + +# mktime function that checks that the result is reversible +def safe_mktime(date_tuple): + try: + res = time.mktime(date_tuple) + chk = time.localtime(res) + except OverflowError: + print("safe_mktime:", date_tuple, "overflow error") + return None + if chk[0:5] != date_tuple[0:5]: + print("safe_mktime:", date_tuple[0:5], " -> ", res, " -> ", chk[0:5]) + return None + return res + + +# localtime function that checks that the result is reversible +def safe_localtime(timestamp): + try: + res = time.localtime(timestamp) + chk = time.mktime(res) + except OverflowError: + print("safe_localtime:", timestamp, "overflow error") + return None + if chk != timestamp: + print("safe_localtime:", timestamp, " -> ", res, " -> ", chk) + return None + return res + + +# look for smallest valid timestamps by iterating backwards on tuple +def test_bwd(date_tuple): + curr_stamp = safe_mktime(date_tuple) + year = date_tuple[0] + month = date_tuple[1] - 1 + if month < 1: + year -= 1 + month = 12 + while year >= MIN_YEAR: + while month >= 1: + next_tuple = (year, month) + date_tuple[2:] + next_stamp = safe_mktime(next_tuple) + # at this stage, only test consistency and monotonicity + if next_stamp is None or next_stamp >= curr_stamp: + return date_tuple + date_tuple = next_tuple + curr_stamp = next_stamp + month -= 1 + year -= 1 + month = 12 + return date_tuple + + +# test day-by-day to ensure that every date is properly converted +def test_fwd(start_date): + DAYS_PER_MONTH = (0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31) + curr_stamp = safe_mktime(start_date) + curr_date = safe_localtime(curr_stamp) + while curr_date[0] <= MAX_YEAR: + if curr_date[2] < 15: + skip_days = 13 + else: + skip_days = 1 + next_stamp = curr_stamp + skip_days * 86400 + next_date = safe_localtime(next_stamp) + if next_date is None: + return curr_date + if next_date[2] != curr_date[2] + skip_days: + # next month + if next_date[2] != 1: + print("wrong day of month:", next_date) + return curr_date + # check the number of days in previous month + month_days = DAYS_PER_MONTH[curr_date[1]] + if month_days == 28 and curr_date[0] % 4 == 0: + if curr_date[0] % 100 != 0 or curr_date[0] % 400 == 0: + month_days += 1 + if curr_date[2] != month_days: + print("wrong day count in prev month:", curr_date[2], "vs", month_days) + return curr_date + if next_date[1] != curr_date[1] + 1: + # next year + if curr_date[1] != 12: + print("wrong month count in prev year:", curr_date[1]) + return curr_date + if next_date[1] != 1: + print("wrong month:", next_date) + return curr_date + if next_date[0] != curr_date[0] + 1: + print("wrong year:", next_date) + return curr_date + curr_stamp = next_stamp + curr_date = next_date + return curr_date + + +small_date = test_bwd(SAFE_DATE) +large_date = test_fwd(small_date) +print("tested from", small_date[0:3], "to", large_date[0:3]) +print(small_date[0:3], "wday is", small_date[6]) +print(large_date[0:3], "wday is", large_date[6]) diff --git a/tests/extmod/time_res.py b/tests/extmod/time_res.py index 548bef1f1747c..ef20050b914d7 100644 --- a/tests/extmod/time_res.py +++ b/tests/extmod/time_res.py @@ -37,9 +37,12 @@ def test(): time.sleep_ms(100) for func_name, _ in EXPECTED_MAP: try: - time_func = getattr(time, func_name, None) or globals()[func_name] + if func_name.endswith("_time"): + time_func = globals()[func_name] + else: + time_func = getattr(time, func_name) now = time_func() # may raise AttributeError - except (KeyError, AttributeError): + except AttributeError: continue try: results_map[func_name].add(now) diff --git a/tests/extmod/tls_dtls.py b/tests/extmod/tls_dtls.py index b2d716769d3f7..753ab2fee4f40 100644 --- a/tests/extmod/tls_dtls.py +++ b/tests/extmod/tls_dtls.py @@ -34,9 +34,19 @@ def ioctl(self, req, arg): # Wrap the DTLS Server dtls_server_ctx = SSLContext(PROTOCOL_DTLS_SERVER) dtls_server_ctx.verify_mode = CERT_NONE -dtls_server = dtls_server_ctx.wrap_socket(server_socket, do_handshake_on_connect=False) +dtls_server = dtls_server_ctx.wrap_socket( + server_socket, do_handshake_on_connect=False, client_id=b"dummy_client_id" +) print("Wrapped DTLS Server") +# wrap DTLS server with invalid client_id +try: + dtls_server = dtls_server_ctx.wrap_socket( + server_socket, do_handshake_on_connect=False, client_id=4 + ) +except OSError: + print("Failed to wrap DTLS Server with invalid client_id") + # Wrap the DTLS Client dtls_client_ctx = SSLContext(PROTOCOL_DTLS_CLIENT) dtls_client_ctx.verify_mode = CERT_NONE diff --git a/tests/extmod/tls_dtls.py.exp b/tests/extmod/tls_dtls.py.exp index 78d72bff18816..dbd005d0edfb8 100644 --- a/tests/extmod/tls_dtls.py.exp +++ b/tests/extmod/tls_dtls.py.exp @@ -1,3 +1,4 @@ Wrapped DTLS Server +Failed to wrap DTLS Server with invalid client_id Wrapped DTLS Client OK diff --git a/tests/extmod/ssl_noleak.py b/tests/extmod/tls_noleak.py similarity index 100% rename from tests/extmod/ssl_noleak.py rename to tests/extmod/tls_noleak.py diff --git a/tests/extmod/tls_threads.py b/tests/extmod/tls_threads.py new file mode 100644 index 0000000000000..1e0c3d23d2f38 --- /dev/null +++ b/tests/extmod/tls_threads.py @@ -0,0 +1,58 @@ +# Ensure that SSL sockets can be allocated from multiple +# threads without thread safety issues + +try: + import _thread + import io + import tls + import time +except ImportError: + print("SKIP") + raise SystemExit + +import unittest + + +class TestSocket(io.IOBase): + def write(self, buf): + return len(buf) + + def readinto(self, buf): + return 0 + + def ioctl(self, cmd, arg): + return 0 + + def setblocking(self, value): + pass + + +ITERS = 256 + + +class TLSThreads(unittest.TestCase): + def test_sslsocket_threaded(self): + self.done = False + # only run in two threads: too much RAM demand otherwise, and rp2 only + # supports two anyhow + _thread.start_new_thread(self._alloc_many_sockets, (True,)) + self._alloc_many_sockets(False) + while not self.done: + time.sleep(0.1) + print("done") + + def _alloc_many_sockets(self, set_done_flag): + print("start", _thread.get_ident()) + ctx = tls.SSLContext(tls.PROTOCOL_TLS_CLIENT) + ctx.verify_mode = tls.CERT_NONE + for n in range(ITERS): + s = TestSocket() + s = ctx.wrap_socket(s, do_handshake_on_connect=False) + s.close() # Free associated resources now from thread, not in a GC pass + print("done", _thread.get_ident()) + if set_done_flag: + self.done = True + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/extmod/uctypes_addressof.py b/tests/extmod/uctypes_addressof.py index c83089d0f72af..213fcc05eee2b 100644 --- a/tests/extmod/uctypes_addressof.py +++ b/tests/extmod/uctypes_addressof.py @@ -12,5 +12,8 @@ print(uctypes.addressof(uctypes.bytearray_at(1 << i, 8))) # Test address that is bigger than the greatest small-int but still within the address range. -large_addr = maxsize + 1 -print(uctypes.addressof(uctypes.bytearray_at(large_addr, 8)) == large_addr) +try: + large_addr = maxsize + 1 + print(uctypes.addressof(uctypes.bytearray_at(large_addr, 8)) == large_addr) +except OverflowError: + print(True) # systems with 64-bit bigints will overflow on the above operation diff --git a/tests/extmod/uctypes_array_load_store.py b/tests/extmod/uctypes_array_load_store.py index 9de079998566f..695352da57983 100644 --- a/tests/extmod/uctypes_array_load_store.py +++ b/tests/extmod/uctypes_array_load_store.py @@ -6,6 +6,13 @@ print("SKIP") raise SystemExit +# 'int' needs to be able to represent UINT64 for this test +try: + int("FF" * 8, 16) +except OverflowError: + print("SKIP") + raise SystemExit + N = 5 for endian in ("NATIVE", "LITTLE_ENDIAN", "BIG_ENDIAN"): diff --git a/tests/extmod/vfs_blockdev_invalid.py b/tests/extmod/vfs_blockdev_invalid.py index 4d00f4b00273a..29d6bd6b2f9f7 100644 --- a/tests/extmod/vfs_blockdev_invalid.py +++ b/tests/extmod/vfs_blockdev_invalid.py @@ -70,8 +70,8 @@ def test(vfs_class): try: with fs.open("test", "r") as f: print("opened") - except OSError as e: - print("OSError", e) + except Exception as e: + print(type(e), e) # This variant should succeed on open, may fail on read # unless the filesystem cached the contents already @@ -81,8 +81,8 @@ def test(vfs_class): bdev.read_res = res print("read 1", f.read(1)) print("read rest", f.read()) - except OSError as e: - print("OSError", e) + except Exception as e: + print(type(e), e) test(vfs.VfsLfs2) diff --git a/tests/extmod/vfs_blockdev_invalid.py.exp b/tests/extmod/vfs_blockdev_invalid.py.exp index 13695e0d88916..0ea0353501de3 100644 --- a/tests/extmod/vfs_blockdev_invalid.py.exp +++ b/tests/extmod/vfs_blockdev_invalid.py.exp @@ -2,27 +2,27 @@ opened read 1 a read rest aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa -OSError [Errno 5] EIO + [Errno 5] EIO read 1 a read rest aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa -OSError [Errno 22] EINVAL + [Errno 22] EINVAL read 1 a read rest aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa -OSError [Errno 22] EINVAL + [Errno 22] EINVAL read 1 a read rest aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa -OSError [Errno 22] EINVAL + can't convert str to int read 1 a read rest aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa opened read 1 a read rest aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa -OSError [Errno 5] EIO -OSError [Errno 5] EIO -OSError [Errno 5] EIO -OSError [Errno 5] EIO -OSError [Errno 5] EIO -OSError [Errno 5] EIO -OSError [Errno 5] EIO -OSError [Errno 5] EIO + [Errno 5] EIO + [Errno 5] EIO + [Errno 5] EIO + [Errno 5] EIO + [Errno 5] EIO + [Errno 5] EIO + can't convert str to int + can't convert str to int diff --git a/tests/extmod/vfs_fat_ilistdir_del.py b/tests/extmod/vfs_fat_ilistdir_del.py index a6e24ec92f3b5..964e6b12868eb 100644 --- a/tests/extmod/vfs_fat_ilistdir_del.py +++ b/tests/extmod/vfs_fat_ilistdir_del.py @@ -1,8 +1,7 @@ # Test ilistdir __del__ for VfsFat using a RAM device. -import gc try: - import os, vfs + import gc, os, vfs vfs.VfsFat except (ImportError, AttributeError): diff --git a/tests/extmod/vfs_lfs.py b/tests/extmod/vfs_lfs.py index 3ad57fd9c38f1..40d58e9c9f743 100644 --- a/tests/extmod/vfs_lfs.py +++ b/tests/extmod/vfs_lfs.py @@ -136,7 +136,7 @@ def test(bdev, vfs_class): print(fs.getcwd()) fs.chdir("../testdir") print(fs.getcwd()) - fs.chdir("../..") + fs.chdir("..") print(fs.getcwd()) fs.chdir(".//testdir") print(fs.getcwd()) diff --git a/tests/extmod/vfs_lfs_error.py b/tests/extmod/vfs_lfs_error.py index 2ac7629bfa8fc..73cdf3437330b 100644 --- a/tests/extmod/vfs_lfs_error.py +++ b/tests/extmod/vfs_lfs_error.py @@ -1,7 +1,7 @@ # Test for VfsLittle using a RAM device, testing error handling try: - import vfs + import errno, vfs vfs.VfsLfs1 vfs.VfsLfs2 @@ -41,14 +41,14 @@ def test(bdev, vfs_class): # mkfs with too-small block device try: vfs_class.mkfs(RAMBlockDevice(1)) - except OSError: - print("mkfs OSError") + except OSError as er: + print("mkfs OSError", er.errno > 0) # mount with invalid filesystem try: vfs_class(bdev) - except OSError: - print("mount OSError") + except OSError as er: + print("mount OSError", er.errno > 0) # set up for following tests vfs_class.mkfs(bdev) @@ -60,60 +60,60 @@ def test(bdev, vfs_class): # ilistdir try: fs.ilistdir("noexist") - except OSError: - print("ilistdir OSError") + except OSError as er: + print("ilistdir OSError", er) # remove try: fs.remove("noexist") - except OSError: - print("remove OSError") + except OSError as er: + print("remove OSError", er) # rmdir try: fs.rmdir("noexist") - except OSError: - print("rmdir OSError") + except OSError as er: + print("rmdir OSError", er) # rename try: fs.rename("noexist", "somethingelse") - except OSError: - print("rename OSError") + except OSError as er: + print("rename OSError", er) # mkdir try: fs.mkdir("testdir") - except OSError: - print("mkdir OSError") + except OSError as er: + print("mkdir OSError", er) # chdir to nonexistent try: fs.chdir("noexist") - except OSError: - print("chdir OSError") + except OSError as er: + print("chdir OSError", er) print(fs.getcwd()) # check still at root # chdir to file try: fs.chdir("testfile") - except OSError: - print("chdir OSError") + except OSError as er: + print("chdir OSError", er) print(fs.getcwd()) # check still at root # stat try: fs.stat("noexist") - except OSError: - print("stat OSError") + except OSError as er: + print("stat OSError", er) # error during seek with fs.open("testfile", "r") as f: f.seek(1 << 30) # SEEK_SET try: f.seek(1 << 30, 1) # SEEK_CUR - except OSError: - print("seek OSError") + except OSError as er: + print("seek OSError", er) bdev = RAMBlockDevice(30) diff --git a/tests/extmod/vfs_lfs_error.py.exp b/tests/extmod/vfs_lfs_error.py.exp index f4327f6962ec3..440607ed84b22 100644 --- a/tests/extmod/vfs_lfs_error.py.exp +++ b/tests/extmod/vfs_lfs_error.py.exp @@ -1,28 +1,28 @@ test -mkfs OSError -mount OSError -ilistdir OSError -remove OSError -rmdir OSError -rename OSError -mkdir OSError -chdir OSError +mkfs OSError True +mount OSError True +ilistdir OSError [Errno 2] ENOENT +remove OSError [Errno 2] ENOENT +rmdir OSError [Errno 2] ENOENT +rename OSError [Errno 2] ENOENT +mkdir OSError [Errno 17] EEXIST +chdir OSError [Errno 2] ENOENT / -chdir OSError +chdir OSError [Errno 2] ENOENT / -stat OSError -seek OSError +stat OSError [Errno 2] ENOENT +seek OSError [Errno 22] EINVAL test -mkfs OSError -mount OSError -ilistdir OSError -remove OSError -rmdir OSError -rename OSError -mkdir OSError -chdir OSError +mkfs OSError True +mount OSError True +ilistdir OSError [Errno 2] ENOENT +remove OSError [Errno 2] ENOENT +rmdir OSError [Errno 2] ENOENT +rename OSError [Errno 2] ENOENT +mkdir OSError [Errno 17] EEXIST +chdir OSError [Errno 2] ENOENT / -chdir OSError +chdir OSError [Errno 2] ENOENT / -stat OSError -seek OSError +stat OSError [Errno 2] ENOENT +seek OSError [Errno 22] EINVAL diff --git a/tests/extmod/vfs_lfs_ilistdir_del.py b/tests/extmod/vfs_lfs_ilistdir_del.py index 7b59bc412d983..828c85a258856 100644 --- a/tests/extmod/vfs_lfs_ilistdir_del.py +++ b/tests/extmod/vfs_lfs_ilistdir_del.py @@ -1,8 +1,7 @@ # Test ilistdir __del__ for VfsLittle using a RAM device. -import gc try: - import vfs + import gc, vfs vfs.VfsLfs2 except (ImportError, AttributeError): @@ -71,5 +70,10 @@ def test(bdev, vfs_class): fs.open("/test", "w").close() -bdev = RAMBlockDevice(30) +try: + bdev = RAMBlockDevice(30) +except MemoryError: + print("SKIP") + raise SystemExit + test(bdev, vfs.VfsLfs2) diff --git a/tests/extmod/vfs_mountinfo.py b/tests/extmod/vfs_mountinfo.py index f674e80763409..b31dc60ce76d7 100644 --- a/tests/extmod/vfs_mountinfo.py +++ b/tests/extmod/vfs_mountinfo.py @@ -5,7 +5,6 @@ except ImportError: print("SKIP") raise SystemExit -import errno class Filesystem: diff --git a/tests/extmod/vfs_posix.py b/tests/extmod/vfs_posix.py index d060c0b9c84f3..b3ca2753ba9cf 100644 --- a/tests/extmod/vfs_posix.py +++ b/tests/extmod/vfs_posix.py @@ -29,7 +29,21 @@ print(type(os.stat("/"))) # listdir and ilistdir -print(type(os.listdir("/"))) +target = "/" +try: + import platform + + # On Android non-root users are permitted full filesystem access only to + # selected directories. To let this test pass on bionic, the internal + # user-accessible storage area root is enumerated instead of the + # filesystem root. "/storage/emulated/0" should be there on pretty much + # any recent-ish device; querying the proper location requires a JNI + # round-trip, not really worth it. + if platform.platform().startswith("Android-"): + target = "/storage/emulated/0" +except ImportError: + pass +print(type(os.listdir(target))) # mkdir os.mkdir(temp_dir) diff --git a/tests/extmod/vfs_posix_ilistdir_del.py b/tests/extmod/vfs_posix_ilistdir_del.py index 78d7c854c543c..8b5984cd81c05 100644 --- a/tests/extmod/vfs_posix_ilistdir_del.py +++ b/tests/extmod/vfs_posix_ilistdir_del.py @@ -1,8 +1,7 @@ # Test ilistdir __del__ for VfsPosix. -import gc try: - import os, vfs + import gc, os, vfs vfs.VfsPosix except (ImportError, AttributeError): diff --git a/tests/extmod/vfs_posix_paths.py b/tests/extmod/vfs_posix_paths.py index b4fedc6716f01..c06318748a327 100644 --- a/tests/extmod/vfs_posix_paths.py +++ b/tests/extmod/vfs_posix_paths.py @@ -31,7 +31,7 @@ fs.mkdir("subdir/one") print('listdir("/"):', sorted(i[0] for i in fs.ilistdir("/"))) print('listdir("."):', sorted(i[0] for i in fs.ilistdir("."))) -print('getcwd() in {"", "/"}:', fs.getcwd() in {"", "/"}) +print('getcwd() in ("", "/"):', fs.getcwd() in ("", "/")) print('chdir("subdir"):', fs.chdir("subdir")) print("getcwd():", fs.getcwd()) print('mkdir("two"):', fs.mkdir("two")) diff --git a/tests/extmod/vfs_posix_paths.py.exp b/tests/extmod/vfs_posix_paths.py.exp index ecc13222aaaeb..d6a0960efd249 100644 --- a/tests/extmod/vfs_posix_paths.py.exp +++ b/tests/extmod/vfs_posix_paths.py.exp @@ -1,6 +1,6 @@ listdir("/"): ['subdir'] listdir("."): ['subdir'] -getcwd() in {"", "/"}: True +getcwd() in ("", "/"): True chdir("subdir"): None getcwd(): /subdir mkdir("two"): None diff --git a/tests/extmod/vfs_posix_readonly.py b/tests/extmod/vfs_posix_readonly.py new file mode 100644 index 0000000000000..e7821381006fd --- /dev/null +++ b/tests/extmod/vfs_posix_readonly.py @@ -0,0 +1,99 @@ +# Test for VfsPosix + +try: + import gc, os, vfs, errno + + vfs.VfsPosix +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit + +# We need a directory for testing that doesn't already exist. +# Skip the test if it does exist. +temp_dir = "vfs_posix_readonly_test_dir" +try: + os.stat(temp_dir) + raise SystemExit("Target directory {} exists".format(temp_dir)) +except OSError: + pass + +# mkdir (skip test if whole filesystem is readonly) +try: + os.mkdir(temp_dir) +except OSError as e: + if e.errno == errno.EROFS: + print("SKIP") + raise SystemExit + +fs_factory = lambda: vfs.VfsPosix(temp_dir) + +# mount +fs = fs_factory() +vfs.mount(fs, "/vfs") + +with open("/vfs/file", "w") as f: + f.write("content") + +# test reading works +with open("/vfs/file") as f: + print("file:", f.read()) + +os.mkdir("/vfs/emptydir") + +# umount +vfs.umount("/vfs") + +# mount read-only +fs = fs_factory() +vfs.mount(fs, "/vfs", readonly=True) + +# test reading works +with open("/vfs/file") as f: + print("file 2:", f.read()) + +# test writing fails +try: + with open("/vfs/test_write", "w"): + pass + print("opened") +except OSError as er: + print(repr(er)) + +# test removing fails +try: + os.unlink("/vfs/file") + print("unlinked") +except OSError as er: + print(repr(er)) + +# test renaming fails +try: + os.rename("/vfs/file2", "/vfs/renamed") + print("renamed") +except OSError as er: + print(repr(er)) + +# test removing directory fails +try: + os.rmdir("/vfs/emptydir") + print("rmdir'd") +except OSError as er: + print(repr(er)) + +# test creating directory fails +try: + os.mkdir("/vfs/emptydir2") + print("mkdir'd") +except OSError as er: + print(repr(er)) + +# umount +vfs.umount("/vfs") + +fs = fs_factory() +vfs.mount(fs, "/vfs") + +os.rmdir("/vfs/emptydir") +os.unlink("/vfs/file") + +os.rmdir(temp_dir) diff --git a/tests/extmod/vfs_posix_readonly.py.exp b/tests/extmod/vfs_posix_readonly.py.exp new file mode 100644 index 0000000000000..40e4316775ff3 --- /dev/null +++ b/tests/extmod/vfs_posix_readonly.py.exp @@ -0,0 +1,7 @@ +file: content +file 2: content +OSError(30,) +OSError(30,) +OSError(30,) +OSError(30,) +OSError(30,) diff --git a/tests/extmod/vfs_rom.py b/tests/extmod/vfs_rom.py index 770b6863b9c43..18ae1f5cf96c8 100644 --- a/tests/extmod/vfs_rom.py +++ b/tests/extmod/vfs_rom.py @@ -25,7 +25,7 @@ # An mpy file with four constant objects: str, bytes, long-int, float. test_mpy = ( # header - b"M\x06\x00\x1f" # mpy file header + b"M\x06\x00\x1e" # mpy file header, -msmall-int-bits=30 b"\x06" # n_qstr b"\x05" # n_obj # qstrs @@ -394,6 +394,7 @@ class TestMounted(TestBase): def setUp(self): self.orig_sys_path = list(sys.path) self.orig_cwd = os.getcwd() + sys.path = [] vfs.mount(vfs.VfsRom(self.romfs), "/test_rom") def tearDown(self): diff --git a/tests/extmod/websocket_toobig.py b/tests/extmod/websocket_toobig.py new file mode 100644 index 0000000000000..f4c5a74bbceac --- /dev/null +++ b/tests/extmod/websocket_toobig.py @@ -0,0 +1,28 @@ +try: + import io + import errno + import websocket +except ImportError: + print("SKIP") + raise SystemExit + +try: + buf = "x" * 65536 +except MemoryError: + print("SKIP") + raise SystemExit + + +# do a websocket write and then return the raw data from the stream +def ws_write(msg, sz): + s = io.BytesIO() + ws = websocket.websocket(s) + ws.write(msg) + s.seek(0) + return s.read(sz) + + +try: + print(ws_write(buf, 1)) +except OSError as e: + print("ioctl: ENOBUFS:", e.errno == errno.ENOBUFS) diff --git a/tests/extmod/websocket_toobig.py.exp b/tests/extmod/websocket_toobig.py.exp new file mode 100644 index 0000000000000..3bbd95282fdfe --- /dev/null +++ b/tests/extmod/websocket_toobig.py.exp @@ -0,0 +1 @@ +ioctl: ENOBUFS: True diff --git a/tests/extmod_hardware/machine_counter.py b/tests/extmod_hardware/machine_counter.py new file mode 100644 index 0000000000000..62ac1fed47ce7 --- /dev/null +++ b/tests/extmod_hardware/machine_counter.py @@ -0,0 +1,90 @@ +# Test machine.Counter implementation +# +# IMPORTANT: This test requires hardware connections: the out_pin and in_pin +# must be wired together. + +try: + from machine import Counter +except ImportError: + print("SKIP") + raise SystemExit + +import sys +from machine import Pin + +if "esp32" in sys.platform: + id = 0 + out_pin = 4 + in_pin = 5 +else: + print("Please add support for this test on this platform.") + raise SystemExit + +import unittest + +out_pin = Pin(out_pin, mode=Pin.OUT) +in_pin = Pin(in_pin, mode=Pin.IN) + + +def toggle(times): + for _ in range(times): + out_pin(1) + out_pin(0) + + +class TestCounter(unittest.TestCase): + def setUp(self): + out_pin(0) + self.counter = Counter(id, in_pin) + + def tearDown(self): + self.counter.deinit() + + def assertCounter(self, value): + self.assertEqual(self.counter.value(), value) + + def test_connections(self): + # Test the hardware connections are correct. If this test fails, all tests will fail. + out_pin(1) + self.assertEqual(1, in_pin()) + out_pin(0) + self.assertEqual(0, in_pin()) + + def test_count_rising(self): + self.assertCounter(0) + toggle(100) + self.assertCounter(100) + out_pin(1) + self.assertEqual(self.counter.value(0), 101) + self.assertCounter(0) # calling value(0) resets + out_pin(0) + self.assertCounter(0) # no rising edge + out_pin(1) + self.assertCounter(1) + + def test_change_directions(self): + self.assertCounter(0) + toggle(100) + self.assertCounter(100) + self.counter.init(in_pin, direction=Counter.DOWN) + self.assertCounter(0) # calling init() zeroes the counter + self.counter.value(100) # need to manually reset the value + self.assertCounter(100) + toggle(25) + self.assertCounter(75) + + def test_count_falling(self): + self.counter.init(in_pin, direction=Counter.UP, edge=Counter.FALLING) + toggle(20) + self.assertCounter(20) + out_pin(1) + self.assertCounter(20) # no falling edge + out_pin(0) + self.assertCounter(21) + self.counter.value(-(2**24)) + toggle(20) + self.assertCounter(-(2**24 - 20)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/extmod_hardware/machine_encoder.py b/tests/extmod_hardware/machine_encoder.py new file mode 100644 index 0000000000000..c218c8bfb646a --- /dev/null +++ b/tests/extmod_hardware/machine_encoder.py @@ -0,0 +1,153 @@ +# Test machine.Encoder implementation +# +# IMPORTANT: This test requires hardware connections: +# - out0_pin and in0_pin must be wired together. +# - out1_pin and in1_pin must be wired together. + +try: + from machine import Encoder +except ImportError: + print("SKIP") + raise SystemExit + +import sys +import unittest +from machine import Pin +from target_wiring import encoder_loopback_id, encoder_loopback_out_pins, encoder_loopback_in_pins + +PRINT = False +PIN_INIT_VALUE = 1 + +id = encoder_loopback_id +out0_pin, out1_pin = encoder_loopback_out_pins +in0_pin, in1_pin = encoder_loopback_in_pins + +out0_pin = Pin(out0_pin, mode=Pin.OUT) +in0_pin = Pin(in0_pin, mode=Pin.IN) +out1_pin = Pin(out1_pin, mode=Pin.OUT) +in1_pin = Pin(in1_pin, mode=Pin.IN) + + +class TestEncoder(unittest.TestCase): + def setUp(self): + out0_pin(PIN_INIT_VALUE) + out1_pin(PIN_INIT_VALUE) + self.enc = Encoder(id, in0_pin, in1_pin, phases=1) + self.enc2 = Encoder(id + 1, in0_pin, in1_pin, phases=2) + self.enc4 = Encoder(id + 2, in0_pin, in1_pin, phases=4) + self.pulses = 0 # track the expected encoder position in software + if PRINT: + print( + "\nout0_pin() out1_pin() enc.value() enc2.value() enc4.value() |", + out0_pin(), + out1_pin(), + "|", + self.enc.value(), + self.enc2.value(), + self.enc4.value(), + ) + + def tearDown(self): + self.enc.deinit() + try: + self.enc2.deinit() + except: + pass + try: + self.enc4.deinit() + except: + pass + + def rotate(self, pulses): + for _ in range(abs(pulses)): + self.pulses += 1 if (pulses > 0) else -1 + if pulses > 0: + if self.pulses % 2: + out0_pin(not out0_pin()) + else: + out1_pin(not out1_pin()) + else: + if self.pulses % 2: + out1_pin(not out1_pin()) + else: + out0_pin(not out0_pin()) + if PRINT: + print( + "out0_pin() out1_pin() enc.value() enc2.value() enc4.value() pulses self.pulses |", + out0_pin(), + out1_pin(), + "|", + self.enc.value(), + self.enc2.value(), + self.enc4.value(), + "|", + pulses, + self.pulses, + ) + + def assertPosition(self, value, value2=None, value4=None): + self.assertEqual(self.enc.value(), value) + if not value2 is None: + self.assertEqual(self.enc2.value(), value2) + if not value4 is None: + self.assertEqual(self.enc4.value(), value4) + pass + + @unittest.skipIf(sys.platform == "mimxrt", "cannot read back the pin") + def test_connections(self): + # Test the hardware connections are correct. If this test fails, all tests will fail. + for ch, outp, inp in ((0, out0_pin, in0_pin), (1, out1_pin, in1_pin)): + print("Testing channel ", ch) + outp(1) + self.assertEqual(1, inp()) + outp(0) + self.assertEqual(0, inp()) + + def test_basics(self): + self.assertPosition(0) + self.rotate(100) + self.assertPosition(100 // 4, 100 // 2, 100) + self.rotate(-100) + self.assertPosition(0) + + def test_partial(self): + # With phase=1 (default), need 4x pulses to count a rotation + self.assertPosition(0) + self.rotate(1) + self.assertPosition(1, 1, 1) + self.rotate(1) + self.assertPosition(1, 1, 2) + self.rotate(1) + self.assertPosition(1, 2, 3) + self.rotate(1) + self.assertPosition(1, 2, 4) # +4 + self.rotate(1) + self.assertPosition(2, 3, 5) + self.rotate(1) + self.assertPosition(2, 3, 6) + self.rotate(1) + self.assertPosition(2, 4, 7) + self.rotate(1) + self.assertPosition(2, 4, 8) # +4 + self.rotate(-1) + self.assertPosition(2, 4, 7) + self.rotate(-3) + self.assertPosition(1, 2, 4) # -4 + self.rotate(-4) + self.assertPosition(0, 0, 0) # -4 + self.rotate(-1) + self.assertPosition(0, 0, -1) + self.rotate(-1) + self.assertPosition(0, -1, -2) + self.rotate(-1) + self.assertPosition(0, -1, -3) + self.rotate(-1) + self.assertPosition(-1, -2, -4) # -4 + self.rotate(-1) + self.assertPosition(-1, -2, -5) + self.rotate(-3) + self.assertPosition(-2, -4, -8) # -4 + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/extmod_hardware/machine_i2c_target.py b/tests/extmod_hardware/machine_i2c_target.py new file mode 100644 index 0000000000000..763e6f4771e0f --- /dev/null +++ b/tests/extmod_hardware/machine_i2c_target.py @@ -0,0 +1,307 @@ +# Test machine.I2CTarget. +# +# IMPORTANT: This test requires hardware connections: a SoftI2C instance must be +# wired to a hardware I2C target. See pin definitions below. + +import sys + +try: + from machine import Pin, SoftI2C, I2CTarget +except ImportError: + print("SKIP") + raise SystemExit + +import unittest + +ADDR = 67 + +kwargs_target = {} + +# Configure pins based on the target. +if sys.platform == "alif" and sys.implementation._build == "ALIF_ENSEMBLE": + args_controller = {"scl": "P1_1", "sda": "P1_0"} + args_target = (0,) # on pins P0_3/P0_2 +elif sys.platform == "esp32": + args_controller = {"scl": 5, "sda": 6} + args_target = (0,) # on pins 9/8 for C3 and S3, 18/19 for others + kwargs_target = {"scl": 9, "sda": 8} +elif sys.platform == "rp2": + args_controller = {"scl": 5, "sda": 4} + args_target = (1,) +elif sys.platform == "pyboard": + if sys.implementation._build == "NUCLEO_WB55": + args_controller = {"scl": "B8", "sda": "B9"} + args_target = (3,) + else: + args_controller = {"scl": "X1", "sda": "X2"} + args_target = ("X",) +elif "zephyr-nucleo_wb55rg" in sys.implementation._machine: + # PB8=I2C1_SCL, PB9=I2C1_SDA (on Arduino header D15/D14) + # PC0=I2C3_SCL, PC1=I2C3_SDA (on Arduino header A0/A1) + args_controller = {"scl": Pin(("gpiob", 8)), "sda": Pin(("gpiob", 9))} + args_target = ("i2c3",) +elif "zephyr-rpi_pico" in sys.implementation._machine: + args_controller = {"scl": Pin(("gpio0", 5)), "sda": Pin(("gpio0", 4))} + args_target = ("i2c1",) # on gpio7/gpio6 +elif sys.platform == "mimxrt": + if "Teensy" in sys.implementation._machine: + args_controller = {"scl": "A6", "sda": "A3"} # D20/D17 + else: + args_controller = {"scl": "D0", "sda": "D1"} + args_target = (0,) # pins 19/18 On Teensy 4.x +elif sys.platform == "samd": + args_controller = {"scl": "D5", "sda": "D1"} + args_target = () +else: + print("Please add support for this test on this platform.") + raise SystemExit + + +def config_pull_up(): + Pin(args_controller["scl"], Pin.OPEN_DRAIN, Pin.PULL_UP) + Pin(args_controller["sda"], Pin.OPEN_DRAIN, Pin.PULL_UP) + + +class TestMemory(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.mem = bytearray(8) + cls.i2c = SoftI2C(**args_controller) + cls.i2c_target = I2CTarget(*args_target, **kwargs_target, addr=ADDR, mem=cls.mem) + config_pull_up() + + @classmethod + def tearDownClass(cls): + cls.i2c_target.deinit() + + def test_scan(self): + self.assertIn(ADDR, self.i2c.scan()) + + def test_write(self): + self.mem[:] = b"01234567" + self.i2c.writeto_mem(ADDR, 0, b"test") + self.assertEqual(self.mem, bytearray(b"test4567")) + self.i2c.writeto_mem(ADDR, 4, b"TEST") + self.assertEqual(self.mem, bytearray(b"testTEST")) + + def test_write_wrap(self): + self.mem[:] = b"01234567" + self.i2c.writeto_mem(ADDR, 6, b"test") + self.assertEqual(self.mem, bytearray(b"st2345te")) + + @unittest.skipIf(sys.platform == "esp32", "write lengths larger than buffer unsupported") + def test_write_wrap_large(self): + self.mem[:] = b"01234567" + self.i2c.writeto_mem(ADDR, 0, b"testTESTmore") + self.assertEqual(self.mem, bytearray(b"moreTEST")) + + def test_read(self): + self.mem[:] = b"01234567" + self.assertEqual(self.i2c.readfrom_mem(ADDR, 0, 4), b"0123") + self.assertEqual(self.i2c.readfrom_mem(ADDR, 4, 4), b"4567") + + def test_read_wrap(self): + self.mem[:] = b"01234567" + self.assertEqual(self.i2c.readfrom_mem(ADDR, 0, 4), b"0123") + self.assertEqual(self.i2c.readfrom_mem(ADDR, 2, 4), b"2345") + self.assertEqual(self.i2c.readfrom_mem(ADDR, 6, 4), b"6701") + + @unittest.skipIf(sys.platform == "esp32", "read lengths larger than buffer unsupported") + def test_read_wrap_large(self): + self.mem[:] = b"01234567" + self.assertEqual(self.i2c.readfrom_mem(ADDR, 0, 12), b"012345670123") + + def test_write_read(self): + self.mem[:] = b"01234567" + self.assertEqual(self.i2c.writeto(ADDR, b"\x02"), 1) + self.assertEqual(self.i2c.readfrom(ADDR, 4), b"2345") + + @unittest.skipIf(sys.platform == "esp32", "read after read unsupported") + def test_write_read_read(self): + self.mem[:] = b"01234567" + self.assertEqual(self.i2c.writeto(ADDR, b"\x02"), 1) + self.assertEqual(self.i2c.readfrom(ADDR, 4), b"2345") + self.assertEqual(self.i2c.readfrom(ADDR, 4), b"7012") + + +@unittest.skipUnless(hasattr(I2CTarget, "IRQ_END_READ"), "IRQ unsupported") +class TestMemoryIRQ(unittest.TestCase): + @staticmethod + def irq_handler(i2c_target): + flags = i2c_target.irq().flags() + TestMemoryIRQ.events[TestMemoryIRQ.num_events] = flags + TestMemoryIRQ.events[TestMemoryIRQ.num_events + 1] = i2c_target.memaddr + TestMemoryIRQ.num_events += 2 + + @classmethod + def setUpClass(cls): + cls.mem = bytearray(8) + cls.events = [0] * 8 + cls.num_events = 0 + cls.i2c = SoftI2C(**args_controller) + cls.i2c_target = I2CTarget(*args_target, **kwargs_target, addr=ADDR, mem=cls.mem) + cls.i2c_target.irq(TestMemoryIRQ.irq_handler) + config_pull_up() + + @classmethod + def tearDownClass(cls): + cls.i2c_target.deinit() + + @unittest.skipIf(sys.platform == "esp32", "scan doesn't trigger IRQ_END_WRITE") + def test_scan(self): + TestMemoryIRQ.num_events = 0 + self.i2c.scan() + self.assertEqual(self.events[: self.num_events], [I2CTarget.IRQ_END_WRITE, 0]) + + def test_write(self): + TestMemoryIRQ.num_events = 0 + self.mem[:] = b"01234567" + self.i2c.writeto_mem(ADDR, 2, b"test") + self.assertEqual(self.mem, bytearray(b"01test67")) + self.assertEqual(self.events[: self.num_events], [I2CTarget.IRQ_END_WRITE, 2]) + + def test_read(self): + TestMemoryIRQ.num_events = 0 + self.mem[:] = b"01234567" + self.assertEqual(self.i2c.readfrom_mem(ADDR, 2, 4), b"2345") + self.assertEqual(self.events[: self.num_events], [I2CTarget.IRQ_END_READ, 2]) + + +@unittest.skipUnless(hasattr(I2CTarget, "IRQ_WRITE_REQ"), "IRQ unsupported") +@unittest.skipIf(sys.platform == "mimxrt", "not working") +@unittest.skipIf(sys.platform == "pyboard", "can't queue more than one byte") +@unittest.skipIf(sys.platform == "samd", "not working") +@unittest.skipIf(sys.platform == "zephyr", "must call readinto/write in IRQ handler") +class TestPolling(unittest.TestCase): + @staticmethod + def irq_handler(i2c_target, buf=bytearray(1)): + flags = i2c_target.irq().flags() + if flags & I2CTarget.IRQ_READ_REQ: + i2c_target.write(b"0123") + + @classmethod + def setUpClass(cls): + cls.i2c = SoftI2C(**args_controller) + cls.i2c_target = I2CTarget(*args_target, addr=ADDR) + cls.i2c_target.irq( + TestPolling.irq_handler, + I2CTarget.IRQ_WRITE_REQ | I2CTarget.IRQ_READ_REQ, + hard=True, + ) + config_pull_up() + + @classmethod + def tearDownClass(cls): + cls.i2c_target.deinit() + + def test_read(self): + # Can't write data up front, must wait until IRQ_READ_REQ. + # self.assertEqual(self.i2c_target.write(b"abcd"), 4) + self.assertEqual(self.i2c.readfrom(ADDR, 4), b"0123") + + def test_write(self): + # Can do the read outside the IRQ, but requires IRQ_WRITE_REQ trigger to be set. + self.assertEqual(self.i2c.writeto(ADDR, b"0123"), 4) + buf = bytearray(8) + self.assertEqual(self.i2c_target.readinto(buf), 4) + self.assertEqual(buf, b"0123\x00\x00\x00\x00") + + +@unittest.skipUnless(hasattr(I2CTarget, "IRQ_ADDR_MATCH_READ"), "IRQ unsupported") +class TestIRQ(unittest.TestCase): + @staticmethod + def irq_handler(i2c_target, buf=bytearray(1)): + flags = i2c_target.irq().flags() + TestIRQ.events[TestIRQ.num_events] = flags + TestIRQ.num_events += 1 + if flags & I2CTarget.IRQ_READ_REQ: + i2c_target.write(b"Y") + if flags & I2CTarget.IRQ_WRITE_REQ: + i2c_target.readinto(buf) + TestIRQ.events[TestIRQ.num_events] = buf[0] + TestIRQ.num_events += 1 + + @classmethod + def setUpClass(cls): + cls.events = [0] * 8 + cls.num_events = 0 + cls.i2c = SoftI2C(**args_controller) + cls.i2c_target = I2CTarget(*args_target, addr=ADDR) + cls.i2c_target.irq( + TestIRQ.irq_handler, + I2CTarget.IRQ_ADDR_MATCH_READ + | I2CTarget.IRQ_ADDR_MATCH_WRITE + | I2CTarget.IRQ_WRITE_REQ + | I2CTarget.IRQ_READ_REQ + | I2CTarget.IRQ_END_READ + | I2CTarget.IRQ_END_WRITE, + hard=True, + ) + config_pull_up() + + @classmethod + def tearDownClass(cls): + cls.i2c_target.deinit() + + def test_scan(self): + TestIRQ.num_events = 0 + self.i2c.scan() + self.assertEqual( + self.events[: self.num_events], + [ + I2CTarget.IRQ_ADDR_MATCH_WRITE, + I2CTarget.IRQ_END_WRITE, + ], + ) + + def test_write(self): + TestIRQ.num_events = 0 + self.i2c.writeto(ADDR, b"XYZ") + self.assertEqual( + self.events[: self.num_events], + [ + I2CTarget.IRQ_ADDR_MATCH_WRITE, + I2CTarget.IRQ_WRITE_REQ, + ord(b"X"), + I2CTarget.IRQ_WRITE_REQ, + ord(b"Y"), + I2CTarget.IRQ_WRITE_REQ, + ord(b"Z"), + I2CTarget.IRQ_END_WRITE, + ], + ) + + def test_read(self): + TestIRQ.num_events = 0 + self.assertEqual(self.i2c.readfrom(ADDR, 1), b"Y") + self.assertEqual( + self.events[: self.num_events], + [ + I2CTarget.IRQ_ADDR_MATCH_READ, + I2CTarget.IRQ_READ_REQ, + I2CTarget.IRQ_READ_REQ, + I2CTarget.IRQ_END_READ, + ], + ) + + def test_write_read(self): + TestIRQ.num_events = 0 + self.i2c.writeto(ADDR, b"X", False) + self.assertEqual(self.i2c.readfrom(ADDR, 1), b"Y") + self.assertEqual( + self.events[: self.num_events], + [ + I2CTarget.IRQ_ADDR_MATCH_WRITE, + I2CTarget.IRQ_WRITE_REQ, + ord(b"X"), + I2CTarget.IRQ_END_WRITE, + I2CTarget.IRQ_ADDR_MATCH_READ, + I2CTarget.IRQ_READ_REQ, + I2CTarget.IRQ_READ_REQ, + I2CTarget.IRQ_END_READ, + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/feature_check/float.py b/tests/feature_check/float.py deleted file mode 100644 index d6d2a99d2429d..0000000000000 --- a/tests/feature_check/float.py +++ /dev/null @@ -1,13 +0,0 @@ -# detect how many bits of precision the floating point implementation has - -try: - float -except NameError: - print(0) -else: - if float("1.0000001") == float("1.0"): - print(30) - elif float("1e300") == float("inf"): - print(32) - else: - print(64) diff --git a/tests/feature_check/float.py.exp b/tests/feature_check/float.py.exp deleted file mode 100644 index 900731ffd51ff..0000000000000 --- a/tests/feature_check/float.py.exp +++ /dev/null @@ -1 +0,0 @@ -64 diff --git a/tests/feature_check/inlineasm_rv32_zba.py b/tests/feature_check/inlineasm_rv32_zba.py new file mode 100644 index 0000000000000..81228819042ee --- /dev/null +++ b/tests/feature_check/inlineasm_rv32_zba.py @@ -0,0 +1,10 @@ +# check if RISC-V 32 inline asm supported Zba opcodes + + +@micropython.asm_rv32 +def f(): + sh1add(a0, a0, a0) + + +f() +print("rv32_zba") diff --git a/tests/feature_check/inlineasm_rv32_zba.py.exp b/tests/feature_check/inlineasm_rv32_zba.py.exp new file mode 100644 index 0000000000000..fde22f5f40090 --- /dev/null +++ b/tests/feature_check/inlineasm_rv32_zba.py.exp @@ -0,0 +1 @@ +rv32_zba diff --git a/tests/feature_check/int_64.py b/tests/feature_check/int_64.py new file mode 100644 index 0000000000000..4d053782ca82b --- /dev/null +++ b/tests/feature_check/int_64.py @@ -0,0 +1,2 @@ +# Check whether 64-bit long integers are supported +print(1 << 62) diff --git a/tests/feature_check/int_64.py.exp b/tests/feature_check/int_64.py.exp new file mode 100644 index 0000000000000..aef5454e66263 --- /dev/null +++ b/tests/feature_check/int_64.py.exp @@ -0,0 +1 @@ +4611686018427387904 diff --git a/tests/feature_check/io_module.py b/tests/feature_check/io_module.py deleted file mode 100644 index 9094e605316ba..0000000000000 --- a/tests/feature_check/io_module.py +++ /dev/null @@ -1,6 +0,0 @@ -try: - import io - - print("io") -except ImportError: - print("no") diff --git a/tests/feature_check/io_module.py.exp b/tests/feature_check/io_module.py.exp deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/tests/feature_check/repl_emacs_check.py.exp b/tests/feature_check/repl_emacs_check.py.exp index 82a4e28ee4f84..5fe8ba1cd2df8 100644 --- a/tests/feature_check/repl_emacs_check.py.exp +++ b/tests/feature_check/repl_emacs_check.py.exp @@ -1,5 +1,5 @@ MicroPython \.\+ version -Use \.\+ +Type "help()" for more information. >>> # Check for emacs keys in REPL >>> t = \.\+ >>> t == 2 diff --git a/tests/feature_check/repl_words_move_check.py.exp b/tests/feature_check/repl_words_move_check.py.exp index 82a4e28ee4f84..5fe8ba1cd2df8 100644 --- a/tests/feature_check/repl_words_move_check.py.exp +++ b/tests/feature_check/repl_words_move_check.py.exp @@ -1,5 +1,5 @@ MicroPython \.\+ version -Use \.\+ +Type "help()" for more information. >>> # Check for emacs keys in REPL >>> t = \.\+ >>> t == 2 diff --git a/tests/feature_check/target_info.py b/tests/feature_check/target_info.py index f60f3b319192e..e95530023d7cb 100644 --- a/tests/feature_check/target_info.py +++ b/tests/feature_check/target_info.py @@ -19,5 +19,21 @@ "xtensa", "xtensawin", "rv32imc", -][sys_mpy >> 10] -print(platform, arch) + "rv64imc", +][(sys_mpy >> 10) & 0x0F] +arch_flags = sys_mpy >> 16 +build = getattr(sys.implementation, "_build", "unknown") +thread = getattr(sys.implementation, "_thread", None) + +# Detect how many bits of precision the floating point implementation has. +try: + if float("1.0000001") == float("1.0"): + float_prec = 30 + elif float("1e300") == float("inf"): + float_prec = 32 + else: + float_prec = 64 +except NameError: + float_prec = 0 + +print(platform, arch, arch_flags, build, thread, float_prec, len("α") == 1) diff --git a/tests/float/cmath_fun.py b/tests/float/cmath_fun.py index 39011733b02b4..0037d7c65596c 100644 --- a/tests/float/cmath_fun.py +++ b/tests/float/cmath_fun.py @@ -51,6 +51,9 @@ print("%.5g" % ret) elif type(ret) == tuple: print("%.5g %.5g" % ret) + elif f_name == "exp": + # exp amplifies REPR_C inaccuracies, so we need to check one digit less + print("complex(%.4g, %.4g)" % (real, ret.imag)) else: # some test (eg cmath.sqrt(-0.5)) disagree with CPython with tiny real part real = ret.real diff --git a/tests/float/float_array.py b/tests/float/float_array.py index 3d128da83819f..cfff3b220c657 100644 --- a/tests/float/float_array.py +++ b/tests/float/float_array.py @@ -19,4 +19,10 @@ def test(a): test(array("f")) test(array("d")) -print("{:.4f}".format(array("f", bytes(array("I", [0x3DCCCCCC])))[0])) +# hand-crafted floats, including non-standard nan +for float_hex in (0x3DCCCCCC, 0x7F800024, 0x7FC00004): + f = array("f", bytes(array("I", [float_hex])))[0] + if type(f) is float: + print("{:.4e}".format(f)) + else: + print(f) diff --git a/tests/float/float_format.py b/tests/float/float_format.py index 98ed0eb096fa4..0eb8b232b063a 100644 --- a/tests/float/float_format.py +++ b/tests/float/float_format.py @@ -2,14 +2,25 @@ # general rounding for val in (116, 1111, 1234, 5010, 11111): - print("%.0f" % val) - print("%.1f" % val) - print("%.3f" % val) + print("Test on %d / 1000:" % val) + for fmt in ("%.5e", "%.3e", "%.1e", "%.0e", "%.3f", "%.1f", "%.0f", "%.3g", "%.1g", "%.0g"): + print(fmt, fmt % (val / 1000)) + +# make sure round-up to the next unit is handled properly +for val in range(4, 9): + divi = 10**val + print("Test on 99994 / (10 ** %d):" % val) + for fmt in ("%.5e", "%.3e", "%.1e", "%.0e", "%.3f", "%.1f", "%.0f", "%.3g", "%.1g", "%.0g"): + print(fmt, fmt % (99994 / divi)) # make sure rounding is done at the correct precision for prec in range(8): print(("%%.%df" % prec) % 6e-5) +# make sure trailing zeroes are added properly +for prec in range(8): + print(("%%.%df" % prec) % 1e19) + # check certain cases that had a digit value of 10 render as a ":" character print("%.2e" % float("9" * 51 + "e-39")) print("%.2e" % float("9" * 40 + "e-21")) diff --git a/tests/float/float_format_accuracy.py b/tests/float/float_format_accuracy.py new file mode 100644 index 0000000000000..f9467f9c05d89 --- /dev/null +++ b/tests/float/float_format_accuracy.py @@ -0,0 +1,73 @@ +# Test accuracy of `repr` conversions. +# This test also increases code coverage for corner cases. + +try: + import array, math, random +except ImportError: + print("SKIP") + raise SystemExit + +# The largest errors come from seldom used very small numbers, near the +# limit of the representation. So we keep them out of this test to keep +# the max relative error display useful. +if float("1e-100") == 0.0: + # single-precision + float_type = "f" + float_size = 4 + # testing range + min_expo = -96 # i.e. not smaller than 1.0e-29 + # Expected results (given >=50'000 samples): + # - MICROPY_FLTCONV_IMPL_EXACT: 100% exact conversions + # - MICROPY_FLTCONV_IMPL_APPROX: >=98.53% exact conversions, max relative error <= 1.01e-7 + min_success = 0.980 # with only 1200 samples, the success rate is lower + max_rel_err = 1.1e-7 + # REPR_C is typically used with FORMAT_IMPL_BASIC, which has a larger error + is_REPR_C = float("1.0000001") == float("1.0") + if is_REPR_C: # REPR_C + min_success = 0.83 + max_rel_err = 5.75e-07 +else: + # double-precision + float_type = "d" + float_size = 8 + # testing range + min_expo = -845 # i.e. not smaller than 1.0e-254 + # Expected results (given >=200'000 samples): + # - MICROPY_FLTCONV_IMPL_EXACT: 100% exact conversions + # - MICROPY_FLTCONV_IMPL_APPROX: >=99.83% exact conversions, max relative error <= 2.7e-16 + min_success = 0.997 # with only 1200 samples, the success rate is lower + max_rel_err = 2.7e-16 + + +# Deterministic pseudorandom generator. Designed to be uniform +# on mantissa values and exponents, not on the represented number +def pseudo_randfloat(): + rnd_buff = bytearray(float_size) + for _ in range(float_size): + rnd_buff[_] = random.getrandbits(8) + return array.array(float_type, rnd_buff)[0] + + +random.seed(42) +stats = 0 +N = 1200 +max_err = 0 +for _ in range(N): + f = pseudo_randfloat() + while type(f) is not float or math.isinf(f) or math.isnan(f) or math.frexp(f)[1] <= min_expo: + f = pseudo_randfloat() + + str_f = repr(f) + f2 = float(str_f) + if f2 == f: + stats += 1 + else: + error = abs((f2 - f) / f) + if max_err < error: + max_err = error + +print(N, "values converted") +if stats / N >= min_success and max_err <= max_rel_err: + print("float format accuracy OK") +else: + print("FAILED: repr rate=%.3f%% max_err=%.3e" % (100 * stats / N, max_err)) diff --git a/tests/float/float_format_ints.py b/tests/float/float_format_ints.py index df4444166c5fa..7b7b30c4b340b 100644 --- a/tests/float/float_format_ints.py +++ b/tests/float/float_format_ints.py @@ -12,14 +12,42 @@ print(title, "with format", f_fmt, "gives", f_fmt.format(f)) print(title, "with format", g_fmt, "gives", g_fmt.format(f)) +# The tests below check border cases involving all mantissa bits. +# In case of REPR_C, where the mantissa is missing two bits, the +# the string representation for such numbers might not always be exactly +# the same but nevertheless be correct, so we must allow a few exceptions. +is_REPR_C = float("1.0000001") == float("1.0") + # 16777215 is 2^24 - 1, the largest integer that can be completely held # in a float32. -print("{:f}".format(16777215)) +val_str = "{:f}".format(16777215) + +# When using REPR_C, 16777215.0 is the same as 16777212.0 or 16777214.4 +# (depending on the implementation of pow() function, the result may differ) +if is_REPR_C and (val_str == "16777212.000000" or val_str == "16777214.400000"): + val_str = "16777215.000000" + +print(val_str) + # 4294967040 = 16777215 * 128 is the largest integer that is exactly # represented by a float32 and that will also fit within a (signed) int32. # The upper bound of our integer-handling code is actually double this, # but that constant might cause trouble on systems using 32 bit ints. -print("{:f}".format(2147483520)) +val_str = "{:f}".format(2147483520) + +# When using FLOAT_IMPL_FLOAT, 2147483520.0 == 2147483500.0 +# Both representations are valid, the second being "simpler" +is_float32 = float("1e300") == float("inf") +if is_float32 and val_str == "2147483500.000000": + val_str = "2147483520.000000" + +# When using REPR_C, 2147483520.0 is the same as 2147483200.0 +# Both representations are valid, the second being "simpler" +if is_REPR_C and val_str == "2147483200.000000": + val_str = "2147483520.000000" + +print(val_str) + # Very large positive integers can be a test for precision and resolution. # This is a weird way to represent 1e38 (largest power of 10 for float32). print("{:.6e}".format(float("9" * 30 + "e8"))) diff --git a/tests/float/float_parse_doubleprec.py b/tests/float/float_parse_doubleprec.py index 81fcadcee88b4..c1b0b4823b038 100644 --- a/tests/float/float_parse_doubleprec.py +++ b/tests/float/float_parse_doubleprec.py @@ -19,3 +19,9 @@ print(float("1.00000000000000000000e-307")) print(float("10.0000000000000000000e-308")) print(float("100.000000000000000000e-309")) + +# ensure repr() adds an extra digit when needed for accurate parsing +print(float(repr(float("2.0") ** 100)) == float("2.0") ** 100) + +# ensure repr does not add meaningless extra digits (1.234999999999) +print(repr(1.2345)) diff --git a/tests/float/float_struct_e.py b/tests/float/float_struct_e.py index 403fbc5db4cde..ba4134f3393ed 100644 --- a/tests/float/float_struct_e.py +++ b/tests/float/float_struct_e.py @@ -32,7 +32,7 @@ for i in (j, -j): x = struct.pack("", "=", "^"): for fill in ("", " ", "0", "@"): for sign in ("", "+", "-", " "): - # An empty precision defaults to 6, but when uPy is + # An empty precision defaults to 6, but when MicroPython is # configured to use a float, we can only use a # precision of 6 with numbers less than 10 and still # get results that compare to CPython (which uses @@ -164,7 +164,7 @@ def test_fmt(conv, fill, alignment, sign, prefix, width, precision, type, arg): for alignment in ("", "<", ">", "=", "^"): for fill in ("", " ", "0", "@"): for sign in ("", "+", "-", " "): - # An empty precision defaults to 6, but when uPy is + # An empty precision defaults to 6, but when MicroPython is # configured to use a float, we can only use a # precision of 6 with numbers less than 10 and still # get results that compare to CPython (which uses diff --git a/tests/float/string_format_fp30.py b/tests/float/string_format_fp30.py deleted file mode 100644 index 5f0b213daa342..0000000000000 --- a/tests/float/string_format_fp30.py +++ /dev/null @@ -1,42 +0,0 @@ -def test(fmt, *args): - print("{:8s}".format(fmt) + ">" + fmt.format(*args) + "<") - - -test("{:10.4}", 123.456) -test("{:10.4e}", 123.456) -test("{:10.4e}", -123.456) -# test("{:10.4f}", 123.456) -# test("{:10.4f}", -123.456) -test("{:10.4g}", 123.456) -test("{:10.4g}", -123.456) -test("{:10.4n}", 123.456) -test("{:e}", 100) -test("{:f}", 200) -test("{:g}", 300) - -test("{:10.4E}", 123.456) -test("{:10.4E}", -123.456) -# test("{:10.4F}", 123.456) -# test("{:10.4F}", -123.456) -test("{:10.4G}", 123.456) -test("{:10.4G}", -123.456) - -test("{:06e}", float("inf")) -test("{:06e}", float("-inf")) -test("{:06e}", float("nan")) - -# The following fails right now -# test("{:10.1}", 0.0) - -print("%.0f" % (1.750000 % 0.08333333333)) -# Below isn't compatible with single-precision float -# print("%.1f" % (1.750000 % 0.08333333333)) -# print("%.2f" % (1.750000 % 0.08333333333)) -# print("%.12f" % (1.750000 % 0.08333333333)) - -# tests for errors in format string - -try: - "{:10.1b}".format(0.0) -except ValueError: - print("ValueError") diff --git a/tests/float/string_format_modulo.py b/tests/float/string_format_modulo.py index 3c206b7393588..55314643351db 100644 --- a/tests/float/string_format_modulo.py +++ b/tests/float/string_format_modulo.py @@ -6,7 +6,7 @@ print("%u" % 1.0) # these 3 have different behaviour in Python 3.x versions -# uPy raises a TypeError, following Python 3.5 (earlier versions don't) +# MicroPython raises a TypeError, following Python 3.5 (earlier versions don't) # print("%x" % 18.0) # print("%o" % 18.0) # print("%X" % 18.0) diff --git a/tests/float/string_format_modulo3.py b/tests/float/string_format_modulo3.py index f9d9c43cdf42d..f8aeeda20f29c 100644 --- a/tests/float/string_format_modulo3.py +++ b/tests/float/string_format_modulo3.py @@ -1,3 +1,3 @@ -# uPy and CPython outputs differ for the following +# Test corner cases where MicroPython and CPython outputs used to differ in the past print("%.1g" % -9.9) # round up 'g' with '-' sign print("%.2g" % 99.9) # round up diff --git a/tests/float/string_format_modulo3.py.exp b/tests/float/string_format_modulo3.py.exp deleted file mode 100644 index 71432b3404519..0000000000000 --- a/tests/float/string_format_modulo3.py.exp +++ /dev/null @@ -1,2 +0,0 @@ --10 -100 diff --git a/tests/frozen/frozentest.py b/tests/frozen/frozentest.py index 78cdd60bf0431..74bdfb3b267f2 100644 --- a/tests/frozen/frozentest.py +++ b/tests/frozen/frozentest.py @@ -1,4 +1,4 @@ -print("uPy") +print("interned") print("a long string that is not interned") print("a string that has unicode αβγ chars") print(b"bytes 1234\x01") diff --git a/tests/import/builtin_ext.py b/tests/import/builtin_ext.py index 9d2344d400dae..79ee901ea6774 100644 --- a/tests/import/builtin_ext.py +++ b/tests/import/builtin_ext.py @@ -1,3 +1,9 @@ +try: + import uos, utime +except ImportError: + print("SKIP") + raise SystemExit + # Verify that sys is a builtin. import sys diff --git a/tests/import/builtin_import.py b/tests/import/builtin_import.py index 734498d1be47b..364b0bae912ae 100644 --- a/tests/import/builtin_import.py +++ b/tests/import/builtin_import.py @@ -20,3 +20,12 @@ __import__("xyz", None, None, None, -1) except ValueError: print("ValueError") + +# globals is not checked for level=0 +__import__("builtins", "globals") + +# globals must be a dict (or None) for level>0 +try: + __import__("builtins", "globals", None, None, 1) +except TypeError: + print("TypeError") diff --git a/tests/import/import_broken.py b/tests/import/import_broken.py index 3c7cf4a498528..440922157ef10 100644 --- a/tests/import/import_broken.py +++ b/tests/import/import_broken.py @@ -1,3 +1,9 @@ +try: + Exception.__class__ +except AttributeError: + print("SKIP") + raise SystemExit + import sys, pkg # Modules we import are usually added to sys.modules. diff --git a/tests/import/import_file.py b/tests/import/import_file.py index 90ec4e41e77aa..4cf307641c62d 100644 --- a/tests/import/import_file.py +++ b/tests/import/import_file.py @@ -1,3 +1,7 @@ +if "__file__" not in globals(): + print("SKIP") + raise SystemExit + import import1b print(import1b.__file__) diff --git a/tests/import/import_override.py b/tests/import/import_override.py index 029ebe54c1f70..0144e78cb9f32 100644 --- a/tests/import/import_override.py +++ b/tests/import/import_override.py @@ -2,6 +2,10 @@ def custom_import(name, globals, locals, fromlist, level): + # CPython always tries to import _io, so just let that through as-is. + if name == "_io": + return orig_import(name, globals, locals, fromlist, level) + print("import", name, fromlist, level) class M: diff --git a/tests/import/import_override.py.exp b/tests/import/import_override.py.exp deleted file mode 100644 index 365248da6d847..0000000000000 --- a/tests/import/import_override.py.exp +++ /dev/null @@ -1,2 +0,0 @@ -import import1b None 0 -456 diff --git a/tests/import/import_override2.py b/tests/import/import_override2.py new file mode 100644 index 0000000000000..25aac44fe98d0 --- /dev/null +++ b/tests/import/import_override2.py @@ -0,0 +1,18 @@ +# test overriding __import__ combined with importing from the filesystem + + +def custom_import(name, globals, locals, fromlist, level): + if level > 0: + print("import", name, fromlist, level) + return orig_import(name, globals, locals, fromlist, level) + + +orig_import = __import__ +try: + __import__("builtins").__import__ = custom_import +except AttributeError: + print("SKIP") + raise SystemExit + +# import calls __import__ behind the scenes +import pkg7.subpkg1.subpkg2.mod3 diff --git a/tests/import/import_pkg7.py.exp b/tests/import/import_pkg7.py.exp deleted file mode 100644 index 8f21a615f6a61..0000000000000 --- a/tests/import/import_pkg7.py.exp +++ /dev/null @@ -1,8 +0,0 @@ -pkg __name__: pkg7 -pkg __name__: pkg7.subpkg1 -pkg __name__: pkg7.subpkg1.subpkg2 -mod1 -mod2 -mod1.foo -mod2.bar -ImportError diff --git a/tests/import/import_pkg9.py b/tests/import/import_pkg9.py index 4de028494f1a2..c2e0b618b6969 100644 --- a/tests/import/import_pkg9.py +++ b/tests/import/import_pkg9.py @@ -13,4 +13,4 @@ import pkg9.mod2 pkg9.mod1() -print(pkg9.mod2.__name__, type(pkg9.mod2).__name__) +print(pkg9.mod2.__name__, type(pkg9.mod2)) diff --git a/tests/import/import_star.py b/tests/import/import_star.py new file mode 100644 index 0000000000000..2cb21b877d728 --- /dev/null +++ b/tests/import/import_star.py @@ -0,0 +1,59 @@ +# test `from package import *` conventions, including __all__ support +# +# This test requires MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_BASIC_FEATURES + +try: + next(iter([]), 42) +except TypeError: + # 2-argument version of next() not supported + # we are probably not at MICROPY_CONFIG_ROM_LEVEL_BASIC_FEATURES + print("SKIP") + raise SystemExit + +# 1. test default visibility +from pkgstar_default import * + +print("visibleFun" in globals()) +print("VisibleClass" in globals()) +print("_hiddenFun" in globals()) +print("_HiddenClass" in globals()) +print(visibleFun()) + +# 2. test explicit visibility as defined by __all__ (as an array) +from pkgstar_all_array import * + +print("publicFun" in globals()) +print("PublicClass" in globals()) +print("unlistedFun" in globals()) +print("UnlistedClass" in globals()) +print("_privateFun" in globals()) +print("_PrivateClass" in globals()) +print(publicFun()) +# test dynamic import as used in asyncio +print("dynamicFun" in globals()) +print(dynamicFun()) + +# 3. test explicit visibility as defined by __all__ (as an tuple) +from pkgstar_all_tuple import * + +print("publicFun2" in globals()) +print("PublicClass2" in globals()) +print("unlistedFun2" in globals()) +print("UnlistedClass2" in globals()) +print(publicFun2()) + +# 4. test reporting of missing entries in __all__ +try: + from pkgstar_all_miss import * + + print("missed detection of incorrect __all__ definition") +except AttributeError as er: + print("AttributeError triggered for bad __all__ definition") + +# 5. test reporting of invalid __all__ definition +try: + from pkgstar_all_inval import * + + print("missed detection of incorrect __all__ definition") +except TypeError as er: + print("TypeError triggered for bad __all__ definition") diff --git a/tests/import/import_star_error.py b/tests/import/import_star_error.py index 9e1757b6ef5a2..73d9c863d6f51 100644 --- a/tests/import/import_star_error.py +++ b/tests/import/import_star_error.py @@ -1,5 +1,10 @@ # test errors with import * +if not hasattr(object, "__init__"): + # target doesn't have MICROPY_CPYTHON_COMPAT enabled, so doesn't check for "import *" + print("SKIP") + raise SystemExit + # 'import *' is not allowed in function scope try: exec("def foo(): from x import *") diff --git a/tests/import/module_getattr.py.exp b/tests/import/module_getattr.py.exp deleted file mode 100644 index bc59c12aa16bd..0000000000000 --- a/tests/import/module_getattr.py.exp +++ /dev/null @@ -1 +0,0 @@ -False diff --git a/tests/import/pkg7/subpkg1/subpkg2/mod3.py b/tests/import/pkg7/subpkg1/subpkg2/mod3.py index b0f4279fcf57b..f7f4c1e65f46f 100644 --- a/tests/import/pkg7/subpkg1/subpkg2/mod3.py +++ b/tests/import/pkg7/subpkg1/subpkg2/mod3.py @@ -5,7 +5,9 @@ print(bar) # attempted relative import beyond top-level package +# On older versions of CPython (eg 3.8) this is a ValueError, but on +# newer CPython (eg 3.11) and MicroPython it's an ImportError. try: from .... import mod1 -except ImportError: +except (ImportError, ValueError): print("ImportError") diff --git a/tests/import/pkgstar_all_array/__init__.py b/tests/import/pkgstar_all_array/__init__.py new file mode 100644 index 0000000000000..03b012123fe0c --- /dev/null +++ b/tests/import/pkgstar_all_array/__init__.py @@ -0,0 +1,49 @@ +__all__ = ["publicFun", "PublicClass", "dynamicFun"] + + +# Definitions below should always be imported by a star import +def publicFun(): + return 1 + + +class PublicClass: + def __init__(self): + self._val = 1 + + +# If __all__ support is enabled, definitions below +# should not be imported by a star import +def unlistedFun(): + return 0 + + +class UnlistedClass: + def __init__(self): + self._val = 0 + + +# Definitions below should be not be imported by a star import +# (they start with an underscore, and are not listed in __all__) +def _privateFun(): + return -1 + + +class _PrivateClass: + def __init__(self): + self._val = -1 + + +# Test lazy loaded function, as used by extmod/asyncio: +# Works with a star import only if __all__ support is enabled +_attrs = { + "dynamicFun": "funcs", +} + + +def __getattr__(attr): + mod = _attrs.get(attr, None) + if mod is None: + raise AttributeError(attr) + value = getattr(__import__(mod, globals(), locals(), True, 1), attr) + globals()[attr] = value + return value diff --git a/tests/import/pkgstar_all_array/funcs.py b/tests/import/pkgstar_all_array/funcs.py new file mode 100644 index 0000000000000..7540d70f66bce --- /dev/null +++ b/tests/import/pkgstar_all_array/funcs.py @@ -0,0 +1,2 @@ +def dynamicFun(): + return 777 diff --git a/tests/import/pkgstar_all_inval/__init__.py b/tests/import/pkgstar_all_inval/__init__.py new file mode 100644 index 0000000000000..7022476c19be8 --- /dev/null +++ b/tests/import/pkgstar_all_inval/__init__.py @@ -0,0 +1 @@ +__all__ = 42 diff --git a/tests/import/pkgstar_all_miss/__init__.py b/tests/import/pkgstar_all_miss/__init__.py new file mode 100644 index 0000000000000..f9bbb538072bf --- /dev/null +++ b/tests/import/pkgstar_all_miss/__init__.py @@ -0,0 +1,8 @@ +__all__ = ("existingFun", "missingFun") + + +def existingFun(): + return None + + +# missingFun is not defined, should raise an error on import diff --git a/tests/import/pkgstar_all_tuple/__init__.py b/tests/import/pkgstar_all_tuple/__init__.py new file mode 100644 index 0000000000000..433ddc8e97639 --- /dev/null +++ b/tests/import/pkgstar_all_tuple/__init__.py @@ -0,0 +1,22 @@ +__all__ = ("publicFun2", "PublicClass2") + + +# Definitions below should always be imported by a star import +def publicFun2(): + return 2 + + +class PublicClass2: + def __init__(self): + self._val = 2 + + +# If __all__ support is enabled, definitions below +# should not be imported by a star import +def unlistedFun2(): + return 0 + + +class UnlistedClass2: + def __init__(self): + self._val = 0 diff --git a/tests/import/pkgstar_default/__init__.py b/tests/import/pkgstar_default/__init__.py new file mode 100644 index 0000000000000..4947e4ce7f180 --- /dev/null +++ b/tests/import/pkgstar_default/__init__.py @@ -0,0 +1,20 @@ +# When __all__ is undefined, star import should only +# show objects that do not start with an underscore + + +def visibleFun(): + return 42 + + +class VisibleClass: + def __init__(self): + self._val = 42 + + +def _hiddenFun(): + return -1 + + +class _HiddenClass: + def __init__(self): + self._val = -1 diff --git a/tests/inlineasm/rv32/asmzba.py b/tests/inlineasm/rv32/asmzba.py new file mode 100644 index 0000000000000..75f3573c864d6 --- /dev/null +++ b/tests/inlineasm/rv32/asmzba.py @@ -0,0 +1,18 @@ +@micropython.asm_rv32 +def test_sh1add(a0, a1): + sh1add(a0, a0, a1) + + +@micropython.asm_rv32 +def test_sh2add(a0, a1): + sh2add(a0, a0, a1) + + +@micropython.asm_rv32 +def test_sh3add(a0, a1): + sh3add(a0, a0, a1) + + +print(hex(test_sh1add(10, 20))) +print(hex(test_sh2add(10, 20))) +print(hex(test_sh3add(10, 20))) diff --git a/tests/inlineasm/rv32/asmzba.py.exp b/tests/inlineasm/rv32/asmzba.py.exp new file mode 100644 index 0000000000000..5f56bd95642a4 --- /dev/null +++ b/tests/inlineasm/rv32/asmzba.py.exp @@ -0,0 +1,3 @@ +0x28 +0x3c +0x64 diff --git a/tests/inlineasm/thumb/asmerrors.py b/tests/inlineasm/thumb/asmerrors.py new file mode 100644 index 0000000000000..a26f9322540fd --- /dev/null +++ b/tests/inlineasm/thumb/asmerrors.py @@ -0,0 +1,4 @@ +try: + exec("@micropython.asm_thumb\ndef l():\n a = di(a2, a2, -1)") +except SyntaxError as e: + print(e) diff --git a/tests/inlineasm/thumb/asmerrors.py.exp b/tests/inlineasm/thumb/asmerrors.py.exp new file mode 100644 index 0000000000000..7590f171e5184 --- /dev/null +++ b/tests/inlineasm/thumb/asmerrors.py.exp @@ -0,0 +1 @@ +expecting an assembler instruction diff --git a/tests/inlineasm/xtensa/asmargs.py b/tests/inlineasm/xtensa/asmargs.py new file mode 100644 index 0000000000000..2bfccfcc69bb2 --- /dev/null +++ b/tests/inlineasm/xtensa/asmargs.py @@ -0,0 +1,44 @@ +# test passing arguments + + +@micropython.asm_xtensa +def arg0(): + movi(a2, 1) + + +print(arg0()) + + +@micropython.asm_xtensa +def arg1(a2): + addi(a2, a2, 1) + + +print(arg1(1)) + + +@micropython.asm_xtensa +def arg2(a2, a3): + add(a2, a2, a3) + + +print(arg2(1, 2)) + + +@micropython.asm_xtensa +def arg3(a2, a3, a4): + add(a2, a2, a3) + add(a2, a2, a4) + + +print(arg3(1, 2, 3)) + + +@micropython.asm_xtensa +def arg4(a2, a3, a4, a5): + add(a2, a2, a3) + add(a2, a2, a4) + add(a2, a2, a5) + + +print(arg4(1, 2, 3, 4)) diff --git a/tests/inlineasm/xtensa/asmargs.py.exp b/tests/inlineasm/xtensa/asmargs.py.exp new file mode 100644 index 0000000000000..e33a6964f46e8 --- /dev/null +++ b/tests/inlineasm/xtensa/asmargs.py.exp @@ -0,0 +1,5 @@ +1 +2 +3 +6 +10 diff --git a/tests/inlineasm/xtensa/asmarith.py b/tests/inlineasm/xtensa/asmarith.py new file mode 100644 index 0000000000000..1c0934eb7a631 --- /dev/null +++ b/tests/inlineasm/xtensa/asmarith.py @@ -0,0 +1,119 @@ +@micropython.asm_xtensa +def f1(a2): + abs_(a2, a2) + + +for value in (10, -10, 0): + print(f1(value)) + + +ADDMI_TEMPLATE = """ +@micropython.asm_xtensa +def f1(a2) -> int: + addmi(a2, a2, {}) +print(f1(0)) +""" + +for value in (-32768, -32767, 32512, 32513, 0): + try: + exec(ADDMI_TEMPLATE.format(value)) + except SyntaxError as error: + print(error) + + +@micropython.asm_xtensa +def a2(a2, a3) -> int: + addx2(a2, a2, a3) + + +@micropython.asm_xtensa +def a4(a2, a3) -> int: + addx4(a2, a2, a3) + + +@micropython.asm_xtensa +def a8(a2, a3) -> int: + addx8(a2, a2, a3) + + +@micropython.asm_xtensa +def s2(a2, a3) -> int: + subx2(a2, a2, a3) + + +@micropython.asm_xtensa +def s4(a2, a3) -> int: + subx4(a2, a2, a3) + + +@micropython.asm_xtensa +def s8(a2, a3) -> int: + subx8(a2, a2, a3) + + +for first, second in ((100, 100), (-100, 100), (-100, -100), (100, -100)): + print("a2", a2(first, second)) + print("a4", a4(first, second)) + print("a8", a8(first, second)) + print("s2", s2(first, second)) + print("s4", s4(first, second)) + print("s8", s8(first, second)) + + +@micropython.asm_xtensa +def f5(a2) -> int: + neg(a2, a2) + + +for value in (0, -100, 100): + print(f5(value)) + + +@micropython.asm_xtensa +def f6(): + movi(a2, 0x100) + movi(a3, 1) + add(a2, a2, a3) + addi(a2, a2, 1) + addi(a2, a2, -2) + sub(a2, a2, a3) + + +print(hex(f6())) + + +@micropython.asm_xtensa +def f7(): + movi(a2, 0x10FF) + movi(a3, 1) + and_(a4, a2, a3) + or_(a4, a4, a3) + movi(a3, 0x200) + xor(a2, a4, a3) + + +print(hex(f7())) + + +@micropython.asm_xtensa +def f8(a2, a3): + add_n(a2, a2, a3) + + +print(f8(100, 200)) + + +@micropython.asm_xtensa +def f9(a2): + addi_n(a2, a2, 1) + + +print(f9(100)) + + +@micropython.asm_xtensa +def f10(a2, a3) -> uint: + mull(a2, a2, a3) + + +print(hex(f10(0xC0000000, 2))) diff --git a/tests/inlineasm/xtensa/asmarith.py.exp b/tests/inlineasm/xtensa/asmarith.py.exp new file mode 100644 index 0000000000000..7aba46a27d9b5 --- /dev/null +++ b/tests/inlineasm/xtensa/asmarith.py.exp @@ -0,0 +1,40 @@ +10 +10 +0 +-32768 +-32767 is not a multiple of 256 +32512 +'addmi' integer 32513 isn't within range -32768..32512 +0 +a2 300 +a4 500 +a8 900 +s2 100 +s4 300 +s8 700 +a2 -100 +a4 -300 +a8 -700 +s2 -300 +s4 -500 +s8 -900 +a2 -300 +a4 -500 +a8 -900 +s2 -100 +s4 -300 +s8 -700 +a2 100 +a4 300 +a8 700 +s2 300 +s4 500 +s8 900 +0 +100 +-100 +0xff +0x201 +300 +101 +0x80000000 diff --git a/tests/inlineasm/xtensa/asmbranch.py b/tests/inlineasm/xtensa/asmbranch.py new file mode 100644 index 0000000000000..22bcd5a7c71a3 --- /dev/null +++ b/tests/inlineasm/xtensa/asmbranch.py @@ -0,0 +1,299 @@ +# test branch instructions + + +@micropython.asm_xtensa +def tball(a2, a3) -> int: + mov(a4, a2) + movi(a2, 0) + ball(a4, a3, end) + movi(a2, -1) + label(end) + + +print(tball(0xFFFFFFFF, 0xFFFFFFFF)) +print(tball(0xFFFEFFFF, 0xFFFFFFFF)) +print(tball(0x00000000, 0xFFFFFFFF)) +print(tball(0xFFFFFFFF, 0x01010101)) + + +@micropython.asm_xtensa +def tbany(a2, a3) -> int: + mov(a4, a2) + movi(a2, 0) + bany(a4, a3, end) + movi(a2, -1) + label(end) + + +print(tbany(0xFFFFFFFF, 0xFFFFFFFF)) +print(tbany(0xFFFEFFFF, 0xFFFFFFFF)) +print(tbany(0x00000000, 0xFFFFFFFF)) +print(tbany(0xFFFFFFFF, 0x01010101)) + + +@micropython.asm_xtensa +def tbbc(a2, a3) -> int: + mov(a4, a2) + movi(a2, 0) + bbc(a4, a3, end) + movi(a2, -1) + label(end) + + +print(tbbc(0xFFFFFFFF, 4)) +print(tbbc(0xFFFEFFFF, 16)) +print(tbbc(0x00000000, 1)) + + +BBCI_TEMPLATE = """ +@micropython.asm_xtensa +def tbbci(a2) -> int: + mov(a3, a2) + movi(a2, 0) + bbci(a3, {}, end) + movi(a2, -1) + label(end) + +print(tbbci({})) +""" + + +for value, bit in ((0xFFFFFFFF, 4), (0xFFFEFFFF, 16), (0x00000000, 1)): + try: + exec(BBCI_TEMPLATE.format(bit, value)) + except SyntaxError as error: + print(error) + + +@micropython.asm_xtensa +def tbbs(a2, a3) -> int: + mov(a4, a2) + movi(a2, 0) + bbs(a4, a3, end) + movi(a2, -1) + label(end) + + +print(tbbs(0x00000000, 4)) +print(tbbs(0x00010000, 16)) +print(tbbs(0xFFFFFFFF, 1)) + + +BBSI_TEMPLATE = """ +@micropython.asm_xtensa +def tbbsi(a2) -> int: + mov(a3, a2) + movi(a2, 0) + bbsi(a3, {}, end) + movi(a2, -1) + label(end) + +print(tbbsi({})) +""" + + +for value, bit in ((0x00000000, 4), (0x00010000, 16), (0xFFFFFFFF, 1)): + try: + exec(BBSI_TEMPLATE.format(bit, value)) + except SyntaxError as error: + print(error) + + +@micropython.asm_xtensa +def tbeq(a2, a3) -> int: + mov(a4, a2) + movi(a2, 0) + beq(a4, a3, end) + movi(a2, -1) + label(end) + + +print(tbeq(0x00000000, 0x00000000)) +print(tbeq(0x00010000, 0x00000000)) +print(tbeq(0xFFFFFFFF, 0xFFFFFFFF)) + + +@micropython.asm_xtensa +def tbeqz(a2) -> int: + mov(a3, a2) + movi(a2, 0) + beqz(a3, end) + movi(a2, -1) + label(end) + + +print(tbeqz(0)) +print(tbeqz(0x12345678)) +print(tbeqz(-1)) + + +@micropython.asm_xtensa +def tbge(a2, a3) -> int: + mov(a4, a2) + movi(a2, 0) + bge(a4, a3, end) + movi(a2, -1) + label(end) + + +print(tbge(0x00000000, 0x00000000)) +print(tbge(0x00010000, 0x00000000)) +print(tbge(0xF0000000, 0xFFFFFFFF)) + + +@micropython.asm_xtensa +def tbgeu(a2, a3) -> int: + mov(a4, a2) + movi(a2, 0) + bgeu(a4, a3, end) + movi(a2, -1) + label(end) + + +print(tbgeu(0x00000000, 0x00000000)) +print(tbgeu(0x00010000, 0x00000000)) +print(tbgeu(0xF0000000, 0xFFFFFFFF)) +print(tbgeu(0xFFFFFFFF, 0xF0000000)) + + +@micropython.asm_xtensa +def tbgez(a2) -> int: + mov(a3, a2) + movi(a2, 0) + bgez(a3, end) + movi(a2, -1) + label(end) + + +print(tbgez(0)) +print(tbgez(0x12345678)) +print(tbgez(-1)) + + +@micropython.asm_xtensa +def tblt(a2, a3) -> int: + mov(a4, a2) + movi(a2, 0) + blt(a4, a3, end) + movi(a2, -1) + label(end) + + +print(tblt(0x00000000, 0x00000000)) +print(tblt(0x00010000, 0x00000000)) +print(tblt(0xF0000000, 0xFFFFFFFF)) + + +@micropython.asm_xtensa +def tbltu(a2, a3) -> int: + mov(a4, a2) + movi(a2, 0) + bltu(a4, a3, end) + movi(a2, -1) + label(end) + + +print(tbltu(0x00000000, 0x00000000)) +print(tbltu(0x00010000, 0x00000000)) +print(tbltu(0xF0000000, 0xFFFFFFFF)) +print(tbltu(0xFFFFFFFF, 0xF0000000)) + + +@micropython.asm_xtensa +def tbltz(a2) -> int: + mov(a3, a2) + movi(a2, 0) + bltz(a3, end) + movi(a2, -1) + label(end) + + +print(tbltz(0)) +print(tbltz(0x12345678)) +print(tbltz(-1)) + + +@micropython.asm_xtensa +def tbnall(a2, a3) -> int: + mov(a4, a2) + movi(a2, 0) + bnall(a4, a3, end) + movi(a2, -1) + label(end) + + +print(tbnall(0xFFFFFFFF, 0xFFFFFFFF)) +print(tbnall(0xFFFEFFFF, 0xFFFFFFFF)) +print(tbnall(0x00000000, 0xFFFFFFFF)) +print(tbnall(0xFFFFFFFF, 0x01010101)) + + +@micropython.asm_xtensa +def tbne(a2, a3) -> int: + mov(a4, a2) + movi(a2, 0) + bne(a4, a3, end) + movi(a2, -1) + label(end) + + +print(tbne(0x00000000, 0x00000000)) +print(tbne(0x00010000, 0x00000000)) +print(tbne(0xFFFFFFFF, 0xFFFFFFFF)) + + +@micropython.asm_xtensa +def tbnez(a2) -> int: + mov(a3, a2) + movi(a2, 0) + bnez(a3, end) + movi(a2, -1) + label(end) + + +print(tbnez(0)) +print(tbnez(0x12345678)) +print(tbnez(-1)) + + +@micropython.asm_xtensa +def tbnone(a2, a3) -> int: + mov(a4, a2) + movi(a2, 0) + bnone(a4, a3, end) + movi(a2, -1) + label(end) + + +print(tbnone(0xFFFFFFFF, 0xFFFFFFFF)) +print(tbnone(0xFFFEFFFF, 0xFFFFFFFF)) +print(tbnone(0x00000000, 0xFFFFFFFF)) +print(tbnone(0x10101010, 0x01010101)) + + +@micropython.asm_xtensa +def tbeqz_n(a2) -> int: + mov(a3, a2) + movi(a2, 0) + beqz_n(a3, end) + movi(a2, -1) + label(end) + + +print(tbeqz_n(0)) +print(tbeqz_n(0x12345678)) +print(tbeqz_n(-1)) + + +@micropython.asm_xtensa +def tbnez_n(a2) -> int: + mov(a3, a2) + movi(a2, 0) + bnez(a3, end) + movi(a2, -1) + label(end) + + +print(tbnez_n(0)) +print(tbnez_n(0x12345678)) +print(tbnez_n(-1)) diff --git a/tests/inlineasm/xtensa/asmbranch.py.exp b/tests/inlineasm/xtensa/asmbranch.py.exp new file mode 100644 index 0000000000000..319a4b435ef2f --- /dev/null +++ b/tests/inlineasm/xtensa/asmbranch.py.exp @@ -0,0 +1,66 @@ +0 +-1 +-1 +0 +0 +0 +-1 +0 +-1 +0 +0 +-1 +0 +0 +-1 +0 +0 +-1 +0 +0 +0 +-1 +0 +0 +-1 +-1 +0 +0 +-1 +0 +0 +-1 +0 +0 +0 +-1 +-1 +-1 +0 +-1 +-1 +0 +-1 +-1 +-1 +0 +-1 +0 +0 +-1 +-1 +0 +-1 +-1 +0 +0 +-1 +-1 +0 +0 +0 +-1 +-1 +-1 +0 +0 diff --git a/tests/inlineasm/xtensa/asmjump.py b/tests/inlineasm/xtensa/asmjump.py new file mode 100644 index 0000000000000..f41c9482319da --- /dev/null +++ b/tests/inlineasm/xtensa/asmjump.py @@ -0,0 +1,26 @@ +@micropython.asm_xtensa +def jump() -> int: + movi(a2, 0) + j(NEXT) + addi(a2, a2, 1) + j(DONE) + label(NEXT) + addi(a2, a2, 2) + label(DONE) + + +print(jump()) + + +@micropython.asm_xtensa +def jumpx() -> int: + call0(ENTRY) + label(ENTRY) + movi(a2, 0) + addi(a3, a0, 12) + jx(a3) + movi(a2, 1) + movi(a2, 2) + + +print(jumpx()) diff --git a/tests/inlineasm/xtensa/asmjump.py.exp b/tests/inlineasm/xtensa/asmjump.py.exp new file mode 100644 index 0000000000000..51993f072d583 --- /dev/null +++ b/tests/inlineasm/xtensa/asmjump.py.exp @@ -0,0 +1,2 @@ +2 +2 diff --git a/tests/inlineasm/xtensa/asmloadstore.py b/tests/inlineasm/xtensa/asmloadstore.py new file mode 100644 index 0000000000000..b185e30520cc9 --- /dev/null +++ b/tests/inlineasm/xtensa/asmloadstore.py @@ -0,0 +1,98 @@ +import array + +# On the 8266 the generated code gets put into the IRAM segment, which is only +# word-addressable. Therefore, to test byte and halfword load/store opcodes +# some memory must be reserved in the DRAM segment. + +BYTE_DATA = array.array("B", (0x11, 0x22, 0x33, 0x44)) +WORD_DATA = array.array("h", (100, 200, -100, -200)) +DWORD_DATA = array.array("i", (100_000, -200_000, 300_000, -400_000)) + + +@micropython.asm_xtensa +def tl32r() -> int: + nop() + j(CODE) + align(4) + label(DATA) + data(1, 1, 2, 3, 4, 5, 6, 7) + align(4) + label(CODE) + nop_n() + nop_n() + l32r(a2, DATA) + + +print(hex(tl32r())) + + +@micropython.asm_xtensa +def tl32i() -> uint: + call0(ENTRY) + label(ENTRY) + l32i(a2, a0, 0) + + +print(hex(tl32i())) + + +@micropython.asm_xtensa +def tl8ui(a2) -> uint: + mov(a3, a2) + l8ui(a2, a3, 1) + + +print(hex(tl8ui(BYTE_DATA))) + + +@micropython.asm_xtensa +def tl16ui(a2) -> uint: + mov(a3, a2) + l16ui(a2, a3, 2) + + +print(tl16ui(WORD_DATA)) + + +@micropython.asm_xtensa +def tl16si(a2) -> int: + mov(a3, a2) + l16si(a2, a3, 6) + + +print(tl16si(WORD_DATA)) + + +@micropython.asm_xtensa +def ts8i(a2, a3): + s8i(a3, a2, 1) + + +ts8i(BYTE_DATA, 0xFF) +print(BYTE_DATA) + + +@micropython.asm_xtensa +def ts16i(a2, a3): + s16i(a3, a2, 2) + + +ts16i(WORD_DATA, -123) +print(WORD_DATA) + + +@micropython.asm_xtensa +def ts32i(a2, a3) -> uint: + s32i(a3, a2, 4) + + +ts32i(DWORD_DATA, -123456) +print(DWORD_DATA) + + +@micropython.asm_xtensa +def tl32i_n(a2) -> uint: + l32i_n(a2, a2, 8) + + +print(tl32i_n(DWORD_DATA)) diff --git a/tests/inlineasm/xtensa/asmloadstore.py.exp b/tests/inlineasm/xtensa/asmloadstore.py.exp new file mode 100644 index 0000000000000..e6672df6f8132 --- /dev/null +++ b/tests/inlineasm/xtensa/asmloadstore.py.exp @@ -0,0 +1,9 @@ +0x4030201 +0xf8002022 +0x22 +200 +-200 +array('B', [17, 255, 51, 68]) +array('h', [100, -123, -100, -200]) +array('i', [100000, -123456, 300000, -400000]) +300000 diff --git a/tests/inlineasm/xtensa/asmmisc.py b/tests/inlineasm/xtensa/asmmisc.py new file mode 100644 index 0000000000000..271ab836625dc --- /dev/null +++ b/tests/inlineasm/xtensa/asmmisc.py @@ -0,0 +1,25 @@ +@micropython.asm_xtensa +def tnop(a2, a3, a4, a5): + nop() + + +out2 = tnop(0x100, 0x200, 0x300, 0x400) +print(out2 == 0x100) + + +@micropython.asm_xtensa +def tnop_n(a2, a3, a4, a5): + nop_n() + + +out2 = tnop_n(0x100, 0x200, 0x300, 0x400) +print(out2 == 0x100) + + +@micropython.asm_xtensa +def tmov_n(a2, a3): + mov_n(a4, a3) + add(a2, a4, a3) + + +print(tmov_n(0, 1)) diff --git a/tests/inlineasm/xtensa/asmmisc.py.exp b/tests/inlineasm/xtensa/asmmisc.py.exp new file mode 100644 index 0000000000000..eefaa35daf0f1 --- /dev/null +++ b/tests/inlineasm/xtensa/asmmisc.py.exp @@ -0,0 +1,3 @@ +True +True +2 diff --git a/tests/inlineasm/xtensa/asmshift.py b/tests/inlineasm/xtensa/asmshift.py new file mode 100644 index 0000000000000..271ca1ccd494c --- /dev/null +++ b/tests/inlineasm/xtensa/asmshift.py @@ -0,0 +1,137 @@ +@micropython.asm_xtensa +def lsl1(a2): + slli(a2, a2, 1) + + +print(hex(lsl1(0x123))) + + +@micropython.asm_xtensa +def lsl23(a2): + slli(a2, a2, 23) + + +print(hex(lsl23(1))) + + +@micropython.asm_xtensa +def lsr1(a2): + srli(a2, a2, 1) + + +print(hex(lsr1(0x123))) + + +@micropython.asm_xtensa +def lsr15(a2): + srli(a2, a2, 15) + + +print(hex(lsr15(0x80000000))) + + +@micropython.asm_xtensa +def asr1(a2): + srai(a2, a2, 1) + + +print(hex(asr1(0x123))) + + +@micropython.asm_xtensa +def asr31(a2): + srai(a2, a2, 31) + + +print(hex(asr31(0x80000000))) + + +@micropython.asm_xtensa +def lsl1r(a2): + movi(a3, 1) + ssl(a3) + sll(a2, a2) + + +print(hex(lsl1r(0x123))) + + +@micropython.asm_xtensa +def lsr1r(a2): + movi(a3, 1) + ssr(a3) + srl(a2, a2) + + +print(hex(lsr1r(0x123))) + + +@micropython.asm_xtensa +def asr1r(a2): + movi(a3, 1) + ssr(a3) + sra(a2, a2) + + +print(hex(asr1r(0x123))) + + +@micropython.asm_xtensa +def sll9(a2): + ssai(9) + sll(a2, a2) + + +print(hex(sll9(1))) + + +@micropython.asm_xtensa +def srlr(a2, a3): + ssa8l(a3) + srl(a2, a2) + + +print(hex(srlr(0x12340000, 2))) + + +@micropython.asm_xtensa +def sllr(a2, a3): + ssa8b(a3) + sll(a2, a2) + + +print(hex(sllr(0x1234, 2))) + + +@micropython.asm_xtensa +def srcr(a2, a3, a4): + ssr(a4) + src(a2, a2, a3) + + +print(hex(srcr(0x00000001, 0x80000000, 2))) + + +@micropython.asm_xtensa +def srai24(a2): + srai(a2, a2, 24) + + +print(hex(srai24(0x12345678))) + + +@micropython.asm_xtensa +def nsar(a2, a3): + nsa(a2, a3) + + +print(nsar(0x12345678, 0)) +print(nsar(0x12345678, -1)) + + +@micropython.asm_xtensa +def nsaur(a2, a3): + nsau(a2, a3) + + +print(nsaur(0x12345678, 0)) diff --git a/tests/inlineasm/xtensa/asmshift.py.exp b/tests/inlineasm/xtensa/asmshift.py.exp new file mode 100644 index 0000000000000..3e2bb3b4aefa9 --- /dev/null +++ b/tests/inlineasm/xtensa/asmshift.py.exp @@ -0,0 +1,17 @@ +0x246 +0x800000 +0x91 +0x10000 +0x91 +-0x1 +0x246 +0x91 +0x91 +0x800000 +0x1234 +0x12340000 +0x60000000 +0x12 +31 +31 +32 diff --git a/tests/internal_bench/class_create-0-empty.py b/tests/internal_bench/class_create-0-empty.py new file mode 100644 index 0000000000000..1fd8ccd925715 --- /dev/null +++ b/tests/internal_bench/class_create-0-empty.py @@ -0,0 +1,11 @@ +import bench + + +def test(num): + for i in range(num // 40): + + class X: + pass + + +bench.run(test) diff --git a/tests/internal_bench/class_create-1-slots.py b/tests/internal_bench/class_create-1-slots.py new file mode 100644 index 0000000000000..9b3e4b9570da2 --- /dev/null +++ b/tests/internal_bench/class_create-1-slots.py @@ -0,0 +1,12 @@ +import bench + + +def test(num): + l = ["x"] + for i in range(num // 40): + + class X: + __slots__ = l + + +bench.run(test) diff --git a/tests/internal_bench/class_create-1.1-slots5.py b/tests/internal_bench/class_create-1.1-slots5.py new file mode 100644 index 0000000000000..ccac77dec9daa --- /dev/null +++ b/tests/internal_bench/class_create-1.1-slots5.py @@ -0,0 +1,12 @@ +import bench + + +def test(num): + l = ["a", "b", "c", "d", "x"] + for i in range(num // 40): + + class X: + __slots__ = l + + +bench.run(test) diff --git a/tests/internal_bench/class_create-2-classattr.py b/tests/internal_bench/class_create-2-classattr.py new file mode 100644 index 0000000000000..049a7dab170c1 --- /dev/null +++ b/tests/internal_bench/class_create-2-classattr.py @@ -0,0 +1,11 @@ +import bench + + +def test(num): + for i in range(num // 40): + + class X: + x = 1 + + +bench.run(test) diff --git a/tests/internal_bench/class_create-2.1-classattr5.py b/tests/internal_bench/class_create-2.1-classattr5.py new file mode 100644 index 0000000000000..5051e7dcca70d --- /dev/null +++ b/tests/internal_bench/class_create-2.1-classattr5.py @@ -0,0 +1,15 @@ +import bench + + +def test(num): + for i in range(num // 40): + + class X: + a = 0 + b = 0 + c = 0 + d = 0 + x = 1 + + +bench.run(test) diff --git a/tests/internal_bench/class_create-2.3-classattr5objs.py b/tests/internal_bench/class_create-2.3-classattr5objs.py new file mode 100644 index 0000000000000..74540865dcd14 --- /dev/null +++ b/tests/internal_bench/class_create-2.3-classattr5objs.py @@ -0,0 +1,20 @@ +import bench + + +class Class: + pass + + +def test(num): + instance = Class() + for i in range(num // 40): + + class X: + a = instance + b = instance + c = instance + d = instance + x = instance + + +bench.run(test) diff --git a/tests/internal_bench/class_create-3-instancemethod.py b/tests/internal_bench/class_create-3-instancemethod.py new file mode 100644 index 0000000000000..e8c201cb2c3c6 --- /dev/null +++ b/tests/internal_bench/class_create-3-instancemethod.py @@ -0,0 +1,12 @@ +import bench + + +def test(num): + for i in range(num // 40): + + class X: + def x(self): + pass + + +bench.run(test) diff --git a/tests/internal_bench/class_create-4-classmethod.py b/tests/internal_bench/class_create-4-classmethod.py new file mode 100644 index 0000000000000..f34962bc67117 --- /dev/null +++ b/tests/internal_bench/class_create-4-classmethod.py @@ -0,0 +1,13 @@ +import bench + + +def test(num): + for i in range(num // 40): + + class X: + @classmethod + def x(cls): + pass + + +bench.run(test) diff --git a/tests/internal_bench/class_create-4.1-classmethod_implicit.py b/tests/internal_bench/class_create-4.1-classmethod_implicit.py new file mode 100644 index 0000000000000..f2d1fcfd18821 --- /dev/null +++ b/tests/internal_bench/class_create-4.1-classmethod_implicit.py @@ -0,0 +1,12 @@ +import bench + + +def test(num): + for i in range(num // 40): + + class X: + def __new__(cls): + pass + + +bench.run(test) diff --git a/tests/internal_bench/class_create-5-staticmethod.py b/tests/internal_bench/class_create-5-staticmethod.py new file mode 100644 index 0000000000000..0633556667544 --- /dev/null +++ b/tests/internal_bench/class_create-5-staticmethod.py @@ -0,0 +1,13 @@ +import bench + + +def test(num): + for i in range(num // 40): + + class X: + @staticmethod + def x(): + pass + + +bench.run(test) diff --git a/tests/internal_bench/class_create-6-getattribute.py b/tests/internal_bench/class_create-6-getattribute.py new file mode 100644 index 0000000000000..10a4fe7ce8d9f --- /dev/null +++ b/tests/internal_bench/class_create-6-getattribute.py @@ -0,0 +1,12 @@ +import bench + + +def test(num): + for i in range(num // 40): + + class X: + def __getattribute__(self, name): + pass + + +bench.run(test) diff --git a/tests/internal_bench/class_create-6.1-getattr.py b/tests/internal_bench/class_create-6.1-getattr.py new file mode 100644 index 0000000000000..b4b9ba2f5525b --- /dev/null +++ b/tests/internal_bench/class_create-6.1-getattr.py @@ -0,0 +1,12 @@ +import bench + + +def test(num): + for i in range(num // 40): + + class X: + def __getattr__(self, name): + pass + + +bench.run(test) diff --git a/tests/internal_bench/class_create-6.2-property.py b/tests/internal_bench/class_create-6.2-property.py new file mode 100644 index 0000000000000..cf847b6dc9c9f --- /dev/null +++ b/tests/internal_bench/class_create-6.2-property.py @@ -0,0 +1,13 @@ +import bench + + +def test(num): + for i in range(num // 40): + + class X: + @property + def x(self): + pass + + +bench.run(test) diff --git a/tests/internal_bench/class_create-6.3-descriptor.py b/tests/internal_bench/class_create-6.3-descriptor.py new file mode 100644 index 0000000000000..7b0a635726380 --- /dev/null +++ b/tests/internal_bench/class_create-6.3-descriptor.py @@ -0,0 +1,17 @@ +import bench + + +class D: + def __get__(self, instance, owner=None): + pass + + +def test(num): + descriptor = D() + for i in range(num // 40): + + class X: + x = descriptor + + +bench.run(test) diff --git a/tests/internal_bench/class_create-7-inherit.py b/tests/internal_bench/class_create-7-inherit.py new file mode 100644 index 0000000000000..f48fb215e0ab8 --- /dev/null +++ b/tests/internal_bench/class_create-7-inherit.py @@ -0,0 +1,14 @@ +import bench + + +def test(num): + class B: + pass + + for i in range(num // 40): + + class X(B): + pass + + +bench.run(test) diff --git a/tests/internal_bench/class_create-7.1-inherit_initsubclass.py b/tests/internal_bench/class_create-7.1-inherit_initsubclass.py new file mode 100644 index 0000000000000..0660fa86258fe --- /dev/null +++ b/tests/internal_bench/class_create-7.1-inherit_initsubclass.py @@ -0,0 +1,16 @@ +import bench + + +def test(num): + class B: + @classmethod + def __init_subclass__(cls): + pass + + for i in range(num // 40): + + class X(B): + pass + + +bench.run(test) diff --git a/tests/internal_bench/class_create-8-metaclass_setname.py b/tests/internal_bench/class_create-8-metaclass_setname.py new file mode 100644 index 0000000000000..e4515b54279d1 --- /dev/null +++ b/tests/internal_bench/class_create-8-metaclass_setname.py @@ -0,0 +1,17 @@ +import bench + + +class D: + def __set_name__(self, owner, name): + pass + + +def test(num): + descriptor = D() + for i in range(num // 40): + + class X: + x = descriptor + + +bench.run(test) diff --git a/tests/internal_bench/class_create-8.1-metaclass_setname5.py b/tests/internal_bench/class_create-8.1-metaclass_setname5.py new file mode 100644 index 0000000000000..5daa3f8471bca --- /dev/null +++ b/tests/internal_bench/class_create-8.1-metaclass_setname5.py @@ -0,0 +1,21 @@ +import bench + + +class D: + def __set_name__(self, owner, name): + pass + + +def test(num): + descriptor = D() + for i in range(num // 40): + + class X: + a = descriptor + b = descriptor + c = descriptor + d = descriptor + x = descriptor + + +bench.run(test) diff --git a/tests/internal_bench/class_instance-0-object.py b/tests/internal_bench/class_instance-0-object.py new file mode 100644 index 0000000000000..401c8ea7e3cc2 --- /dev/null +++ b/tests/internal_bench/class_instance-0-object.py @@ -0,0 +1,11 @@ +import bench + +X = object + + +def test(num): + for i in range(num // 5): + x = X() + + +bench.run(test) diff --git a/tests/internal_bench/class_instance-0.1-object-gc.py b/tests/internal_bench/class_instance-0.1-object-gc.py new file mode 100644 index 0000000000000..7c475963a8e81 --- /dev/null +++ b/tests/internal_bench/class_instance-0.1-object-gc.py @@ -0,0 +1,13 @@ +import bench +import gc + +X = object + + +def test(num): + for i in range(num // 5): + x = X() + gc.collect() + + +bench.run(test) diff --git a/tests/internal_bench/class_instance-1-empty.py b/tests/internal_bench/class_instance-1-empty.py new file mode 100644 index 0000000000000..617d47a86e92b --- /dev/null +++ b/tests/internal_bench/class_instance-1-empty.py @@ -0,0 +1,13 @@ +import bench + + +class X: + pass + + +def test(num): + for i in range(num // 5): + x = X() + + +bench.run(test) diff --git a/tests/internal_bench/class_instance-1.1-classattr.py b/tests/internal_bench/class_instance-1.1-classattr.py new file mode 100644 index 0000000000000..4e667533d4aa1 --- /dev/null +++ b/tests/internal_bench/class_instance-1.1-classattr.py @@ -0,0 +1,13 @@ +import bench + + +class X: + x = 0 + + +def test(num): + for i in range(num // 5): + x = X() + + +bench.run(test) diff --git a/tests/internal_bench/class_instance-1.2-func.py b/tests/internal_bench/class_instance-1.2-func.py new file mode 100644 index 0000000000000..21bf7a1ac48fc --- /dev/null +++ b/tests/internal_bench/class_instance-1.2-func.py @@ -0,0 +1,14 @@ +import bench + + +class X: + def f(self): + pass + + +def test(num): + for i in range(num // 5): + x = X() + + +bench.run(test) diff --git a/tests/internal_bench/class_instance-1.3-empty-gc.py b/tests/internal_bench/class_instance-1.3-empty-gc.py new file mode 100644 index 0000000000000..a5108ef8e8156 --- /dev/null +++ b/tests/internal_bench/class_instance-1.3-empty-gc.py @@ -0,0 +1,15 @@ +import bench +import gc + + +class X: + pass + + +def test(num): + for i in range(num // 5): + x = X() + gc.collect() + + +bench.run(test) diff --git a/tests/internal_bench/class_instance-2-init.py b/tests/internal_bench/class_instance-2-init.py new file mode 100644 index 0000000000000..86619d3154840 --- /dev/null +++ b/tests/internal_bench/class_instance-2-init.py @@ -0,0 +1,14 @@ +import bench + + +class X: + def __init__(self): + pass + + +def test(num): + for i in range(num // 5): + x = X() + + +bench.run(test) diff --git a/tests/internal_bench/class_instance-2.1-init_super.py b/tests/internal_bench/class_instance-2.1-init_super.py new file mode 100644 index 0000000000000..38bca5fef877a --- /dev/null +++ b/tests/internal_bench/class_instance-2.1-init_super.py @@ -0,0 +1,14 @@ +import bench + + +class X: + def __init__(self): + return super().__init__() + + +def test(num): + for i in range(num // 5): + x = X() + + +bench.run(test) diff --git a/tests/internal_bench/class_instance-2.2-new.py b/tests/internal_bench/class_instance-2.2-new.py new file mode 100644 index 0000000000000..dc5e78ea5ef98 --- /dev/null +++ b/tests/internal_bench/class_instance-2.2-new.py @@ -0,0 +1,14 @@ +import bench + + +class X: + def __new__(cls): + return super().__new__(cls) + + +def test(num): + for i in range(num // 5): + x = X() + + +bench.run(test) diff --git a/tests/internal_bench/class_instance-3-del.py b/tests/internal_bench/class_instance-3-del.py new file mode 100644 index 0000000000000..af700f72a94a8 --- /dev/null +++ b/tests/internal_bench/class_instance-3-del.py @@ -0,0 +1,14 @@ +import bench + + +class X: + def __del__(self): + pass + + +def test(num): + for i in range(num // 5): + x = X() + + +bench.run(test) diff --git a/tests/internal_bench/class_instance-3.1-del-gc.py b/tests/internal_bench/class_instance-3.1-del-gc.py new file mode 100644 index 0000000000000..311c71c3571b0 --- /dev/null +++ b/tests/internal_bench/class_instance-3.1-del-gc.py @@ -0,0 +1,16 @@ +import bench +import gc + + +class X: + def __del__(self): + pass + + +def test(num): + for i in range(num // 5): + x = X() + gc.collect() + + +bench.run(test) diff --git a/tests/internal_bench/class_instance-4-slots.py b/tests/internal_bench/class_instance-4-slots.py new file mode 100644 index 0000000000000..51b067fedf0b7 --- /dev/null +++ b/tests/internal_bench/class_instance-4-slots.py @@ -0,0 +1,13 @@ +import bench + + +class X: + __slots__ = ["x"] + + +def test(num): + for i in range(num // 5): + x = X() + + +bench.run(test) diff --git a/tests/internal_bench/class_instance-4.1-slots5.py b/tests/internal_bench/class_instance-4.1-slots5.py new file mode 100644 index 0000000000000..8f5c2ecb456d8 --- /dev/null +++ b/tests/internal_bench/class_instance-4.1-slots5.py @@ -0,0 +1,13 @@ +import bench + + +class X: + __slots__ = ["a", "b", "c", "d", "x"] + + +def test(num): + for i in range(num // 5): + x = X() + + +bench.run(test) diff --git a/tests/internal_bench/var-6.2-instance-speciallookup.py b/tests/internal_bench/var-6.2-instance-speciallookup.py new file mode 100644 index 0000000000000..fee12b2f930c3 --- /dev/null +++ b/tests/internal_bench/var-6.2-instance-speciallookup.py @@ -0,0 +1,19 @@ +import bench + + +class Foo: + def __init__(self): + self.num = 20000000 + + def __delattr__(self, name): # just trigger the 'special lookups' flag on the class + pass + + +def test(num): + o = Foo() + i = 0 + while i < o.num: + i += 1 + + +bench.run(test) diff --git a/tests/internal_bench/var-6.3-instance-property.py b/tests/internal_bench/var-6.3-instance-property.py new file mode 100644 index 0000000000000..b4426ef7928e1 --- /dev/null +++ b/tests/internal_bench/var-6.3-instance-property.py @@ -0,0 +1,17 @@ +import bench + + +class Foo: + @property + def num(self): + return 20000000 + + +def test(num): + o = Foo() + i = 0 + while i < o.num: + i += 1 + + +bench.run(test) diff --git a/tests/internal_bench/var-6.4-instance-descriptor.py b/tests/internal_bench/var-6.4-instance-descriptor.py new file mode 100644 index 0000000000000..b4df69f878f10 --- /dev/null +++ b/tests/internal_bench/var-6.4-instance-descriptor.py @@ -0,0 +1,20 @@ +import bench + + +class Descriptor: + def __get__(self, instance, owner=None): + return 20000000 + + +class Foo: + num = Descriptor() + + +def test(num): + o = Foo() + i = 0 + while i < o.num: + i += 1 + + +bench.run(test) diff --git a/tests/internal_bench/var-6.5-instance-getattr.py b/tests/internal_bench/var-6.5-instance-getattr.py new file mode 100644 index 0000000000000..3b2ef6721105f --- /dev/null +++ b/tests/internal_bench/var-6.5-instance-getattr.py @@ -0,0 +1,16 @@ +import bench + + +class Foo: + def __getattr__(self, name): + return 20000000 + + +def test(num): + o = Foo() + i = 0 + while i < o.num: + i += 1 + + +bench.run(test) diff --git a/tests/internal_bench/var-6.6-instance-builtin_ordered.py b/tests/internal_bench/var-6.6-instance-builtin_ordered.py new file mode 100644 index 0000000000000..02d7525230192 --- /dev/null +++ b/tests/internal_bench/var-6.6-instance-builtin_ordered.py @@ -0,0 +1,12 @@ +import bench + + +def test(num): + i = 0 + o = set() # object with largest rom-frozen ordered locals_dict + n = "__contains__" # last element in that dict for longest lookup + while i < num: + i += hasattr(o, n) # True, converts to 1 + + +bench.run(test) diff --git a/tests/internal_bench/var-9-getattr.py b/tests/internal_bench/var-9-getattr.py new file mode 100644 index 0000000000000..69d2bfed2e0ac --- /dev/null +++ b/tests/internal_bench/var-9-getattr.py @@ -0,0 +1,16 @@ +import bench + + +class Foo: + pass + + +def test(num): + o = Foo() + o.num = num + i = 0 + while i < getattr(o, "num", num): + i += 1 + + +bench.run(test) diff --git a/tests/internal_bench/var-9.1-getattr_default.py b/tests/internal_bench/var-9.1-getattr_default.py new file mode 100644 index 0000000000000..e803d39b32667 --- /dev/null +++ b/tests/internal_bench/var-9.1-getattr_default.py @@ -0,0 +1,15 @@ +import bench + + +class Foo: + pass + + +def test(num): + o = Foo() + i = 0 + while i < getattr(o, "num", num): + i += 1 + + +bench.run(test) diff --git a/tests/internal_bench/var-9.2-getattr_default_special.py b/tests/internal_bench/var-9.2-getattr_default_special.py new file mode 100644 index 0000000000000..c48ec0742cff6 --- /dev/null +++ b/tests/internal_bench/var-9.2-getattr_default_special.py @@ -0,0 +1,16 @@ +import bench + + +class Foo: + def __delattr__(self, name): # just trigger the 'special lookups' flag on the class + pass + + +def test(num): + o = Foo() + i = 0 + while i < getattr(o, "num", num): + i += 1 + + +bench.run(test) diff --git a/tests/internal_bench/var-9.3-except_ok.py b/tests/internal_bench/var-9.3-except_ok.py new file mode 100644 index 0000000000000..efc1a8f858cd0 --- /dev/null +++ b/tests/internal_bench/var-9.3-except_ok.py @@ -0,0 +1,23 @@ +import bench + + +class Foo: + pass + + +def test(num): + o = Foo() + o.num = num + + def get(): + try: + return o.num + except AttributeError: + return num + + i = 0 + while i < get(): + i += 1 + + +bench.run(test) diff --git a/tests/internal_bench/var-9.4-except_selfinduced.py b/tests/internal_bench/var-9.4-except_selfinduced.py new file mode 100644 index 0000000000000..544609ca4b6b3 --- /dev/null +++ b/tests/internal_bench/var-9.4-except_selfinduced.py @@ -0,0 +1,22 @@ +import bench + + +class Foo: + pass + + +def test(num): + o = Foo() + + def get(): + try: + raise AttributeError + except AttributeError: + return num + + i = 0 + while i < get(): + i += 1 + + +bench.run(test) diff --git a/tests/internal_bench/var-9.5-except_error.py b/tests/internal_bench/var-9.5-except_error.py new file mode 100644 index 0000000000000..caf83fa46ab3d --- /dev/null +++ b/tests/internal_bench/var-9.5-except_error.py @@ -0,0 +1,22 @@ +import bench + + +class Foo: + pass + + +def test(num): + o = Foo() + + def get(): + try: + return o.num + except AttributeError: + return num + + i = 0 + while i < get(): + i += 1 + + +bench.run(test) diff --git a/tests/internal_bench/var-9.6-except_error_special.py b/tests/internal_bench/var-9.6-except_error_special.py new file mode 100644 index 0000000000000..8bc395b4d7ef8 --- /dev/null +++ b/tests/internal_bench/var-9.6-except_error_special.py @@ -0,0 +1,23 @@ +import bench + + +class Foo: + def __delattr__(self, name): # just trigger the 'special lookups' flag on the class + pass + + +def test(num): + o = Foo() + + def get(): + try: + return o.num + except AttributeError: + return num + + i = 0 + while i < get(): + i += 1 + + +bench.run(test) diff --git a/tests/io/builtin_print_file.py b/tests/io/builtin_print_file.py index 822356a6cc305..e00f58635a9d2 100644 --- a/tests/io/builtin_print_file.py +++ b/tests/io/builtin_print_file.py @@ -13,5 +13,5 @@ try: print(file=1) -except (AttributeError, OSError): # CPython and uPy differ in error message +except (AttributeError, OSError): # CPython and MicroPython differ in error message print("Error") diff --git a/tests/io/file_seek.py b/tests/io/file_seek.py index 3990df8409051..b9f9593786fe6 100644 --- a/tests/io/file_seek.py +++ b/tests/io/file_seek.py @@ -30,5 +30,5 @@ try: f.seek(1) except (OSError, ValueError): - # CPy raises ValueError, uPy raises OSError + # CPy raises ValueError, MPy raises OSError print("OSError or ValueError") diff --git a/tests/micropython/builtin_execfile.py b/tests/micropython/builtin_execfile.py index 75a867bb94023..4fd4d66d4ee25 100644 --- a/tests/micropython/builtin_execfile.py +++ b/tests/micropython/builtin_execfile.py @@ -75,3 +75,24 @@ def open(self, file, mode): # Unmount the VFS object. vfs.umount(fs) + + +class EvilFilesystem: + def mount(self, readonly, mkfs): + print("mount", readonly, mkfs) + + def umount(self): + print("umount") + + def open(self, file, mode): + return None + + +fs = EvilFilesystem() +vfs.mount(fs, "/test_mnt") +try: + execfile("/test_mnt/test.py") + print("ExecFile succeeded") +except OSError: + print("OSError") +vfs.umount(fs) diff --git a/tests/micropython/builtin_execfile.py.exp b/tests/micropython/builtin_execfile.py.exp index 49703d570763d..d93dee547b6ea 100644 --- a/tests/micropython/builtin_execfile.py.exp +++ b/tests/micropython/builtin_execfile.py.exp @@ -5,3 +5,6 @@ open /test.py rb 123 TypeError umount +mount False False +OSError +umount diff --git a/tests/micropython/const_error.py b/tests/micropython/const_error.py index d35be530a7c8d..950360e4dc782 100644 --- a/tests/micropython/const_error.py +++ b/tests/micropython/const_error.py @@ -18,8 +18,6 @@ def test_syntax(code): # these operations are not supported within const test_syntax("A = const(1 @ 2)") -test_syntax("A = const(1 / 2)") -test_syntax("A = const(1 ** -2)") test_syntax("A = const(1 << -2)") test_syntax("A = const(1 >> -2)") test_syntax("A = const(1 % 0)") diff --git a/tests/micropython/const_error.py.exp b/tests/micropython/const_error.py.exp index 3edc3efe9c3e9..bef69eb32ea7d 100644 --- a/tests/micropython/const_error.py.exp +++ b/tests/micropython/const_error.py.exp @@ -5,5 +5,3 @@ SyntaxError SyntaxError SyntaxError SyntaxError -SyntaxError -SyntaxError diff --git a/tests/micropython/const_float.py b/tests/micropython/const_float.py new file mode 100644 index 0000000000000..c3a0df0276bf8 --- /dev/null +++ b/tests/micropython/const_float.py @@ -0,0 +1,23 @@ +# test constant optimisation, with consts that are floats + +from micropython import const + +# check we can make consts from floats +F1 = const(2.5) +F2 = const(-0.3) +print(type(F1), F1) +print(type(F2), F2) + +# check arithmetic with floats +F3 = const(F1 + F2) +F4 = const(F1**2) +print(F3, F4) + +# check int operations with float results +F5 = const(1 / 2) +F6 = const(2**-2) +print(F5, F6) + +# note: we also test float expression folding when +# we're compiling test cases in tests/float, as +# many expressions are resolved at compile time. diff --git a/tests/micropython/const_float.py.exp b/tests/micropython/const_float.py.exp new file mode 100644 index 0000000000000..17a86a6d936c2 --- /dev/null +++ b/tests/micropython/const_float.py.exp @@ -0,0 +1,4 @@ + 2.5 + -0.3 +2.2 6.25 +0.5 0.25 diff --git a/tests/micropython/const_math.py b/tests/micropython/const_math.py new file mode 100644 index 0000000000000..7ee5edc6d3240 --- /dev/null +++ b/tests/micropython/const_math.py @@ -0,0 +1,18 @@ +# Test expressions based on math module constants +try: + import math +except ImportError: + print("SKIP") + raise SystemExit + +from micropython import const + +# check that we can make consts from math constants +# (skip if the target has MICROPY_COMP_MODULE_CONST disabled) +try: + exec("two_pi = const(2.0 * math.pi)") +except SyntaxError: + print("SKIP") + raise SystemExit + +print(math.cos(two_pi)) diff --git a/tests/micropython/const_math.py.exp b/tests/micropython/const_math.py.exp new file mode 100644 index 0000000000000..d3827e75a5cad --- /dev/null +++ b/tests/micropython/const_math.py.exp @@ -0,0 +1 @@ +1.0 diff --git a/tests/micropython/decorator_error.py b/tests/micropython/decorator_error.py index 94772ac1e5fef..b5eae65b29209 100644 --- a/tests/micropython/decorator_error.py +++ b/tests/micropython/decorator_error.py @@ -1,4 +1,4 @@ -# test syntax errors for uPy-specific decorators +# test syntax errors for MicroPython-specific decorators def test_syntax(code): diff --git a/tests/micropython/emg_exc.py b/tests/micropython/emg_exc.py index a74757d028ff2..cf918b3dc8ac5 100644 --- a/tests/micropython/emg_exc.py +++ b/tests/micropython/emg_exc.py @@ -1,10 +1,9 @@ # test that emergency exceptions work -import micropython import sys try: - import io + import io, micropython except ImportError: print("SKIP") raise SystemExit diff --git a/tests/micropython/emg_exc.py.native.exp b/tests/micropython/emg_exc.py.native.exp new file mode 100644 index 0000000000000..879fa7ef5b943 --- /dev/null +++ b/tests/micropython/emg_exc.py.native.exp @@ -0,0 +1 @@ +ValueError(1,) diff --git a/tests/micropython/extreme_exc.py b/tests/micropython/extreme_exc.py index ad819e408fd82..17323b566d3b5 100644 --- a/tests/micropython/extreme_exc.py +++ b/tests/micropython/extreme_exc.py @@ -1,6 +1,10 @@ # test some extreme cases of allocating exceptions and tracebacks -import micropython +try: + import micropython +except ImportError: + print("SKIP") + raise SystemExit # Check for stackless build, which can't call functions without # allocating a frame on the heap. diff --git a/tests/micropython/heap_lock.py b/tests/micropython/heap_lock.py index f2892a6dc581e..209a3143461a5 100644 --- a/tests/micropython/heap_lock.py +++ b/tests/micropython/heap_lock.py @@ -1,6 +1,10 @@ # check that heap_lock/heap_unlock work as expected -import micropython +try: + import micropython +except ImportError: + print("SKIP") + raise SystemExit l = [] l2 = list(range(100)) diff --git a/tests/micropython/heap_locked.py b/tests/micropython/heap_locked.py index d9e5b5d4090af..d9d99493dd6b7 100644 --- a/tests/micropython/heap_locked.py +++ b/tests/micropython/heap_locked.py @@ -1,8 +1,10 @@ # test micropython.heap_locked() -import micropython +try: + import micropython -if not hasattr(micropython, "heap_locked"): + micropython.heap_locked +except (AttributeError, ImportError): print("SKIP") raise SystemExit diff --git a/tests/micropython/heapalloc.py b/tests/micropython/heapalloc.py index e19f8d0255deb..010bf878a07b0 100644 --- a/tests/micropython/heapalloc.py +++ b/tests/micropython/heapalloc.py @@ -1,6 +1,10 @@ # check that we can do certain things without allocating heap memory -import micropython +try: + import micropython +except ImportError: + print("SKIP") + raise SystemExit # Check for stackless build, which can't call functions without # allocating a frame on heap. diff --git a/tests/micropython/heapalloc_exc_compressed.py b/tests/micropython/heapalloc_exc_compressed.py index aa071d641c21c..96fe3ca4f7e00 100644 --- a/tests/micropython/heapalloc_exc_compressed.py +++ b/tests/micropython/heapalloc_exc_compressed.py @@ -1,4 +1,10 @@ -import micropython +try: + import micropython + + micropython.heap_lock +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit # Tests both code paths for built-in exception raising. # mp_obj_new_exception_msg_varg (exception requires decompression at raise-time to format) diff --git a/tests/micropython/heapalloc_exc_compressed_emg_exc.py b/tests/micropython/heapalloc_exc_compressed_emg_exc.py index 48ce9dd69e6e2..31d937b8f9345 100644 --- a/tests/micropython/heapalloc_exc_compressed_emg_exc.py +++ b/tests/micropython/heapalloc_exc_compressed_emg_exc.py @@ -1,4 +1,10 @@ -import micropython +try: + import micropython + + micropython.heap_lock +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit # Does the full test from heapalloc_exc_compressed.py but while the heap is # locked (this can only work when the emergency exception buf is enabled). diff --git a/tests/micropython/heapalloc_exc_raise.py b/tests/micropython/heapalloc_exc_raise.py index 99810e0075064..917ccf42f1e16 100644 --- a/tests/micropython/heapalloc_exc_raise.py +++ b/tests/micropython/heapalloc_exc_raise.py @@ -1,6 +1,12 @@ # Test that we can raise and catch (preallocated) exception # without memory allocation. -import micropython +try: + import micropython + + micropython.heap_lock +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit e = ValueError("error") diff --git a/tests/micropython/heapalloc_fail_bytearray.py b/tests/micropython/heapalloc_fail_bytearray.py index 1bf7ddd600b93..9ee73d1c583f2 100644 --- a/tests/micropython/heapalloc_fail_bytearray.py +++ b/tests/micropython/heapalloc_fail_bytearray.py @@ -1,6 +1,12 @@ # test handling of failed heap allocation with bytearray -import micropython +try: + import micropython + + micropython.heap_lock +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit class GetSlice: diff --git a/tests/micropython/heapalloc_fail_dict.py b/tests/micropython/heapalloc_fail_dict.py index ce2d158bd09c6..04c79183578ff 100644 --- a/tests/micropython/heapalloc_fail_dict.py +++ b/tests/micropython/heapalloc_fail_dict.py @@ -1,6 +1,12 @@ # test handling of failed heap allocation with dict -import micropython +try: + import micropython + + micropython.heap_lock +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit # create dict x = 1 diff --git a/tests/micropython/heapalloc_fail_list.py b/tests/micropython/heapalloc_fail_list.py index 9a2e9a555f3e2..7afb6dc1019b3 100644 --- a/tests/micropython/heapalloc_fail_list.py +++ b/tests/micropython/heapalloc_fail_list.py @@ -1,6 +1,12 @@ # test handling of failed heap allocation with list -import micropython +try: + import micropython + + micropython.heap_lock +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit class GetSlice: diff --git a/tests/micropython/heapalloc_fail_memoryview.py b/tests/micropython/heapalloc_fail_memoryview.py index da2d1abffa634..17e3e1262472c 100644 --- a/tests/micropython/heapalloc_fail_memoryview.py +++ b/tests/micropython/heapalloc_fail_memoryview.py @@ -1,6 +1,12 @@ # test handling of failed heap allocation with memoryview -import micropython +try: + import micropython + + micropython.heap_lock +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit class GetSlice: diff --git a/tests/micropython/heapalloc_fail_set.py b/tests/micropython/heapalloc_fail_set.py index 3c347660ad763..0c4d85eef62a8 100644 --- a/tests/micropython/heapalloc_fail_set.py +++ b/tests/micropython/heapalloc_fail_set.py @@ -1,6 +1,12 @@ # test handling of failed heap allocation with set -import micropython +try: + import micropython + + micropython.heap_lock +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit # create set x = 1 diff --git a/tests/micropython/heapalloc_fail_tuple.py b/tests/micropython/heapalloc_fail_tuple.py index de79385e3e3cb..11718a8107b3a 100644 --- a/tests/micropython/heapalloc_fail_tuple.py +++ b/tests/micropython/heapalloc_fail_tuple.py @@ -1,6 +1,12 @@ # test handling of failed heap allocation with tuple -import micropython +try: + import micropython + + micropython.heap_lock +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit # create tuple x = 1 diff --git a/tests/micropython/heapalloc_inst_call.py b/tests/micropython/heapalloc_inst_call.py index 14d8826bf06ce..f78aa3cf87798 100644 --- a/tests/micropython/heapalloc_inst_call.py +++ b/tests/micropython/heapalloc_inst_call.py @@ -1,6 +1,13 @@ # Test that calling clazz.__call__() with up to at least 3 arguments # doesn't require heap allocation. -import micropython + +try: + import micropython + + micropython.heap_lock +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit class Foo0: diff --git a/tests/micropython/heapalloc_int_from_bytes.py b/tests/micropython/heapalloc_int_from_bytes.py index 5fe50443ae335..3310ea95d1472 100644 --- a/tests/micropython/heapalloc_int_from_bytes.py +++ b/tests/micropython/heapalloc_int_from_bytes.py @@ -1,6 +1,13 @@ # Test that int.from_bytes() for small number of bytes generates # small int. -import micropython + +try: + import micropython + + micropython.heap_lock +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit micropython.heap_lock() print(int.from_bytes(b"1", "little")) diff --git a/tests/micropython/heapalloc_slice.py b/tests/micropython/heapalloc_slice.py new file mode 100644 index 0000000000000..62d96595c719e --- /dev/null +++ b/tests/micropython/heapalloc_slice.py @@ -0,0 +1,18 @@ +# slice operations that don't require allocation +try: + from micropython import heap_lock, heap_unlock +except (ImportError, AttributeError): + heap_lock = heap_unlock = lambda: 0 + +b = bytearray(range(10)) + +m = memoryview(b) + +heap_lock() + +b[3:5] = b"aa" +m[5:7] = b"bb" + +heap_unlock() + +print(b) diff --git a/tests/micropython/heapalloc_str.py b/tests/micropython/heapalloc_str.py index 39aa56ccd7834..6372df5d37b13 100644 --- a/tests/micropython/heapalloc_str.py +++ b/tests/micropython/heapalloc_str.py @@ -1,5 +1,12 @@ # String operations which don't require allocation -import micropython + +try: + import micropython + + micropython.heap_lock +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit micropython.heap_lock() diff --git a/tests/micropython/heapalloc_super.py b/tests/micropython/heapalloc_super.py index 51afae3d83d06..6839eee330acd 100644 --- a/tests/micropython/heapalloc_super.py +++ b/tests/micropython/heapalloc_super.py @@ -1,5 +1,12 @@ # test super() operations which don't require allocation -import micropython + +try: + import micropython + + micropython.heap_lock +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit # Check for stackless build, which can't call functions without # allocating a frame on heap. diff --git a/tests/micropython/heapalloc_traceback.py.native.exp b/tests/micropython/heapalloc_traceback.py.native.exp new file mode 100644 index 0000000000000..d6ac26aa829e1 --- /dev/null +++ b/tests/micropython/heapalloc_traceback.py.native.exp @@ -0,0 +1,3 @@ +StopIteration +StopIteration: + diff --git a/tests/micropython/heapalloc_yield_from.py b/tests/micropython/heapalloc_yield_from.py index 950717189087d..9d4f8c6940f16 100644 --- a/tests/micropython/heapalloc_yield_from.py +++ b/tests/micropython/heapalloc_yield_from.py @@ -1,6 +1,12 @@ # Check that yield-from can work without heap allocation -import micropython +try: + import micropython + + micropython.heap_lock +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit # Yielding from a function generator diff --git a/tests/micropython/import_mpy_native.py b/tests/micropython/import_mpy_native.py index ad63e025db2c8..59181b203bba9 100644 --- a/tests/micropython/import_mpy_native.py +++ b/tests/micropython/import_mpy_native.py @@ -54,12 +54,13 @@ def open(self, path, mode): # these are the test .mpy files # CIRCUITPY-CHANGE -valid_header = bytes([ord("C"), 6, mpy_arch, 31]) +small_int_bits = 31 +valid_header = bytes([77, 6, (mpy_arch & 0x3F), small_int_bits]) # fmt: off user_files = { # bad architecture (mpy_arch needed for sub-version) # CIRCUITPY-CHANGE - '/mod0.mpy': bytes([ord('C'), 6, 0xfc | mpy_arch, 31]), + '/mod0.mpy': bytes([ord('C'), 6, 0xfc | (mpy_arch & 3), small_int_bits]), # test loading of viper and asm '/mod1.mpy': valid_header + ( diff --git a/tests/micropython/import_mpy_native_gc.py b/tests/micropython/import_mpy_native_gc.py index 0d5b79f0a8dd4..5159cc9e9f9cf 100644 --- a/tests/micropython/import_mpy_native_gc.py +++ b/tests/micropython/import_mpy_native_gc.py @@ -57,9 +57,13 @@ def open(self, path, mode): # CIRCUITPY-CHANGE: 'C' instead of 'M' mpy marker. features0_file_contents = { # -march=x64 - 0x806: b'C\x06\n\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x8a\x02\xe93\x00\x00\x00\xf3\x0f\x1e\xfaSH\x8b\x1d\x7f\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6\xf3\x0f\x1e\xfaATUSH\x8b\x1dI\x00\x00\x00H\x8bG\x08L\x8bc(H\x8bx\x08A\xff\xd4H\x8d5#\x00\x00\x00H\x89\xc5H\x8b\x051\x00\x00\x00\x0f\xb7x\x02\xffShH\x89\xefA\xff\xd4H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00 \x11$\r&\xa5 \x01"\xff', + 0x806: b"C\x06\x0b\x1f\x03\x002build/test_x64.native.mpy\x00\x08add1\x00\x0cunused\x00\x91B\xe9I\x00\x00\x00H\x8b\x05\xf4\x00\x00\x00H\x8b\x00\xc3H\x8b\x05\xf9\x00\x00\x00\xbe\x02\x00\x00\x00\x8b8H\x8b\x05\xdb\x00\x00\x00H\x8b@ \xff\xe0H\x8b\x05\xce\x00\x00\x00S\xbe\x02\x00\x00\x00H\x8bX \xffP\x18\xbe\x02\x00\x00\x00H\x8dx\x01H\x89\xd8[\xff\xe0AVAUATUSH\x8b\x1d\xa3\x00\x00\x00H\x8bG\x08L\x8bk(H\x8bx\x08A\xff\xd5L\x8b5\x95\x00\x00\x00L\x8bchH\x8d5r\x00\x00\x00H\x89\xc5H\x8b\x05\x88\x00\x00\x00A\x0f\xb7~\x04\xc7\x00@\xe2\x01\x00A\xff\xd4H\x8d5C\x00\x00\x00\xbfV\x00\x00\x00A\xff\xd4A\x0f\xb7~\x02H\x8d5\x1f\x00\x00\x00A\xff\xd4H\x89\xefA\xff\xd5H\x8b\x03[]A\\A]A^\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00+\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00P\x04\x11@\rB\tD\xaf4\x016\xad8\x01:\xaf<\x01>\xff", # -march=armv6m - 0x1006: b"C\x06\x12\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x88\x02\x18\xe0\x00\x00\x10\xb5\tK\tJ{D\x9cX\x02!\xe3h\x98G\x03\x00\x01 \x00+\x02\xd0XC\x01;\xfa\xe7\x02!#i\x98G\x10\xbd\xc0Fj\x00\x00\x00\x00\x00\x00\x00\xf8\xb5\nN\nK~D\xf4XChgiXh\xb8G\x05\x00\x07K\x08I\xf3XyDX\x88ck\x98G(\x00\xb8G h\xf8\xbd\xc0F:\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x1e\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00 \x11<\r>\xa58\x01:\xff", + 0x1006: b"C\x06\x13\x1f\x03\x008build/test_armv6m.native.mpy\x00\x08add1\x00\x0cunused\x00\x8eb0\xe0\x00\x00\x00\x00\x00\x00\x02K\x03J{D\x9bX\x18hpG\xd0\x00\x00\x00\x00\x00\x00\x00\x10\xb5\x05K\x05I\x06J{D\x9aX[X\x10h\x02!\x1bi\x98G\x10\xbd\xb8\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x10\xb5\x06K\x06J{D\x9bX\x02!\x1ci\xdbh\x98G\x02!\x010\xa0G\x10\xbd\xc0F\x96\x00\x00\x00\x00\x00\x00\x00\xf7\xb5\x12O\x12K\x7fD\xfdX\x12Lki|D\x00\x93ChXh\x00\x9b\x98G\x0fK\x01\x90\x0fJ\xfbXnk\x1a`\x0eK!\x00\xffX\xb8\x88\xb0G!\x00V \x081\xb0G!\x00x\x88\x101\xb0G\x01\x98\x00\x9b\x98G(h\xfe\xbd\xc0Fr\x00\x00\x00\x00\x00\x00\x00R\x00\x00\x00\x08\x00\x00\x00@\xe2\x01\x00\x04\x00\x00\x00\x00\x00\x00\x00\t\x00\x00\x00\x00\x00\x00\x00\x1d\x00\x00\x00\x00\x00\x00\x00A\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00P\x04\x11p\rr\tt\xafd\x01f\xadh\x01j\xafl\x01n\xff", + # -march=xtensawin + 0x2806: b"C\x06+\x1f\x03\x00>build/test_xtensawin.native.mpy\x00\x08add1\x00\x0cunused\x00\x8a\x12\x06\x16\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x006A\x00\x81\xf9\xff(\x08\x1d\xf0\x00\x006A\x00\x91\xfb\xff\x81\xf5\xff\xa8\t\x88H\x0c+\xe0\x08\x00-\n\x1d\xf0\x00\x006A\x00\x81\xf0\xff\xad\x02xH\x888\x0c+\xe0\x08\x00\x0c+\x1b\xaa\xe0\x07\x00-\n\x1d\xf06A\x00a\xe9\xff\x88\x122&\x05\xa2(\x01\xe0\x03\x00q\xe6\xff\x81\xea\xff\x92\xa7\x89\xa0\x99\x11H\xd6]\n\xb1\xe3\xff\xa2\x17\x02\x99\x08\xe0\x04\x00\xb1\xe2\xff\\j\xe0\x04\x00\xb1\xe1\xff\xa2\x17\x01\xe0\x04\x00\xad\x05\xe0\x03\x00(\x06\x1d\xf0p\x18\x04\x00\x00\x00\x00@\x00\x00\x00\x00\x00\x00\x00(\x00\x00\x00\x00\x00\x00\x00\x1c\x00\x00\x00\x11\x02\r\x04\x07\x06\x03\t\x0c\xaf\x01\x01\x03\xad\x05\x01\x07\xaf\t\x01\x0b\xff", + # -march=rv32imc + 0x2C06: b'C\x06/\x1f\x03\x00:build/test_rv32imc.native.mpy\x00\x08add1\x00\x0cunused\x00\x8fb\x97\x0f\x00\x00g\x80\x0f\x05\x97\x07\x00\x00\x83\xa7\x07\x0e\x88C\x82\x80\x97\x07\x00\x00\x83\xa7G\r\x17\x07\x00\x00\x03\'\xc7\r\x9cK\x08C\x89E\x82\x87A\x11\x97\x07\x00\x00\x83\xa7\xa7\x0b"\xc4\x80K\xdcG\x06\xc6\x89E\x82\x97\xa2\x87"D\xb2@\x89E\x05\x05A\x01\x82\x87\\A\x01\x11"\xcc\x17\x04\x00\x00\x03$$\tN\xc6\xc8C\x83)D\x01\x06\xce&\xcaJ\xc8R\xc4\x82\x99\x17\n\x00\x00\x03*\xca\x07\x03)D\x03\xaa\x84\xf9g\x03UJ\x00\x93\x87\x07$\x17\x07\x00\x00\x03\'\x07\x07\x1c\xc3\x97\x05\x00\x00\x93\x85\xe5\x03\x02\x99\x97\x05\x00\x00\x93\x85\xc5\x03\x13\x05`\x05\x02\x99\x03U*\x00\x97\x05\x00\x00\x93\x85%\x03\x02\x99&\x85\x82\x99\x08@\xf2@bD\xd2DBI\xb2I"J\x05a\x82\x80\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00,\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00P\x04\x11t\rv\xafx\xadz\t|\xafh\x01j\xadl\x01n\xafp\x01r\xff', } # Populate armv7m-derived archs based on armv6m. @@ -67,7 +71,7 @@ def open(self, path, mode): features0_file_contents[arch] = features0_file_contents[0x1006] # Check that a .mpy exists for the target (ignore sub-version in lookup). -sys_implementation_mpy = sys.implementation._mpy & ~(3 << 8) +sys_implementation_mpy = (sys.implementation._mpy & ~(3 << 8)) & 0xFFFF if sys_implementation_mpy not in features0_file_contents: print("SKIP") raise SystemExit diff --git a/tests/micropython/kbd_intr.py b/tests/micropython/kbd_intr.py index 81977aaa52f6f..e1ed7185ef0a2 100644 --- a/tests/micropython/kbd_intr.py +++ b/tests/micropython/kbd_intr.py @@ -1,10 +1,10 @@ # test the micropython.kbd_intr() function -import micropython - try: + import micropython + micropython.kbd_intr -except AttributeError: +except (ImportError, AttributeError): print("SKIP") raise SystemExit diff --git a/tests/micropython/meminfo.py b/tests/micropython/meminfo.py index 9df341fbb8334..f4dd8fdb6042c 100644 --- a/tests/micropython/meminfo.py +++ b/tests/micropython/meminfo.py @@ -1,12 +1,14 @@ # tests meminfo functions in micropython module -import micropython +try: + import micropython -# these functions are not always available -if not hasattr(micropython, "mem_info"): + micropython.mem_info +except (ImportError, AttributeError): print("SKIP") -else: - micropython.mem_info() - micropython.mem_info(1) - micropython.qstr_info() - micropython.qstr_info(1) + raise SystemExit + +micropython.mem_info() +micropython.mem_info(1) +micropython.qstr_info() +micropython.qstr_info(1) diff --git a/tests/micropython/memstats.py b/tests/micropython/memstats.py index dee3a4ce2f225..0e2e7b1c0b326 100644 --- a/tests/micropython/memstats.py +++ b/tests/micropython/memstats.py @@ -1,17 +1,19 @@ # tests meminfo functions in micropython module -import micropython +try: + import micropython -# these functions are not always available -if not hasattr(micropython, "mem_total"): + micropython.mem_total +except (ImportError, AttributeError): print("SKIP") -else: - t = micropython.mem_total() - c = micropython.mem_current() - p = micropython.mem_peak() + raise SystemExit - l = list(range(10000)) +t = micropython.mem_total() +c = micropython.mem_current() +p = micropython.mem_peak() - print(micropython.mem_total() > t) - print(micropython.mem_current() > c) - print(micropython.mem_peak() > p) +l = list(range(10000)) + +print(micropython.mem_total() > t) +print(micropython.mem_current() > c) +print(micropython.mem_peak() > p) diff --git a/tests/micropython/opt_level.py b/tests/micropython/opt_level.py index dd5493a7a3cb8..789197d8825bf 100644 --- a/tests/micropython/opt_level.py +++ b/tests/micropython/opt_level.py @@ -1,4 +1,10 @@ -import micropython as micropython +# test micropython.opt_level() + +try: + import micropython +except ImportError: + print("SKIP") + raise SystemExit # check we can get and set the level micropython.opt_level(0) diff --git a/tests/micropython/opt_level_lineno.py b/tests/micropython/opt_level_lineno.py index 40650b6819221..4ca76625de49a 100644 --- a/tests/micropython/opt_level_lineno.py +++ b/tests/micropython/opt_level_lineno.py @@ -1,7 +1,24 @@ -import micropython as micropython +# test micropython.opt_level() and line numbers + +try: + import micropython +except ImportError: + print("SKIP") + raise SystemExit # check that level 3 doesn't store line numbers # the expected output is that any line is printed as "line 1" micropython.opt_level(3) -# CIRCUITPY-CHANGE: use traceback.print_exception() instead of sys.print_exception() -exec("try:\n xyz\nexcept NameError as er:\n import traceback\n traceback.print_exception(er)") + +# force bytecode emitter, because native emitter doesn't store line numbers +exec(""" +@micropython.bytecode +def f(): + try: + xyz + except NameError as er: + # CIRCUITPY-CHANGE: use traceback.print_exception() instead of sys.print_exception() + import traceback + traceback.print_exception(er) +f() +""") diff --git a/tests/micropython/opt_level_lineno.py.exp b/tests/micropython/opt_level_lineno.py.exp index 469b90ba7938a..b50f0628c81fb 100644 --- a/tests/micropython/opt_level_lineno.py.exp +++ b/tests/micropython/opt_level_lineno.py.exp @@ -1,3 +1,3 @@ Traceback (most recent call last): - File "", line 1, in + File "", line 1, in f NameError: name 'xyz' isn't defined diff --git a/tests/micropython/ringio_big.py b/tests/micropython/ringio_big.py new file mode 100644 index 0000000000000..ddbbae12a630c --- /dev/null +++ b/tests/micropython/ringio_big.py @@ -0,0 +1,29 @@ +# Check that micropython.RingIO works correctly. + +try: + import micropython + + micropython.RingIO +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit + +try: + # The maximum possible size + micropython.RingIO(bytearray(65535)) + micropython.RingIO(65534) + + try: + # Buffer may not be too big + micropython.RingIO(bytearray(65536)) + except ValueError as ex: + print(type(ex)) + + try: + # Size may not be too big + micropython.RingIO(65535) + except ValueError as ex: + print(type(ex)) +except MemoryError: + print("SKIP") + raise SystemExit diff --git a/tests/micropython/ringio_big.py.exp b/tests/micropython/ringio_big.py.exp new file mode 100644 index 0000000000000..72af34b383872 --- /dev/null +++ b/tests/micropython/ringio_big.py.exp @@ -0,0 +1,2 @@ + + diff --git a/tests/micropython/schedule.py b/tests/micropython/schedule.py index 6a91459ea3d54..f3dd32661267b 100644 --- a/tests/micropython/schedule.py +++ b/tests/micropython/schedule.py @@ -1,10 +1,10 @@ # test micropython.schedule() function -import micropython - try: + import micropython + micropython.schedule -except AttributeError: +except (ImportError, AttributeError): print("SKIP") raise SystemExit diff --git a/tests/micropython/stack_use.py b/tests/micropython/stack_use.py index 266885d9d1897..5d36fdca2fe01 100644 --- a/tests/micropython/stack_use.py +++ b/tests/micropython/stack_use.py @@ -1,7 +1,11 @@ # tests stack_use function in micropython module -import micropython -if not hasattr(micropython, "stack_use"): +try: + import micropython + + micropython.stack_use +except (ImportError, AttributeError): print("SKIP") -else: - print(type(micropython.stack_use())) # output varies + raise SystemExit + +print(type(micropython.stack_use())) # output varies diff --git a/tests/micropython/test_normalize_newlines.py b/tests/micropython/test_normalize_newlines.py new file mode 100644 index 0000000000000..f19aaa69a3f75 --- /dev/null +++ b/tests/micropython/test_normalize_newlines.py @@ -0,0 +1,14 @@ +# Test for normalize_newlines functionality +# This test verifies that test framework handles various newline combinations correctly + +# Note: This is more of an integration test since normalize_newlines is in the test framework +# The actual testing happens when this test is run through run-tests.py + +print("Testing newline handling") +print("Line 1\r\nLine 2") # Windows-style line ending - should be normalized +print("Line 3") # Normal line +print("Line 4") # Normal line +print("Line 5\nLine 6") # Unix-style line ending - already normalized + +# Test that literal \r in strings is preserved +print(repr("test\rstring")) # Should show 'test\rstring' not 'test\nstring' diff --git a/tests/micropython/test_normalize_newlines.py.exp b/tests/micropython/test_normalize_newlines.py.exp new file mode 100644 index 0000000000000..c4395468cf109 --- /dev/null +++ b/tests/micropython/test_normalize_newlines.py.exp @@ -0,0 +1,8 @@ +Testing newline handling +Line 1 +Line 2 +Line 3 +Line 4 +Line 5 +Line 6 +'test\rstring' diff --git a/tests/micropython/viper_large_jump.py b/tests/micropython/viper_large_jump.py new file mode 100644 index 0000000000000..1c5913dec1ea2 --- /dev/null +++ b/tests/micropython/viper_large_jump.py @@ -0,0 +1,20 @@ +COUNT = 600 + + +try: + code = """ +@micropython.viper +def f() -> int: + x = 0 + while x < 10: +""" + for i in range(COUNT): + code += " x += 1\n" + code += " return x" + exec(code) +except MemoryError: + print("SKIP-TOO-LARGE") + raise SystemExit + + +print(f()) diff --git a/tests/micropython/viper_large_jump.py.exp b/tests/micropython/viper_large_jump.py.exp new file mode 100644 index 0000000000000..e9f960cf4ac4e --- /dev/null +++ b/tests/micropython/viper_large_jump.py.exp @@ -0,0 +1 @@ +600 diff --git a/tests/micropython/viper_ptr16_load_boundary.py b/tests/micropython/viper_ptr16_load_boundary.py new file mode 100644 index 0000000000000..0d4c3105b685b --- /dev/null +++ b/tests/micropython/viper_ptr16_load_boundary.py @@ -0,0 +1,39 @@ +# Test boundary conditions for various architectures + +GET_TEMPLATE = """ +@micropython.viper +def get{off}(src: ptr16) -> uint: + return uint(src[{off}]) +print(hex(get{off}(buffer))) +""" + + +BIT_THRESHOLDS = (5, 8, 11, 12) +SIZE = 2 + + +@micropython.viper +def get_index(src: ptr16, i: int) -> int: + return src[i] + + +def data(start, len): + output = bytearray(len) + for idx in range(len): + output[idx] = (start + idx) & 0xFF + return output + + +buffer = bytearray((((1 << max(BIT_THRESHOLDS)) + 1) // 1024) * 1024) +val = 0 +for bit in BIT_THRESHOLDS: + print("---", bit) + pre, idx, post = ((1 << bit) - (2 * SIZE), (1 << bit) - (1 * SIZE), 1 << bit) + buffer[pre:post] = data(val, 3 * SIZE) + val = val + (3 * SIZE) + + pre, idx, post = pre // SIZE, idx // SIZE, post // SIZE + print(hex(get_index(buffer, pre)), hex(get_index(buffer, idx)), hex(get_index(buffer, post))) + exec(GET_TEMPLATE.format(off=pre)) + exec(GET_TEMPLATE.format(off=idx)) + exec(GET_TEMPLATE.format(off=post)) diff --git a/tests/micropython/viper_ptr16_load_boundary.py.exp b/tests/micropython/viper_ptr16_load_boundary.py.exp new file mode 100644 index 0000000000000..56f1d32290468 --- /dev/null +++ b/tests/micropython/viper_ptr16_load_boundary.py.exp @@ -0,0 +1,20 @@ +--- 5 +0x100 0x302 0x504 +0x100 +0x302 +0x504 +--- 8 +0x706 0x908 0xb0a +0x706 +0x908 +0xb0a +--- 11 +0xd0c 0xf0e 0x1110 +0xd0c +0xf0e +0x1110 +--- 12 +0x1312 0x1514 0x1716 +0x1312 +0x1514 +0x1716 diff --git a/tests/micropython/viper_ptr16_store_boundary.py b/tests/micropython/viper_ptr16_store_boundary.py new file mode 100644 index 0000000000000..7c774d4d1ca18 --- /dev/null +++ b/tests/micropython/viper_ptr16_store_boundary.py @@ -0,0 +1,63 @@ +# Test boundary conditions for various architectures + +SET_TEMPLATE = """ +@micropython.viper +def set{off}(dest: ptr16): + saved = dest + dest[{off}] = {val} + assert int(saved) == int(dest) +""" + +BIT_THRESHOLDS = (5, 8, 11, 12) +SIZE = 2 +MASK = (1 << (8 * SIZE)) - 1 + +next_int = 1 +test_buffer = bytearray(SIZE) + + +def next_value() -> uint: + global next_int + global test_buffer + for index in range(1, SIZE): + test_buffer[index - 1] = test_buffer[index] + test_buffer[SIZE - 1] = next_int + next_int += 1 + output = 0 + for byte in test_buffer: + output = (output << 8) | byte + return output & MASK + + +def get_index(src, i): + return src[i * SIZE] + (src[(i * SIZE) + 1] << 8) + + +@micropython.viper +def set_index(dest: ptr16, i: int, val: uint): + saved = dest + dest[i] = val + assert int(saved) == int(dest) + + +try: + buffer = bytearray((((1 << max(BIT_THRESHOLDS)) // 1024) + 1) * 1024) + + for bit in BIT_THRESHOLDS: + offset = (1 << bit) - (2 * SIZE) + for index in range(0, 3 * SIZE, SIZE): + exec(SET_TEMPLATE.format(off=(offset + index) // SIZE, val=next_value())) +except MemoryError: + print("SKIP-TOO-LARGE") + raise SystemExit + + +for bit in BIT_THRESHOLDS: + print("---", bit) + offset = (1 << bit) - (2 * SIZE) + for index in range(0, 3 * SIZE, SIZE): + globals()["set{}".format((offset + index) // SIZE)](buffer) + print(hex(get_index(buffer, (offset + index) // SIZE))) + for index in range(0, 3 * SIZE, SIZE): + set_index(buffer, (offset + index) // SIZE, next_value()) + print(hex(get_index(buffer, (offset + index) // SIZE))) diff --git a/tests/micropython/viper_ptr16_store_boundary.py.exp b/tests/micropython/viper_ptr16_store_boundary.py.exp new file mode 100644 index 0000000000000..007a50b3edafb --- /dev/null +++ b/tests/micropython/viper_ptr16_store_boundary.py.exp @@ -0,0 +1,28 @@ +--- 5 +0x1 +0x102 +0x203 +0xc0d +0xd0e +0xe0f +--- 8 +0x304 +0x405 +0x506 +0xf10 +0x1011 +0x1112 +--- 11 +0x607 +0x708 +0x809 +0x1213 +0x1314 +0x1415 +--- 12 +0x90a +0xa0b +0xb0c +0x1516 +0x1617 +0x1718 diff --git a/tests/micropython/viper_ptr32_load_boundary.py b/tests/micropython/viper_ptr32_load_boundary.py new file mode 100644 index 0000000000000..971d1113c49bb --- /dev/null +++ b/tests/micropython/viper_ptr32_load_boundary.py @@ -0,0 +1,39 @@ +# Test boundary conditions for various architectures + +GET_TEMPLATE = """ +@micropython.viper +def get{off}(src: ptr32) -> uint: + return uint(src[{off}]) +print(hex(get{off}(buffer))) +""" + + +BIT_THRESHOLDS = (5, 8, 11, 12) +SIZE = 4 + + +@micropython.viper +def get_index(src: ptr32, i: int) -> int: + return src[i] + + +def data(start, len): + output = bytearray(len) + for idx in range(len): + output[idx] = (start + idx) & 0xFF + return output + + +buffer = bytearray((((1 << max(BIT_THRESHOLDS)) + 1) // 1024) * 1024) +val = 0 +for bit in BIT_THRESHOLDS: + print("---", bit) + pre, idx, post = (((1 << bit) - (2 * SIZE)), ((1 << bit) - (1 * SIZE)), (1 << bit)) + buffer[pre:post] = data(val, 3 * SIZE) + val = val + (3 * SIZE) + + pre, idx, post = pre // SIZE, idx // SIZE, post // SIZE + print(hex(get_index(buffer, pre)), hex(get_index(buffer, idx)), hex(get_index(buffer, post))) + exec(GET_TEMPLATE.format(off=pre)) + exec(GET_TEMPLATE.format(off=idx)) + exec(GET_TEMPLATE.format(off=post)) diff --git a/tests/micropython/viper_ptr32_load_boundary.py.exp b/tests/micropython/viper_ptr32_load_boundary.py.exp new file mode 100644 index 0000000000000..1e22a8b361333 --- /dev/null +++ b/tests/micropython/viper_ptr32_load_boundary.py.exp @@ -0,0 +1,20 @@ +--- 5 +0x3020100 0x7060504 0xb0a0908 +0x3020100 +0x7060504 +0xb0a0908 +--- 8 +0xf0e0d0c 0x13121110 0x17161514 +0xf0e0d0c +0x13121110 +0x17161514 +--- 11 +0x1b1a1918 0x1f1e1d1c 0x23222120 +0x1b1a1918 +0x1f1e1d1c +0x23222120 +--- 12 +0x27262524 0x2b2a2928 0x2f2e2d2c +0x27262524 +0x2b2a2928 +0x2f2e2d2c diff --git a/tests/micropython/viper_ptr32_store_boundary.py b/tests/micropython/viper_ptr32_store_boundary.py new file mode 100644 index 0000000000000..96ca74ad3ca70 --- /dev/null +++ b/tests/micropython/viper_ptr32_store_boundary.py @@ -0,0 +1,68 @@ +# Test boundary conditions for various architectures + +SET_TEMPLATE = """ +@micropython.viper +def set{off}(dest: ptr32): + saved = dest + dest[{off}] = {val} + assert int(saved) == int(dest) +""" + +BIT_THRESHOLDS = (5, 8, 11, 12) +SIZE = 4 +MASK = (1 << (8 * SIZE)) - 1 + +next_int = 1 +test_buffer = bytearray(SIZE) + + +def next_value() -> uint: + global next_int + global test_buffer + for index in range(1, SIZE): + test_buffer[index - 1] = test_buffer[index] + test_buffer[SIZE - 1] = next_int + next_int += 1 + output = 0 + for byte in test_buffer: + output = (output << 8) | byte + return output & MASK + + +def get_index(src, i): + return ( + src[i * SIZE] + + (src[(i * SIZE) + 1] << 8) + + (src[(i * SIZE) + 2] << 16) + + (src[(i * SIZE) + 3] << 24) + ) + + +@micropython.viper +def set_index(dest: ptr32, i: int, val: uint): + saved = dest + dest[i] = val + assert int(dest) == int(saved) + + +try: + buffer = bytearray((((1 << max(BIT_THRESHOLDS)) // 1024) + 1) * 1024) + + for bit in BIT_THRESHOLDS: + offset = (1 << bit) - (2 * SIZE) + for index in range(0, 3 * SIZE, SIZE): + exec(SET_TEMPLATE.format(off=(offset + index) // SIZE, val=next_value())) +except MemoryError: + print("SKIP-TOO-LARGE") + raise SystemExit + + +for bit in BIT_THRESHOLDS: + print("---", bit) + offset = (1 << bit) - (2 * SIZE) + for index in range(0, 3 * SIZE, SIZE): + globals()["set{}".format((offset + index) // SIZE)](buffer) + print(hex(get_index(buffer, (offset + index) // SIZE))) + for index in range(0, 3 * SIZE, SIZE): + set_index(buffer, (offset + index) // SIZE, next_value()) + print(hex(get_index(buffer, (offset + index) // SIZE))) diff --git a/tests/micropython/viper_ptr32_store_boundary.py.exp b/tests/micropython/viper_ptr32_store_boundary.py.exp new file mode 100644 index 0000000000000..7a9a51624743e --- /dev/null +++ b/tests/micropython/viper_ptr32_store_boundary.py.exp @@ -0,0 +1,28 @@ +--- 5 +0x1 +0x102 +0x10203 +0xa0b0c0d +0xb0c0d0e +0xc0d0e0f +--- 8 +0x1020304 +0x2030405 +0x3040506 +0xd0e0f10 +0xe0f1011 +0xf101112 +--- 11 +0x4050607 +0x5060708 +0x6070809 +0x10111213 +0x11121314 +0x12131415 +--- 12 +0x708090a +0x8090a0b +0x90a0b0c +0x13141516 +0x14151617 +0x15161718 diff --git a/tests/micropython/viper_ptr8_load_boundary.py b/tests/micropython/viper_ptr8_load_boundary.py new file mode 100644 index 0000000000000..57e06da5709ab --- /dev/null +++ b/tests/micropython/viper_ptr8_load_boundary.py @@ -0,0 +1,38 @@ +# Test boundary conditions for various architectures + +GET_TEMPLATE = """ +@micropython.viper +def get{off}(src: ptr8) -> uint: + return uint(src[{off}]) +print(hex(get{off}(buffer))) +""" + + +BIT_THRESHOLDS = (5, 8, 11, 12) +SIZE = 1 + + +@micropython.viper +def get_index(src: ptr8, i: int) -> int: + return src[i] + + +def data(start, len): + output = bytearray(len) + for idx in range(len): + output[idx] = (start + idx) & 0xFF + return output + + +buffer = bytearray((((1 << max(BIT_THRESHOLDS)) + 1) // 1024) * 1024) +val = 0 +for bit in BIT_THRESHOLDS: + print("---", bit) + pre, idx, post = (((1 << bit) - (2 * SIZE)), ((1 << bit) - (1 * SIZE)), (1 << bit)) + buffer[pre:post] = data(val, 3 * SIZE) + val = val + (3 * SIZE) + + print(hex(get_index(buffer, pre)), hex(get_index(buffer, idx)), hex(get_index(buffer, post))) + exec(GET_TEMPLATE.format(off=pre)) + exec(GET_TEMPLATE.format(off=idx)) + exec(GET_TEMPLATE.format(off=post)) diff --git a/tests/micropython/viper_ptr8_load_boundary.py.exp b/tests/micropython/viper_ptr8_load_boundary.py.exp new file mode 100644 index 0000000000000..a0e423686b8fa --- /dev/null +++ b/tests/micropython/viper_ptr8_load_boundary.py.exp @@ -0,0 +1,20 @@ +--- 5 +0x0 0x1 0x2 +0x0 +0x1 +0x2 +--- 8 +0x3 0x4 0x5 +0x3 +0x4 +0x5 +--- 11 +0x6 0x7 0x8 +0x6 +0x7 +0x8 +--- 12 +0x9 0xa 0xb +0x9 +0xa +0xb diff --git a/tests/micropython/viper_ptr8_store_boundary.py b/tests/micropython/viper_ptr8_store_boundary.py new file mode 100644 index 0000000000000..68b76fd598b88 --- /dev/null +++ b/tests/micropython/viper_ptr8_store_boundary.py @@ -0,0 +1,63 @@ +# Test boundary conditions for various architectures + +SET_TEMPLATE = """ +@micropython.viper +def set{off}(dest: ptr8): + saved = dest + dest[{off}] = {val} + assert int(saved) == int(dest) +""" + +BIT_THRESHOLDS = (5, 8, 11, 12) +SIZE = 1 +MASK = (1 << (8 * SIZE)) - 1 + +next_int = 1 +test_buffer = bytearray(SIZE) + + +def next_value() -> uint: + global next_int + global test_buffer + for index in range(1, SIZE): + test_buffer[index - 1] = test_buffer[index] + test_buffer[SIZE - 1] = next_int + next_int += 1 + output = 0 + for byte in test_buffer: + output = (output << 8) | byte + return output & MASK + + +def get_index(src: ptr8, i: int): + return src[i] + + +@micropython.viper +def set_index(dest: ptr8, i: int, val: uint): + saved = dest + dest[i] = val + assert int(dest) == int(saved) + + +try: + buffer = bytearray((((1 << max(BIT_THRESHOLDS)) // 1024) + 1) * 1024) + + for bit in BIT_THRESHOLDS: + offset = (1 << bit) - (2 * SIZE) + for index in range(0, 3 * SIZE, SIZE): + exec(SET_TEMPLATE.format(off=(offset + index) // SIZE, val=next_value())) +except MemoryError: + print("SKIP-TOO-LARGE") + raise SystemExit + + +for bit in BIT_THRESHOLDS: + print("---", bit) + offset = (1 << bit) - (2 * SIZE) + for index in range(0, 3 * SIZE, SIZE): + globals()["set{}".format((offset + index) // SIZE)](buffer) + print(hex(get_index(buffer, (offset + index) // SIZE))) + for index in range(0, 3 * SIZE, SIZE): + set_index(buffer, (offset + index) // SIZE, next_value()) + print(hex(get_index(buffer, (offset + index) // SIZE))) diff --git a/tests/micropython/viper_ptr8_store_boundary.py.exp b/tests/micropython/viper_ptr8_store_boundary.py.exp new file mode 100644 index 0000000000000..621295d81a896 --- /dev/null +++ b/tests/micropython/viper_ptr8_store_boundary.py.exp @@ -0,0 +1,28 @@ +--- 5 +0x1 +0x2 +0x3 +0xd +0xe +0xf +--- 8 +0x4 +0x5 +0x6 +0x10 +0x11 +0x12 +--- 11 +0x7 +0x8 +0x9 +0x13 +0x14 +0x15 +--- 12 +0xa +0xb +0xc +0x16 +0x17 +0x18 diff --git a/tests/misc/non_compliant.py b/tests/misc/non_compliant.py index 31c9fa17c3bed..8608f2322f8e8 100644 --- a/tests/misc/non_compliant.py +++ b/tests/misc/non_compliant.py @@ -39,7 +39,7 @@ except NotImplementedError: print("NotImplementedError") -# uPy raises TypeError, should be ValueError +# MicroPython raises TypeError, should be ValueError try: "%c" % b"\x01\x02" except (TypeError, ValueError): @@ -100,10 +100,10 @@ print("NotImplementedError") # CIRCUITPY-CHANGE: We do check these. -# struct pack with too many args, not checked by uPy +# struct pack with too many args, not checked by MicroPython # print(struct.pack("bb", 1, 2, 3)) -# struct pack with too few args, not checked by uPy +# struct pack with too few args, not checked by MicroPython # print(struct.pack("bb", 1)) # array slice assignment with unsupported RHS diff --git a/tests/misc/non_compliant_lexer.py b/tests/misc/non_compliant_lexer.py index e1c21f3d713c4..04c605953e705 100644 --- a/tests/misc/non_compliant_lexer.py +++ b/tests/misc/non_compliant_lexer.py @@ -11,7 +11,7 @@ def test(code): print("NotImplementedError") -# uPy requires spaces between literal numbers and keywords, CPy doesn't +# MPy requires spaces between literal numbers and keywords, CPy doesn't try: eval("1and 0") except SyntaxError: diff --git a/tests/misc/print_exception.py.native.exp b/tests/misc/print_exception.py.native.exp new file mode 100644 index 0000000000000..59e856ae3c44a --- /dev/null +++ b/tests/misc/print_exception.py.native.exp @@ -0,0 +1,18 @@ +caught +Exception: msg + +caught +Exception: fail + +finally +caught +Exception: fail + +reraise +Exception: fail + +caught +Exception: fail + +AttributeError: 'function' object has no attribute 'X' + diff --git a/tests/misc/rge_sm.py b/tests/misc/rge_sm.py index 5e071687c495d..33aa4edb7427a 100644 --- a/tests/misc/rge_sm.py +++ b/tests/misc/rge_sm.py @@ -39,14 +39,6 @@ def solve(self, finishtime): if not self.iterate(): break - def solveNSteps(self, nSteps): - for i in range(nSteps): - if not self.iterate(): - break - - def series(self): - return zip(*self.Trajectory) - # 1-loop RGES for the main parameters of the SM # couplings are: g1, g2, g3 of U(1), SU(2), SU(3); yt (top Yukawa), lambda (Higgs quartic) @@ -55,70 +47,36 @@ def series(self): lambda *a: 41.0 / 96.0 / math.pi**2 * a[1] ** 3, # g1 lambda *a: -19.0 / 96.0 / math.pi**2 * a[2] ** 3, # g2 lambda *a: -42.0 / 96.0 / math.pi**2 * a[3] ** 3, # g3 - lambda *a: 1.0 - / 16.0 - / math.pi**2 - * ( - 9.0 / 2.0 * a[4] ** 3 - - 8.0 * a[3] ** 2 * a[4] - - 9.0 / 4.0 * a[2] ** 2 * a[4] - - 17.0 / 12.0 * a[1] ** 2 * a[4] + lambda *a: ( + 1.0 + / 16.0 + / math.pi**2 + * ( + 9.0 / 2.0 * a[4] ** 3 + - 8.0 * a[3] ** 2 * a[4] + - 9.0 / 4.0 * a[2] ** 2 * a[4] + - 17.0 / 12.0 * a[1] ** 2 * a[4] + ) ), # yt - lambda *a: 1.0 - / 16.0 - / math.pi**2 - * ( - 24.0 * a[5] ** 2 - + 12.0 * a[4] ** 2 * a[5] - - 9.0 * a[5] * (a[2] ** 2 + 1.0 / 3.0 * a[1] ** 2) - - 6.0 * a[4] ** 4 - + 9.0 / 8.0 * a[2] ** 4 - + 3.0 / 8.0 * a[1] ** 4 - + 3.0 / 4.0 * a[2] ** 2 * a[1] ** 2 + lambda *a: ( + 1.0 + / 16.0 + / math.pi**2 + * ( + 24.0 * a[5] ** 2 + + 12.0 * a[4] ** 2 * a[5] + - 9.0 * a[5] * (a[2] ** 2 + 1.0 / 3.0 * a[1] ** 2) + - 6.0 * a[4] ** 4 + + 9.0 / 8.0 * a[2] ** 4 + + 3.0 / 8.0 * a[1] ** 4 + + 3.0 / 4.0 * a[2] ** 2 * a[1] ** 2 + ) ), # lambda ) -def drange(start, stop, step): - r = start - while r < stop: - yield r - r += step - - -def phaseDiagram(system, trajStart, trajPlot, h=0.1, tend=1.0, range=1.0): - tstart = 0.0 - for i in drange(0, range, 0.1 * range): - for j in drange(0, range, 0.1 * range): - rk = RungeKutta(system, trajStart(i, j), tstart, h) - rk.solve(tend) - # draw the line - for tr in rk.Trajectory: - x, y = trajPlot(tr) - print(x, y) - print() - # draw the arrow - continue - l = (len(rk.Trajectory) - 1) / 3 - if l > 0 and 2 * l < len(rk.Trajectory): - p1 = rk.Trajectory[l] - p2 = rk.Trajectory[2 * l] - x1, y1 = trajPlot(p1) - x2, y2 = trajPlot(p2) - dx = -0.5 * (y2 - y1) # orthogonal to line - dy = 0.5 * (x2 - x1) # orthogonal to line - # l = math.sqrt(dx*dx + dy*dy) - # if abs(l) > 1e-3: - # l = 0.1 / l - # dx *= l - # dy *= l - print(x1 + dx, y1 + dy) - print(x2, y2) - print(x1 - dx, y1 - dy) - print() - - def singleTraj(system, trajStart, h=0.02, tend=1.0): + is_REPR_C = float("1.0000001") == float("1.0") tstart = 0.0 # compute the trajectory @@ -130,10 +88,15 @@ def singleTraj(system, trajStart, h=0.02, tend=1.0): for i in range(len(rk.Trajectory)): tr = rk.Trajectory[i] - print(" ".join(["{:.4f}".format(t) for t in tr])) - + tr_str = " ".join(["{:.4f}".format(t) for t in tr]) + if is_REPR_C: + # allow two small deviations for REPR_C + if tr_str == "1.0000 0.3559 0.6485 1.1944 0.9271 0.1083": + tr_str = "1.0000 0.3559 0.6485 1.1944 0.9272 0.1083" + if tr_str == "16.0000 0.3894 0.5793 0.7017 0.5686 -0.0168": + tr_str = "16.0000 0.3894 0.5793 0.7017 0.5686 -0.0167" + print(tr_str) -# phaseDiagram(sysSM, (lambda i, j: [0.354, 0.654, 1.278, 0.8 + 0.2 * i, 0.1 + 0.1 * j]), (lambda a: (a[4], a[5])), h=0.1, tend=math.log(10**17)) # initial conditions at M_Z singleTraj(sysSM, [0.354, 0.654, 1.278, 0.983, 0.131], h=0.5, tend=math.log(10**17)) # true values diff --git a/tests/misc/sys_exc_info.py b/tests/misc/sys_exc_info.py index d7e8a2d943b5e..c076dd572b07b 100644 --- a/tests/misc/sys_exc_info.py +++ b/tests/misc/sys_exc_info.py @@ -8,13 +8,14 @@ def f(): - print(sys.exc_info()[0:2]) + e = sys.exc_info() + print(e[0], e[1]) try: raise ValueError("value", 123) except: - print(sys.exc_info()[0:2]) + print(sys.exc_info()[0], sys.exc_info()[1]) f() # Outside except block, sys.exc_info() should be back to None's diff --git a/tests/misc/sys_settrace_cov.py b/tests/misc/sys_settrace_cov.py new file mode 100644 index 0000000000000..579c8a4a25e50 --- /dev/null +++ b/tests/misc/sys_settrace_cov.py @@ -0,0 +1,23 @@ +import sys + +try: + sys.settrace +except AttributeError: + print("SKIP") + raise SystemExit + + +def trace_tick_handler(frame, event, arg): + print("FRAME", frame) + print("LASTI", frame.f_lasti) + return None + + +def f(): + x = 3 + return x + + +sys.settrace(trace_tick_handler) +f() +sys.settrace(None) diff --git a/tests/misc/sys_settrace_cov.py.exp b/tests/misc/sys_settrace_cov.py.exp new file mode 100644 index 0000000000000..423d78ec42b89 --- /dev/null +++ b/tests/misc/sys_settrace_cov.py.exp @@ -0,0 +1,2 @@ +FRAME +LASTI \\d\+ diff --git a/tests/multi_extmod/machine_i2c_target_irq.py b/tests/multi_extmod/machine_i2c_target_irq.py new file mode 100644 index 0000000000000..eafd9dfdca838 --- /dev/null +++ b/tests/multi_extmod/machine_i2c_target_irq.py @@ -0,0 +1,137 @@ +# Test I2CTarget IRQs and clock stretching. +# +# Requires two instances with their SCL and SDA lines connected together. +# Any combination of the below supported boards can be used. +# +# Notes: +# - pull-up resistors may be needed +# - alif use 1.8V signalling + +import sys +import time +from machine import I2C, I2CTarget + +if not hasattr(I2CTarget, "IRQ_ADDR_MATCH_READ"): + print("SKIP") + raise SystemExit + +ADDR = 67 +clock_stretch_us = 200 + +# Configure pins based on the target. +if sys.platform == "alif": + i2c_args = (1,) # pins P3_7/P3_6 + i2c_kwargs = {} +elif sys.platform == "mimxrt": + i2c_args = (0,) # pins 19/18 on Teensy 4.x + i2c_kwargs = {} + clock_stretch_us = 50 # mimxrt cannot delay too long in the IRQ handler +elif sys.platform == "rp2": + i2c_args = (0,) + i2c_kwargs = {"scl": 9, "sda": 8} +elif sys.platform == "pyboard": + i2c_args = ("Y",) + i2c_kwargs = {} +elif sys.platform == "samd": + i2c_args = () # pins SCL/SDA + i2c_kwargs = {} +elif "zephyr-rpi_pico" in sys.implementation._machine: + i2c_args = ("i2c1",) # on gpio7/gpio6 + i2c_kwargs = {} +else: + print("Please add support for this test on this platform.") + raise SystemExit + + +def simple_irq(i2c_target): + flags = i2c_target.irq().flags() + if flags & I2CTarget.IRQ_ADDR_MATCH_READ: + print("IRQ_ADDR_MATCH_READ") + if flags & I2CTarget.IRQ_ADDR_MATCH_WRITE: + print("IRQ_ADDR_MATCH_WRITE") + + # Force clock stretching. + time.sleep_us(clock_stretch_us) + + +class I2CTargetMemory: + def __init__(self, i2c_target, mem): + self.buf1 = bytearray(1) + self.mem = mem + self.memaddr = 0 + self.state = 0 + i2c_target.irq( + self.irq, + I2CTarget.IRQ_ADDR_MATCH_WRITE | I2CTarget.IRQ_READ_REQ | I2CTarget.IRQ_WRITE_REQ, + hard=True, + ) + + def irq(self, i2c_target): + # Force clock stretching. + time.sleep_us(clock_stretch_us) + + flags = i2c_target.irq().flags() + if flags & I2CTarget.IRQ_ADDR_MATCH_WRITE: + self.state = 0 + if flags & I2CTarget.IRQ_READ_REQ: + self.buf1[0] = self.mem[self.memaddr] + self.memaddr += 1 + i2c_target.write(self.buf1) + if flags & I2CTarget.IRQ_WRITE_REQ: + i2c_target.readinto(self.buf1) + if self.state == 0: + self.state = 1 + self.memaddr = self.buf1[0] + else: + self.mem[self.memaddr] = self.buf1[0] + self.memaddr += 1 + self.memaddr %= len(self.mem) + + # Force clock stretching. + time.sleep_us(clock_stretch_us) + + +# I2C controller +def instance0(): + i2c = I2C(*i2c_args, **i2c_kwargs) + multitest.next() + for iteration in range(2): + print("controller iteration", iteration) + multitest.wait("target stage 1") + i2c.writeto_mem(ADDR, 2, "0123") + multitest.broadcast("controller stage 2") + multitest.wait("target stage 3") + print(i2c.readfrom_mem(ADDR, 2, 4)) + multitest.broadcast("controller stage 4") + print("done") + + +# I2C target +def instance1(): + multitest.next() + + for iteration in range(2): + print("target iteration", iteration) + buf = bytearray(b"--------") + if iteration == 0: + # Use built-in memory capability of I2CTarget. + i2c_target = I2CTarget(*i2c_args, **i2c_kwargs, addr=ADDR, mem=buf) + i2c_target.irq( + simple_irq, + I2CTarget.IRQ_ADDR_MATCH_READ | I2CTarget.IRQ_ADDR_MATCH_WRITE, + hard=True, + ) + else: + # Implement a memory device by hand. + i2c_target = I2CTarget(*i2c_args, **i2c_kwargs, addr=ADDR) + I2CTargetMemory(i2c_target, buf) + + multitest.broadcast("target stage 1") + multitest.wait("controller stage 2") + print(buf) + multitest.broadcast("target stage 3") + multitest.wait("controller stage 4") + + i2c_target.deinit() + + print("done") diff --git a/tests/multi_extmod/machine_i2c_target_irq.py.exp b/tests/multi_extmod/machine_i2c_target_irq.py.exp new file mode 100644 index 0000000000000..a17c8f43858b3 --- /dev/null +++ b/tests/multi_extmod/machine_i2c_target_irq.py.exp @@ -0,0 +1,15 @@ +--- instance0 --- +controller iteration 0 +b'0123' +controller iteration 1 +b'0123' +done +--- instance1 --- +target iteration 0 +IRQ_ADDR_MATCH_WRITE +bytearray(b'--0123--') +IRQ_ADDR_MATCH_WRITE +IRQ_ADDR_MATCH_READ +target iteration 1 +bytearray(b'--0123--') +done diff --git a/tests/multi_extmod/machine_i2c_target_memory.py b/tests/multi_extmod/machine_i2c_target_memory.py new file mode 100644 index 0000000000000..6b3f0d03eb7fe --- /dev/null +++ b/tests/multi_extmod/machine_i2c_target_memory.py @@ -0,0 +1,79 @@ +# Test basic use of I2CTarget and a memory buffer. +# +# Requires two instances with their SCL and SDA lines connected together. +# Any combination of the below supported boards can be used. +# +# Notes: +# - pull-up resistors may be needed +# - alif use 1.8V signalling + +import sys +from machine import I2C, I2CTarget + +ADDR = 67 + +# Configure pins based on the target. +if sys.platform == "alif": + i2c_args = (1,) # pins P3_7/P3_6 + i2c_kwargs = {} +elif sys.platform == "esp32": + i2c_args = (1,) # on pins 9/8 + i2c_kwargs = {} +elif sys.platform == "mimxrt": + i2c_args = (0,) # pins 19/18 on Teensy 4.x + i2c_kwargs = {} +elif sys.platform == "rp2": + i2c_args = (0,) + i2c_kwargs = {"scl": 9, "sda": 8} +elif sys.platform == "pyboard": + i2c_args = ("Y",) + i2c_kwargs = {} +elif sys.platform == "samd": + i2c_args = () # pins SCL/SDA + i2c_kwargs = {} +elif "zephyr-rpi_pico" in sys.implementation._machine: + i2c_args = ("i2c1",) # on gpio7/gpio6 + i2c_kwargs = {} +else: + print("Please add support for this test on this platform.") + raise SystemExit + + +def simple_irq(i2c_target): + flags = i2c_target.irq().flags() + if flags & I2CTarget.IRQ_END_READ: + print("IRQ_END_READ", i2c_target.memaddr) + if flags & I2CTarget.IRQ_END_WRITE: + print("IRQ_END_WRITE", i2c_target.memaddr) + + +# I2C controller +def instance0(): + i2c = I2C(*i2c_args, **i2c_kwargs) + multitest.next() + for iteration in range(2): + print("controller iteration", iteration) + multitest.wait("target stage 1") + i2c.writeto_mem(ADDR, 2 + iteration, "0123") + multitest.broadcast("controller stage 2") + multitest.wait("target stage 3") + print(i2c.readfrom_mem(ADDR, 2 + iteration, 4)) + multitest.broadcast("controller stage 4") + print("done") + + +# I2C target +def instance1(): + buf = bytearray(b"--------") + i2c_target = I2CTarget(*i2c_args, **i2c_kwargs, addr=ADDR, mem=buf) + i2c_target.irq(simple_irq) + multitest.next() + for iteration in range(2): + print("target iteration", iteration) + multitest.broadcast("target stage 1") + multitest.wait("controller stage 2") + print(buf) + multitest.broadcast("target stage 3") + multitest.wait("controller stage 4") + i2c_target.deinit() + print("done") diff --git a/tests/multi_extmod/machine_i2c_target_memory.py.exp b/tests/multi_extmod/machine_i2c_target_memory.py.exp new file mode 100644 index 0000000000000..71386cfe769d9 --- /dev/null +++ b/tests/multi_extmod/machine_i2c_target_memory.py.exp @@ -0,0 +1,16 @@ +--- instance0 --- +controller iteration 0 +b'0123' +controller iteration 1 +b'0123' +done +--- instance1 --- +target iteration 0 +IRQ_END_WRITE 2 +bytearray(b'--0123--') +IRQ_END_READ 2 +target iteration 1 +IRQ_END_WRITE 3 +bytearray(b'--00123-') +IRQ_END_READ 3 +done diff --git a/tests/perf_bench/bm_fft.py b/tests/perf_bench/bm_fft.py index 9a2d03d11b97c..e35c1216c139f 100644 --- a/tests/perf_bench/bm_fft.py +++ b/tests/perf_bench/bm_fft.py @@ -15,7 +15,7 @@ def reverse(x, bits): # Initialization n = len(vector) - levels = int(math.log(n) / math.log(2)) + levels = int(round(math.log(n) / math.log(2))) coef = (2 if inverse else -2) * cmath.pi / n exptable = [cmath.rect(1, i * coef) for i in range(n // 2)] vector = [vector[reverse(i, levels)] for i in range(n)] # Copy with bit-reversed permutation diff --git a/tests/perf_bench/bm_pidigits.py b/tests/perf_bench/bm_pidigits.py index bdaa73cec7e9f..c935f103c5b78 100644 --- a/tests/perf_bench/bm_pidigits.py +++ b/tests/perf_bench/bm_pidigits.py @@ -5,6 +5,12 @@ # This benchmark stresses big integer arithmetic. # Adapted from code on: http://benchmarksgame.alioth.debian.org/ +try: + int("0x10000000000000000", 16) +except: + print("SKIP") # No support for >64-bit integers + raise SystemExit + def compose(a, b): aq, ar, as_, at = a diff --git a/tests/perf_bench/core_import_mpy_multi.py b/tests/perf_bench/core_import_mpy_multi.py index 8affa157fa0c5..67deec0508801 100644 --- a/tests/perf_bench/core_import_mpy_multi.py +++ b/tests/perf_bench/core_import_mpy_multi.py @@ -6,7 +6,7 @@ print("SKIP") raise SystemExit -# This is the test.py file that is compiled to test.mpy below. +# This is the test.py file that is compiled to test.mpy below. mpy-cross must be invoked with `-msmall-int-bits=30`. """ class A: def __init__(self, arg): @@ -23,7 +23,7 @@ def f(): x = ("const tuple", None, False, True, 1, 2, 3) result = 123 """ -file_data = b'M\x06\x00\x1f\x14\x03\x0etest.py\x00\x0f\x02A\x00\x02f\x00\x0cresult\x00/-5#\x82I\x81{\x81w\x82/\x81\x05\x81\x17Iom\x82\x13\x06arg\x00\x05\x1cthis will be a string object\x00\x06\x1bthis will be a bytes object\x00\n\x07\x05\x0bconst tuple\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\x81\\\x10\n\x01\x89\x07d`T2\x00\x10\x024\x02\x16\x022\x01\x16\x03"\x80{\x16\x04Qc\x02\x81d\x00\x08\x02(DD\x11\x05\x16\x06\x10\x02\x16\x072\x00\x16\x082\x01\x16\t2\x02\x16\nQc\x03`\x1a\x08\x08\x12\x13@\xb1\xb0\x18\x13Qc@\t\x08\t\x12` Qc@\t\x08\n\x12``Qc\x82@ \x0e\x03\x80\x08+)##\x12\x0b\x12\x0c\x12\r\x12\x0e*\x04Y\x12\x0f\x12\x10\x12\x11*\x03Y#\x00\xc0#\x01\xc0#\x02\xc0Qc' +file_data = b'M\x06\x00\x1e\x14\x03\x0etest.py\x00\x0f\x02A\x00\x02f\x00\x0cresult\x00/-5#\x82I\x81{\x81w\x82/\x81\x05\x81\x17Iom\x82\x13\x06arg\x00\x05\x1cthis will be a string object\x00\x06\x1bthis will be a bytes object\x00\n\x07\x05\x0bconst tuple\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\x81\\\x10\n\x01\x89\x07d`T2\x00\x10\x024\x02\x16\x022\x01\x16\x03"\x80{\x16\x04Qc\x02\x81d\x00\x08\x02(DD\x11\x05\x16\x06\x10\x02\x16\x072\x00\x16\x082\x01\x16\t2\x02\x16\nQc\x03`\x1a\x08\x08\x12\x13@\xb1\xb0\x18\x13Qc@\t\x08\t\x12` Qc@\t\x08\n\x12``Qc\x82@ \x0e\x03\x80\x08+)##\x12\x0b\x12\x0c\x12\r\x12\x0e*\x04Y\x12\x0f\x12\x10\x12\x11*\x03Y#\x00\xc0#\x01\xc0#\x02\xc0Qc' class File(io.IOBase): diff --git a/tests/perf_bench/core_import_mpy_single.py b/tests/perf_bench/core_import_mpy_single.py index 4d9aa67bf2f0e..f472bb6476278 100644 --- a/tests/perf_bench/core_import_mpy_single.py +++ b/tests/perf_bench/core_import_mpy_single.py @@ -8,7 +8,7 @@ print("SKIP") raise SystemExit -# This is the test.py file that is compiled to test.mpy below. +# This is the test.py file that is compiled to test.mpy below. mpy-cross must be invoked with `-msmall-int-bits=30`. # Many known and unknown names/strings are included to test the linking process. """ class A0: @@ -78,7 +78,7 @@ def f1(): x = ("const tuple 9", None, False, True, 1, 2, 3) result = 123 """ -file_data = b"M\x06\x00\x1f\x81=\x1e\x0etest.py\x00\x0f\x04A0\x00\x04A1\x00\x04f0\x00\x04f1\x00\x0cresult\x00/-5\x04a0\x00\x04a1\x00\x04a2\x00\x04a3\x00\x13\x15\x17\x19\x1b\x1d\x1f!#%')+1379;=?ACEGIKMOQSUWY[]_acegikmoqsuwy{}\x7f\x81\x01\x81\x03\x81\x05\x81\x07\x81\t\x81\x0b\x81\r\x81\x0f\x81\x11\x81\x13\x81\x15\x81\x17\x81\x19\x81\x1b\x81\x1d\x81\x1f\x81!\x81#\x81%\x81'\x81)\x81+\x81-\x81/\x811\x813\x815\x817\x819\x81;\x81=\x81?\x81A\x81C\x81E\x81G\x81I\x81K\x81M\x81O\x81Q\x81S\x81U\x81W\x81Y\x81[\x81]\x81_\x81a\x81c\x81e\x81g\x81i\x81k\x81m\x81o\x81q\x81s\x81u\x81w\x81y\x81{\x81}\x81\x7f\x82\x01\x82\x03\x82\x05\x82\x07\x82\t\x82\x0b\x82\r\x82\x0f\x82\x11\x82\x13\x82\x15\x82\x17\x82\x19\x82\x1b\x82\x1d\x82\x1f\x82!\x82#\x82%\x82'\x82)\x82+\x82-\x82/\x821\x823\x825\x827\x829\x82;\x82=\x82?\x82A\x82E\x82G\x82I\x82K\nname0\x00\nname1\x00\nname2\x00\nname3\x00\nname4\x00\nname5\x00\nname6\x00\nname7\x00\nname8\x00\nname9\x00$quite_a_long_name0\x00$quite_a_long_name1\x00$quite_a_long_name2\x00$quite_a_long_name3\x00$quite_a_long_name4\x00$quite_a_long_name5\x00$quite_a_long_name6\x00$quite_a_long_name7\x00$quite_a_long_name8\x00$quite_a_long_name9\x00&quite_a_long_name10\x00&quite_a_long_name11\x00\x05\x1ethis will be a string object 0\x00\x05\x1ethis will be a string object 1\x00\x05\x1ethis will be a string object 2\x00\x05\x1ethis will be a string object 3\x00\x05\x1ethis will be a string object 4\x00\x05\x1ethis will be a string object 5\x00\x05\x1ethis will be a string object 6\x00\x05\x1ethis will be a string object 7\x00\x05\x1ethis will be a string object 8\x00\x05\x1ethis will be a string object 9\x00\x06\x1dthis will be a bytes object 0\x00\x06\x1dthis will be a bytes object 1\x00\x06\x1dthis will be a bytes object 2\x00\x06\x1dthis will be a bytes object 3\x00\x06\x1dthis will be a bytes object 4\x00\x06\x1dthis will be a bytes object 5\x00\x06\x1dthis will be a bytes object 6\x00\x06\x1dthis will be a bytes object 7\x00\x06\x1dthis will be a bytes object 8\x00\x06\x1dthis will be a bytes object 9\x00\n\x07\x05\rconst tuple 0\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 1\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 2\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 3\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 4\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 5\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 6\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 7\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 8\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 9\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\x82d\x10\x12\x01i@i@\x84\x18\x84\x1fT2\x00\x10\x024\x02\x16\x02T2\x01\x10\x034\x02\x16\x032\x02\x16\x042\x03\x16\x05\"\x80{\x16\x06Qc\x04\x82\x0c\x00\n\x02($$$\x11\x07\x16\x08\x10\x02\x16\t2\x00\x16\n2\x01\x16\x0b2\x02\x16\x0c2\x03\x16\rQc\x04@\t\x08\n\x81\x0b Qc@\t\x08\x0b\x81\x0b@Qc@\t\x08\x0c\x81\x0b`QcH\t\n\r\x81\x0b` Qc\x82\x14\x00\x0c\x03h`$$$\x11\x07\x16\x08\x10\x03\x16\t2\x00\x16\n2\x01\x16\x0b2\x02\x16\x0c2\x03\x16\rQc\x04H\t\n\n\x81\x0b``QcH\t\n\x0b\x81\x0b\x80\x07QcH\t\n\x0c\x81\x0b\x80\x08QcH\t\n\r\x81\x0b\x80\tQc\xa08P:\x04\x80\x0b13///---997799<\x1f%\x1f\"\x1f%)\x1f\"//\x12\x0e\x12\x0f\x12\x10\x12\x11\x12\x12\x12\x13\x12\x14*\x07Y\x12\x15\x12\x16\x12\x17\x12\x18\x12\x19\x12\x1a\x12\x08\x12\x07*\x08Y\x12\x1b\x12\x1c\x12\t\x12\x1d\x12\x1e\x12\x1f*\x06Y\x12 \x12!\x12\"\x12#\x12$\x12%*\x06Y\x12&\x12'\x12(\x12)\x12*\x12+*\x06Y\x12,\x12-\x12.\x12/\x120*\x05Y\x121\x122\x123\x124\x125*\x05Y\x126\x127\x128\x129\x12:*\x05Y\x12;\x12<\x12=\x12>\x12?\x12@\x12A\x12B\x12C\x12D\x12E*\x0bY\x12F\x12G\x12H\x12I\x12J\x12K\x12L\x12M\x12N\x12O\x12P*\x0bY\x12Q\x12R\x12S\x12T\x12U\x12V\x12W\x12X\x12Y\x12Z*\nY\x12[\x12\\\x12]\x12^\x12_\x12`\x12a\x12b\x12c\x12d*\nY\x12e\x12f\x12g\x12h\x12i\x12j\x12k\x12l\x12m\x12n\x12o*\x0bY\x12p\x12q\x12r\x12s\x12t\x12u\x12v\x12w\x12x\x12y\x12z*\x0bY\x12{\x12|\x12}\x12~\x12\x7f\x12\x81\x00\x12\x81\x01\x12\x81\x02\x12\x81\x03\x12\x81\x04*\nY\x12\x81\x05\x12\x81\x06\x12\x81\x07\x12\x81\x08\x12\x81\t\x12\x81\n\x12\x81\x0b\x12\x81\x0c\x12\x81\r\x12\x81\x0e\x12\x81\x0f*\x0bY\x12\x81\x10\x12\x81\x11\x12\x81\x12\x12\x81\x13\x12\x81\x14\x12\x81\x15\x12\x81\x16\x12\x81\x17\x12\x81\x18\x12\x81\x19*\nY\x12\x81\x1a\x12\x81\x1b\x12\x81\x1c\x12\x81\x1d\x12\x81\x1e\x12\x81\x1f\x12\x81 \x12\x81!\x12\x81\"\x12\x81#\x12\x81$*\x0bY\x12\x81%\x12\x81&*\x02Y\x12\x81'\x12\x81(\x12\x81)\x12\x81*\x12\x81+\x12\x81,\x12\x81-\x12\x81.\x12\x81/\x12\x810*\nY\x12\x811\x12\x812\x12\x813\x12\x814*\x04Y\x12\x815\x12\x816\x12\x817\x12\x818*\x04Y\x12\x819\x12\x81:\x12\x81;\x12\x81<*\x04YQc\x87p\x08@\x05\x80###############################\x00\xc0#\x01\xc0#\x02\xc0#\x03\xc0#\x04\xc0#\x05\xc0#\x06\xc0#\x07\xc0#\x08\xc0#\t\xc0#\n\xc0#\x0b\xc0#\x0c\xc0#\r\xc0#\x0e\xc0#\x0f\xc0#\x10\xc0#\x11\xc0#\x12\xc0#\x13\xc0#\x14\xc0#\x15\xc0#\x16\xc0#\x17\xc0#\x18\xc0#\x19\xc0#\x1a\xc0#\x1b\xc0#\x1c\xc0#\x1d\xc0Qc" +file_data = b"M\x06\x00\x1e\x81=\x1e\x0etest.py\x00\x0f\x04A0\x00\x04A1\x00\x04f0\x00\x04f1\x00\x0cresult\x00/-5\x04a0\x00\x04a1\x00\x04a2\x00\x04a3\x00\x13\x15\x17\x19\x1b\x1d\x1f!#%')+1379;=?ACEGIKMOQSUWY[]_acegikmoqsuwy{}\x7f\x81\x01\x81\x03\x81\x05\x81\x07\x81\t\x81\x0b\x81\r\x81\x0f\x81\x11\x81\x13\x81\x15\x81\x17\x81\x19\x81\x1b\x81\x1d\x81\x1f\x81!\x81#\x81%\x81'\x81)\x81+\x81-\x81/\x811\x813\x815\x817\x819\x81;\x81=\x81?\x81A\x81C\x81E\x81G\x81I\x81K\x81M\x81O\x81Q\x81S\x81U\x81W\x81Y\x81[\x81]\x81_\x81a\x81c\x81e\x81g\x81i\x81k\x81m\x81o\x81q\x81s\x81u\x81w\x81y\x81{\x81}\x81\x7f\x82\x01\x82\x03\x82\x05\x82\x07\x82\t\x82\x0b\x82\r\x82\x0f\x82\x11\x82\x13\x82\x15\x82\x17\x82\x19\x82\x1b\x82\x1d\x82\x1f\x82!\x82#\x82%\x82'\x82)\x82+\x82-\x82/\x821\x823\x825\x827\x829\x82;\x82=\x82?\x82A\x82E\x82G\x82I\x82K\nname0\x00\nname1\x00\nname2\x00\nname3\x00\nname4\x00\nname5\x00\nname6\x00\nname7\x00\nname8\x00\nname9\x00$quite_a_long_name0\x00$quite_a_long_name1\x00$quite_a_long_name2\x00$quite_a_long_name3\x00$quite_a_long_name4\x00$quite_a_long_name5\x00$quite_a_long_name6\x00$quite_a_long_name7\x00$quite_a_long_name8\x00$quite_a_long_name9\x00&quite_a_long_name10\x00&quite_a_long_name11\x00\x05\x1ethis will be a string object 0\x00\x05\x1ethis will be a string object 1\x00\x05\x1ethis will be a string object 2\x00\x05\x1ethis will be a string object 3\x00\x05\x1ethis will be a string object 4\x00\x05\x1ethis will be a string object 5\x00\x05\x1ethis will be a string object 6\x00\x05\x1ethis will be a string object 7\x00\x05\x1ethis will be a string object 8\x00\x05\x1ethis will be a string object 9\x00\x06\x1dthis will be a bytes object 0\x00\x06\x1dthis will be a bytes object 1\x00\x06\x1dthis will be a bytes object 2\x00\x06\x1dthis will be a bytes object 3\x00\x06\x1dthis will be a bytes object 4\x00\x06\x1dthis will be a bytes object 5\x00\x06\x1dthis will be a bytes object 6\x00\x06\x1dthis will be a bytes object 7\x00\x06\x1dthis will be a bytes object 8\x00\x06\x1dthis will be a bytes object 9\x00\n\x07\x05\rconst tuple 0\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 1\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 2\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 3\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 4\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 5\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 6\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 7\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 8\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 9\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\x82d\x10\x12\x01i@i@\x84\x18\x84\x1fT2\x00\x10\x024\x02\x16\x02T2\x01\x10\x034\x02\x16\x032\x02\x16\x042\x03\x16\x05\"\x80{\x16\x06Qc\x04\x82\x0c\x00\n\x02($$$\x11\x07\x16\x08\x10\x02\x16\t2\x00\x16\n2\x01\x16\x0b2\x02\x16\x0c2\x03\x16\rQc\x04@\t\x08\n\x81\x0b Qc@\t\x08\x0b\x81\x0b@Qc@\t\x08\x0c\x81\x0b`QcH\t\n\r\x81\x0b` Qc\x82\x14\x00\x0c\x03h`$$$\x11\x07\x16\x08\x10\x03\x16\t2\x00\x16\n2\x01\x16\x0b2\x02\x16\x0c2\x03\x16\rQc\x04H\t\n\n\x81\x0b``QcH\t\n\x0b\x81\x0b\x80\x07QcH\t\n\x0c\x81\x0b\x80\x08QcH\t\n\r\x81\x0b\x80\tQc\xa08P:\x04\x80\x0b13///---997799<\x1f%\x1f\"\x1f%)\x1f\"//\x12\x0e\x12\x0f\x12\x10\x12\x11\x12\x12\x12\x13\x12\x14*\x07Y\x12\x15\x12\x16\x12\x17\x12\x18\x12\x19\x12\x1a\x12\x08\x12\x07*\x08Y\x12\x1b\x12\x1c\x12\t\x12\x1d\x12\x1e\x12\x1f*\x06Y\x12 \x12!\x12\"\x12#\x12$\x12%*\x06Y\x12&\x12'\x12(\x12)\x12*\x12+*\x06Y\x12,\x12-\x12.\x12/\x120*\x05Y\x121\x122\x123\x124\x125*\x05Y\x126\x127\x128\x129\x12:*\x05Y\x12;\x12<\x12=\x12>\x12?\x12@\x12A\x12B\x12C\x12D\x12E*\x0bY\x12F\x12G\x12H\x12I\x12J\x12K\x12L\x12M\x12N\x12O\x12P*\x0bY\x12Q\x12R\x12S\x12T\x12U\x12V\x12W\x12X\x12Y\x12Z*\nY\x12[\x12\\\x12]\x12^\x12_\x12`\x12a\x12b\x12c\x12d*\nY\x12e\x12f\x12g\x12h\x12i\x12j\x12k\x12l\x12m\x12n\x12o*\x0bY\x12p\x12q\x12r\x12s\x12t\x12u\x12v\x12w\x12x\x12y\x12z*\x0bY\x12{\x12|\x12}\x12~\x12\x7f\x12\x81\x00\x12\x81\x01\x12\x81\x02\x12\x81\x03\x12\x81\x04*\nY\x12\x81\x05\x12\x81\x06\x12\x81\x07\x12\x81\x08\x12\x81\t\x12\x81\n\x12\x81\x0b\x12\x81\x0c\x12\x81\r\x12\x81\x0e\x12\x81\x0f*\x0bY\x12\x81\x10\x12\x81\x11\x12\x81\x12\x12\x81\x13\x12\x81\x14\x12\x81\x15\x12\x81\x16\x12\x81\x17\x12\x81\x18\x12\x81\x19*\nY\x12\x81\x1a\x12\x81\x1b\x12\x81\x1c\x12\x81\x1d\x12\x81\x1e\x12\x81\x1f\x12\x81 \x12\x81!\x12\x81\"\x12\x81#\x12\x81$*\x0bY\x12\x81%\x12\x81&*\x02Y\x12\x81'\x12\x81(\x12\x81)\x12\x81*\x12\x81+\x12\x81,\x12\x81-\x12\x81.\x12\x81/\x12\x810*\nY\x12\x811\x12\x812\x12\x813\x12\x814*\x04Y\x12\x815\x12\x816\x12\x817\x12\x818*\x04Y\x12\x819\x12\x81:\x12\x81;\x12\x81<*\x04YQc\x87p\x08@\x05\x80###############################\x00\xc0#\x01\xc0#\x02\xc0#\x03\xc0#\x04\xc0#\x05\xc0#\x06\xc0#\x07\xc0#\x08\xc0#\t\xc0#\n\xc0#\x0b\xc0#\x0c\xc0#\r\xc0#\x0e\xc0#\x0f\xc0#\x10\xc0#\x11\xc0#\x12\xc0#\x13\xc0#\x14\xc0#\x15\xc0#\x16\xc0#\x17\xc0#\x18\xc0#\x19\xc0#\x1a\xc0#\x1b\xc0#\x1c\xc0#\x1d\xc0Qc" class File(io.IOBase): diff --git a/tests/run-internalbench.py b/tests/run-internalbench.py index c9f783e474c9c..99c6304afe9d6 100755 --- a/tests/run-internalbench.py +++ b/tests/run-internalbench.py @@ -8,6 +8,10 @@ from glob import glob from collections import defaultdict +run_tests_module = __import__("run-tests") +sys.path.append(run_tests_module.base_path("../tools")) +import pyboard + if os.name == "nt": MICROPYTHON = os.getenv( "MICROPY_MICROPYTHON", "../ports/windows/build-standard/micropython.exe" @@ -15,13 +19,39 @@ else: MICROPYTHON = os.getenv("MICROPY_MICROPYTHON", "../ports/unix/build-standard/micropython") +injected_bench_code = b""" +import time + +class bench_class: + ITERS = 20000000 + + @staticmethod + def run(test): + t = time.ticks_us() + test(bench_class.ITERS) + t = time.ticks_diff(time.ticks_us(), t) + s, us = divmod(t, 1_000_000) + print("{}.{:06}".format(s, us)) + +import sys +sys.modules['bench'] = bench_class +""" + -def run_tests(pyb, test_dict): +def execbench(pyb, filename, iters): + with open(filename, "rb") as f: + pyfile = f.read() + code = (injected_bench_code + pyfile).replace(b"20000000", str(iters).encode("utf-8")) + return pyb.exec(code).replace(b"\r\n", b"\n") + + +def run_tests(pyb, test_dict, iters): test_count = 0 testcase_count = 0 for base_test, tests in sorted(test_dict.items()): print(base_test + ":") + baseline = None for test_file in tests: # run MicroPython if pyb is None: @@ -36,20 +66,25 @@ def run_tests(pyb, test_dict): # run on pyboard pyb.enter_raw_repl() try: - output_mupy = pyb.execfile(test_file).replace(b"\r\n", b"\n") + output_mupy = execbench(pyb, test_file[0], iters) except pyboard.PyboardError: output_mupy = b"CRASH" - output_mupy = float(output_mupy.strip()) + try: + output_mupy = float(output_mupy.strip()) + except ValueError: + output_mupy = -1 test_file[1] = output_mupy testcase_count += 1 - test_count += 1 - baseline = None - for t in tests: if baseline is None: - baseline = t[1] - print(" %.3fs (%+06.2f%%) %s" % (t[1], (t[1] * 100 / baseline) - 100, t[0])) + baseline = test_file[1] + print( + " %.3fs (%+06.2f%%) %s" + % (test_file[1], (test_file[1] * 100 / baseline) - 100, test_file[0]) + ) + + test_count += 1 print("{} tests performed ({} individual testcases)".format(test_count, testcase_count)) @@ -58,27 +93,47 @@ def run_tests(pyb, test_dict): def main(): - cmd_parser = argparse.ArgumentParser(description="Run tests for MicroPython.") - cmd_parser.add_argument("--pyboard", action="store_true", help="run the tests on the pyboard") + cmd_parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description=f"""Run and manage tests for MicroPython. + +{run_tests_module.test_instance_description} +{run_tests_module.test_directory_description} +""", + epilog=run_tests_module.test_instance_epilog, + ) + cmd_parser.add_argument( + "-t", "--test-instance", default="unix", help="the MicroPython instance to test" + ) + cmd_parser.add_argument( + "-b", "--baudrate", default=115200, help="the baud rate of the serial device" + ) + cmd_parser.add_argument("-u", "--user", default="micro", help="the telnet login username") + cmd_parser.add_argument("-p", "--password", default="python", help="the telnet login password") + cmd_parser.add_argument( + "-d", "--test-dirs", nargs="*", help="input test directories (if no files given)" + ) + cmd_parser.add_argument( + "-I", + "--iters", + type=int, + default=200_000, + help="number of test iterations, only for remote instances (default 200,000)", + ) cmd_parser.add_argument("files", nargs="*", help="input test files") args = cmd_parser.parse_args() # Note pyboard support is copied over from run-tests.py, not tests, and likely needs revamping - if args.pyboard: - import pyboard - - pyb = pyboard.Pyboard("/dev/ttyACM0") - pyb.enter_raw_repl() - else: - pyb = None + pyb = run_tests_module.get_test_instance( + args.test_instance, args.baudrate, args.user, args.password + ) if len(args.files) == 0: - if pyb is None: - # run PC tests - test_dirs = ("internal_bench",) + if args.test_dirs: + test_dirs = tuple(args.test_dirs) else: - # run pyboard tests - test_dirs = ("basics", "float", "pyb") + test_dirs = ("internal_bench",) + tests = sorted( test_file for test_files in (glob("{}/*.py".format(dir)) for dir in test_dirs) @@ -95,7 +150,7 @@ def main(): continue test_dict[m.group(1)].append([t, None]) - if not run_tests(pyb, test_dict): + if not run_tests(pyb, test_dict, args.iters): sys.exit(1) diff --git a/tests/run-multitests.py b/tests/run-multitests.py index 387eec7018bb3..e5458ffe0d00c 100755 --- a/tests/run-multitests.py +++ b/tests/run-multitests.py @@ -15,6 +15,8 @@ import subprocess import tempfile +run_tests_module = __import__("run-tests") + test_dir = os.path.abspath(os.path.dirname(__file__)) if os.path.abspath(sys.path[0]) == test_dir: @@ -130,6 +132,11 @@ def get_host_ip(_ip_cache=[]): return _ip_cache[0] +def decode(output): + # Convenience function to convert raw process or serial output to ASCII + return str(output, "ascii", "backslashreplace") + + class PyInstance: def __init__(self): pass @@ -188,7 +195,7 @@ def run_script(self, script): output = p.stdout except subprocess.CalledProcessError as er: err = er - return str(output.strip(), "ascii"), err + return decode(output.strip()), err def start_script(self, script): self.popen = subprocess.Popen( @@ -215,7 +222,7 @@ def readline(self): self.finished = self.popen.poll() is not None return None, None else: - return str(out.rstrip(), "ascii"), None + return decode(out.rstrip()), None def write(self, data): self.popen.stdin.write(data) @@ -227,21 +234,12 @@ def is_finished(self): def wait_finished(self): self.popen.wait() out = self.popen.stdout.read() - return str(out, "ascii"), "" + return decode(out), "" class PyInstancePyboard(PyInstance): - @staticmethod - def map_device_shortcut(device): - if device[0] == "a" and device[1:].isdigit(): - return "/dev/ttyACM" + device[1:] - elif device[0] == "u" and device[1:].isdigit(): - return "/dev/ttyUSB" + device[1:] - else: - return device - def __init__(self, device): - device = self.map_device_shortcut(device) + device = device self.device = device self.pyb = pyboard.Pyboard(device) self.pyb.enter_raw_repl() @@ -262,7 +260,7 @@ def run_script(self, script): output = self.pyb.exec_(script) except pyboard.PyboardError as er: err = er - return str(output.strip(), "ascii"), err + return decode(output.strip()), err def start_script(self, script): self.pyb.enter_raw_repl() @@ -281,13 +279,13 @@ def readline(self): if out.endswith(b"\x04"): self.finished = True out = out[:-1] - err = str(self.pyb.read_until(1, b"\x04"), "ascii") + err = decode(self.pyb.read_until(1, b"\x04")) err = err[:-1] if not out and not err: return None, None else: err = None - return str(out.rstrip(), "ascii"), err + return decode(out.rstrip()), err def write(self, data): self.pyb.serial.write(data) @@ -297,7 +295,7 @@ def is_finished(self): def wait_finished(self): out, err = self.pyb.follow(10, None) - return str(out, "ascii"), str(err, "ascii") + return decode(out), decode(err) def prepare_test_file_list(test_files): @@ -488,9 +486,7 @@ def print_diff(a, b): def run_tests(test_files, instances_truth, instances_test): - skipped_tests = [] - passed_tests = [] - failed_tests = [] + test_results = [] for test_file, num_instances in test_files: instances_str = "|".join(str(instances_test[i]) for i in range(num_instances)) @@ -526,13 +522,13 @@ def run_tests(test_files, instances_truth, instances_test): # Print result of test if skip: print("skip") - skipped_tests.append(test_file) + test_results.append((test_file, "skip", "")) elif output_test == output_truth: print("pass") - passed_tests.append(test_file) + test_results.append((test_file, "pass", "")) else: print("FAIL") - failed_tests.append(test_file) + test_results.append((test_file, "fail", "")) if not cmd_args.show_output: print("### TEST ###") print(output_test, end="") @@ -549,15 +545,7 @@ def run_tests(test_files, instances_truth, instances_test): if cmd_args.show_output: print() - print("{} tests performed".format(len(skipped_tests) + len(passed_tests) + len(failed_tests))) - print("{} tests passed".format(len(passed_tests))) - - if skipped_tests: - print("{} tests skipped: {}".format(len(skipped_tests), " ".join(skipped_tests))) - if failed_tests: - print("{} tests failed: {}".format(len(failed_tests), " ".join(failed_tests))) - - return not failed_tests + return test_results def main(): @@ -565,16 +553,24 @@ def main(): cmd_parser = argparse.ArgumentParser( description="Run network tests for MicroPython", + epilog=( + run_tests_module.test_instance_epilog + + "Each instance arg can optionally have custom env provided, eg. ,ENV=VAR,ENV=VAR...\n" + ), formatter_class=argparse.RawTextHelpFormatter, ) cmd_parser.add_argument( "-s", "--show-output", action="store_true", help="show test output after running" ) cmd_parser.add_argument( - "-t", "--trace-output", action="store_true", help="trace test output while running" + "-c", "--trace-output", action="store_true", help="trace test output while running" ) cmd_parser.add_argument( - "-i", "--instance", action="append", default=[], help="instance(s) to run the tests on" + "-t", + "--test-instance", + action="append", + default=[], + help="instance(s) to run the tests on", ) cmd_parser.add_argument( "-p", @@ -583,13 +579,11 @@ def main(): default=1, help="repeat the test with this many permutations of the instance order", ) - cmd_parser.epilog = ( - "Supported instance types:\r\n" - " -i pyb: physical device (eg. pyboard) on provided repl port.\n" - " -i micropython unix micropython instance, path customised with MICROPY_MICROPYTHON env.\n" - " -i cpython desktop python3 instance, path customised with MICROPY_CPYTHON3 env.\n" - " -i exec: custom program run on provided path.\n" - "Each instance arg can optionally have custom env provided, eg. ,ENV=VAR,ENV=VAR...\n" + cmd_parser.add_argument( + "-r", + "--result-dir", + default=run_tests_module.base_path("results"), + help="directory for test results", ) cmd_parser.add_argument("files", nargs="+", help="input test files") cmd_args = cmd_parser.parse_args() @@ -603,33 +597,36 @@ def main(): instances_truth = [PyInstanceSubProcess([PYTHON_TRUTH]) for _ in range(max_instances)] instances_test = [] - for i in cmd_args.instance: + for i in cmd_args.test_instance: # Each instance arg is ,ENV=VAR,ENV=VAR... i = i.split(",") cmd = i[0] env = i[1:] if cmd.startswith("exec:"): instances_test.append(PyInstanceSubProcess([cmd[len("exec:") :]], env)) - elif cmd == "micropython": + elif cmd == "unix": instances_test.append(PyInstanceSubProcess([MICROPYTHON], env)) elif cmd == "cpython": instances_test.append(PyInstanceSubProcess([CPYTHON3], env)) - elif cmd.startswith("pyb:"): - instances_test.append(PyInstancePyboard(cmd[len("pyb:") :])) + elif cmd == "webassembly" or cmd.startswith("execpty:"): + print("unsupported instance string: {}".format(cmd), file=sys.stderr) + sys.exit(2) else: - print("unknown instance string: {}".format(cmd), file=sys.stderr) - sys.exit(1) + device = run_tests_module.convert_device_shortcut_to_real_device(cmd) + instances_test.append(PyInstancePyboard(device)) for _ in range(max_instances - len(instances_test)): instances_test.append(PyInstanceSubProcess([MICROPYTHON])) + os.makedirs(cmd_args.result_dir, exist_ok=True) all_pass = True try: for i, instances_test_permutation in enumerate(itertools.permutations(instances_test)): if i >= cmd_args.permutations: break - all_pass &= run_tests(test_files, instances_truth, instances_test_permutation) + test_results = run_tests(test_files, instances_truth, instances_test_permutation) + all_pass &= run_tests_module.create_test_report(cmd_args, test_results) finally: for i in instances_truth: diff --git a/tests/run-natmodtests.py b/tests/run-natmodtests.py index 340a7f004b061..6d2a975b82a0b 100755 --- a/tests/run-natmodtests.py +++ b/tests/run-natmodtests.py @@ -9,7 +9,7 @@ import sys import argparse -# CIRCUITPY-CHANGE: no pyboard +run_tests_module = __import__("run-tests") # Paths for host executables CPYTHON3 = os.getenv("MICROPY_CPYTHON3", "python3") @@ -72,6 +72,7 @@ def open(self, path, mode): # CIRCUITPY-CHANGE: no vfs, but still have os os.mount(__FS(), '/__remote') sys.path.insert(0, '/__remote') +{import_prelude} sys.modules['{}'] = __import__('__injected') """ @@ -108,7 +109,7 @@ def run_script(self, script): output = self.pyb.exec_(script) output = output.replace(b"\r\n", b"\n") return output, None - except pyboard.PyboardError as er: + except run_tests_module.pyboard.PyboardError as er: return b"", er @@ -131,7 +132,15 @@ def detect_architecture(target): return platform, arch, None -def run_tests(target_truth, target, args, stats, resolved_arch): +def run_tests(target_truth, target, args, resolved_arch): + global injected_import_hook_code + + prelude = "" + if args.begin: + prelude = args.begin.read() + injected_import_hook_code = injected_import_hook_code.replace("{import_prelude}", prelude) + + test_results = [] for test_file in args.files: # Find supported test test_file_basename = os.path.basename(test_file) @@ -154,9 +163,11 @@ def run_tests(target_truth, target, args, stats, resolved_arch): with open(NATMOD_EXAMPLE_DIR + test_mpy, "rb") as f: test_script += b"__buf=" + bytes(repr(f.read()), "ascii") + b"\n" except OSError: - print("---- {} - mpy file not compiled".format(test_file)) + test_results.append((test_file, "skip", "mpy file not compiled")) + print("skip {} - mpy file not compiled".format(test_file)) continue test_script += bytes(injected_import_hook_code.format(test_module), "ascii") + test_script += b"print('START TEST')\n" test_script += test_file_data # Run test under MicroPython @@ -164,8 +175,18 @@ def run_tests(target_truth, target, args, stats, resolved_arch): # Work out result of test extra = "" + result_out = result_out.removeprefix(b"START TEST\n") if error is None and result_out == b"SKIP\n": result = "SKIP" + elif ( + error is not None + and error.args[0] == "exception" + and error.args[1] == b"" + and b"MemoryError" in error.args[2] + ): + # Test had a MemoryError before anything (should be at least "START TEST") + # was printed, so the test is too big for the target. + result = "LRGE" elif error is not None: result = "FAIL" extra = " - " + str(error) @@ -186,40 +207,63 @@ def run_tests(target_truth, target, args, stats, resolved_arch): result = "pass" # Accumulate statistics - stats["total"] += 1 if result == "pass": - stats["pass"] += 1 + test_results.append((test_file, "pass", "")) elif result == "SKIP": - stats["skip"] += 1 + test_results.append((test_file, "skip", "")) + elif result == "LRGE": + test_results.append((test_file, "skip", "too large")) else: - stats["fail"] += 1 + test_results.append((test_file, "fail", "")) # Print result print("{:4} {}{}".format(result, test_file, extra)) + return test_results + def main(): cmd_parser = argparse.ArgumentParser( - description="Run dynamic-native-module tests under MicroPython" + description="Run dynamic-native-module tests under MicroPython", + epilog=run_tests_module.test_instance_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, ) cmd_parser.add_argument( - "-p", "--pyboard", action="store_true", help="run tests via pyboard.py" + "-t", "--test-instance", default="unix", help="the MicroPython instance to test" ) + cmd_parser.add_argument("--baudrate", default=115200, help="baud rate of the serial device") + cmd_parser.add_argument("--user", default="micro", help="telnet login username") + cmd_parser.add_argument("--password", default="python", help="telnet login password") cmd_parser.add_argument( - "-d", "--device", default="/dev/ttyACM0", help="the device for pyboard.py" + "-a", "--arch", choices=AVAILABLE_ARCHS, help="override native architecture of the target" ) cmd_parser.add_argument( - "-a", "--arch", choices=AVAILABLE_ARCHS, help="override native architecture of the target" + "-b", + "--begin", + type=argparse.FileType("rt"), + default=None, + help="prologue python file to execute before module import", + ) + cmd_parser.add_argument( + "-r", + "--result-dir", + default=run_tests_module.base_path("results"), + help="directory for test results", ) cmd_parser.add_argument("files", nargs="*", help="input test files") args = cmd_parser.parse_args() target_truth = TargetSubprocess([CPYTHON3]) - if args.pyboard: - target = TargetPyboard(pyboard.Pyboard(args.device)) - else: + target = run_tests_module.get_test_instance( + args.test_instance, args.baudrate, args.user, args.password + ) + if target is None: + # Use the unix port of MicroPython. target = TargetSubprocess([MICROPYTHON]) + else: + # Use a remote target. + target = TargetPyboard(target) if hasattr(args, "arch") and args.arch is not None: target_arch = args.arch @@ -235,20 +279,14 @@ def main(): print("platform={} ".format(target_platform), end="") print("arch={}".format(target_arch)) - stats = {"total": 0, "pass": 0, "fail": 0, "skip": 0} - run_tests(target_truth, target, args, stats, target_arch) + os.makedirs(args.result_dir, exist_ok=True) + test_results = run_tests(target_truth, target, args, target_arch) + res = run_tests_module.create_test_report(args, test_results) target.close() target_truth.close() - print("{} tests performed".format(stats["total"])) - print("{} tests passed".format(stats["pass"])) - if stats["fail"]: - print("{} tests failed".format(stats["fail"])) - if stats["skip"]: - print("{} tests skipped".format(stats["skip"])) - - if stats["fail"]: + if not res: sys.exit(1) diff --git a/tests/run-perfbench.py b/tests/run-perfbench.py index 81d873c45997d..039d11a36111e 100755 --- a/tests/run-perfbench.py +++ b/tests/run-perfbench.py @@ -10,10 +10,9 @@ import argparse from glob import glob -sys.path.append("../tools") -import pyboard +run_tests_module = __import__("run-tests") -prepare_script_for_target = __import__("run-tests").prepare_script_for_target +prepare_script_for_target = run_tests_module.prepare_script_for_target # Paths for host executables if os.name == "nt": @@ -45,12 +44,12 @@ def run_script_on_target(target, script): output = b"" err = None - if isinstance(target, pyboard.Pyboard): + if hasattr(target, "enter_raw_repl"): # Run via pyboard interface try: target.enter_raw_repl() output = target.exec_(script) - except pyboard.PyboardError as er: + except run_tests_module.pyboard.PyboardError as er: err = er else: # Run local executable @@ -90,9 +89,9 @@ def run_benchmark_on_target(target, script): def run_benchmarks(args, target, param_n, param_m, n_average, test_list): + test_results = [] skip_complex = run_feature_test(target, "complex") != "complex" skip_native = run_feature_test(target, "native_check") != "native" - target_had_error = False for test_file in sorted(test_list): print(test_file + ": ", end="") @@ -105,6 +104,7 @@ def run_benchmarks(args, target, param_n, param_m, n_average, test_list): and test_file.find("viper_") != -1 ) if skip: + test_results.append((test_file, "skip", "")) print("SKIP") continue @@ -122,9 +122,10 @@ def run_benchmarks(args, target, param_n, param_m, n_average, test_list): f.write(test_script) # Process script through mpy-cross if needed - if isinstance(target, pyboard.Pyboard) or args.via_mpy: + if hasattr(target, "enter_raw_repl") or args.via_mpy: crash, test_script_target = prepare_script_for_target(args, script_text=test_script) if crash: + test_results.append((test_file, "fail", "preparation")) print("CRASH:", test_script_target) continue else: @@ -162,10 +163,13 @@ def run_benchmarks(args, target, param_n, param_m, n_average, test_list): error = "FAIL truth" if error is not None: - if not error.startswith("SKIP"): - target_had_error = True + if error.startswith("SKIP"): + test_results.append((test_file, "skip", error)) + else: + test_results.append((test_file, "fail", error)) print(error) else: + test_results.append((test_file, "pass", "")) t_avg, t_sd = compute_stats(times) s_avg, s_sd = compute_stats(scores) print( @@ -179,7 +183,7 @@ def run_benchmarks(args, target, param_n, param_m, n_average, test_list): sys.stdout.flush() - return target_had_error + return test_results def parse_output(filename): @@ -190,7 +194,13 @@ def parse_output(filename): m = int(m.split("=")[1]) data = [] for l in f: - if ": " in l and ": SKIP" not in l and "CRASH: " not in l: + if ( + ": " in l + and ": SKIP" not in l + and "CRASH: " not in l + and "skipped: " not in l + and "failed: " not in l + ): name, values = l.strip().split(": ") values = tuple(float(v) for v in values.split()) data.append((name,) + values) @@ -246,17 +256,17 @@ def compute_diff(file1, file2, diff_score): def main(): cmd_parser = argparse.ArgumentParser(description="Run benchmarks for MicroPython") cmd_parser.add_argument( - "-t", "--diff-time", action="store_true", help="diff time outputs from a previous run" + "-m", "--diff-time", action="store_true", help="diff time outputs from a previous run" ) cmd_parser.add_argument( "-s", "--diff-score", action="store_true", help="diff score outputs from a previous run" ) cmd_parser.add_argument( - "-p", "--pyboard", action="store_true", help="run tests via pyboard.py" - ) - cmd_parser.add_argument( - "-d", "--device", default="/dev/ttyACM0", help="the device for pyboard.py" + "-t", "--test-instance", default="unix", help="the MicroPython instance to test" ) + cmd_parser.add_argument("--baudrate", default=115200, help="baud rate of the serial device") + cmd_parser.add_argument("--user", default="micro", help="telnet login username") + cmd_parser.add_argument("--password", default="python", help="telnet login password") cmd_parser.add_argument("-a", "--average", default="8", help="averaging number") cmd_parser.add_argument( "--emit", default="bytecode", help="MicroPython emitter to use (bytecode or native)" @@ -264,6 +274,12 @@ def main(): cmd_parser.add_argument("--heapsize", help="heapsize to use (use default if not specified)") cmd_parser.add_argument("--via-mpy", action="store_true", help="compile code to .mpy first") cmd_parser.add_argument("--mpy-cross-flags", default="", help="flags to pass to mpy-cross") + cmd_parser.add_argument( + "-r", + "--result-dir", + default=run_tests_module.base_path("results"), + help="directory for test results", + ) cmd_parser.add_argument( "N", nargs=1, help="N parameter (approximate target CPU frequency in MHz)" ) @@ -282,15 +298,18 @@ def main(): M = int(args.M[0]) n_average = int(args.average) - if args.pyboard: - if not args.mpy_cross_flags: - args.mpy_cross_flags = "-march=armv7m" - target = pyboard.Pyboard(args.device) - target.enter_raw_repl() - else: + target = run_tests_module.get_test_instance( + args.test_instance, args.baudrate, args.user, args.password + ) + if target is None: + # Use the unix port of MicroPython. target = [MICROPYTHON, "-X", "emit=" + args.emit] if args.heapsize is not None: target.extend(["-X", "heapsize=" + args.heapsize]) + else: + # Use a remote target. + if not args.mpy_cross_flags: + args.mpy_cross_flags = "-march=armv7m" if len(args.files) == 0: tests_skip = ("benchrun.py",) @@ -307,13 +326,15 @@ def main(): print("N={} M={} n_average={}".format(N, M, n_average)) - target_had_error = run_benchmarks(args, target, N, M, n_average, tests) + os.makedirs(args.result_dir, exist_ok=True) + test_results = run_benchmarks(args, target, N, M, n_average, tests) + res = run_tests_module.create_test_report(args, test_results) - if isinstance(target, pyboard.Pyboard): + if hasattr(target, "exit_raw_repl"): target.exit_raw_repl() target.close() - if target_had_error: + if not res: sys.exit(1) diff --git a/tests/run-tests.py b/tests/run-tests.py index cb839ae23dcdf..71570292ce7f7 100755 --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -15,13 +15,15 @@ import threading import tempfile -# Maximum time to run a PC-based test, in seconds. -TEST_TIMEOUT = 30 +# Maximum time to run a single test, in seconds. +TEST_TIMEOUT = float(os.environ.get("MICROPY_TEST_TIMEOUT", 30)) # See stackoverflow.com/questions/2632199: __file__ nor sys.argv[0] # are guaranteed to always work, this one should though. BASEPATH = os.path.dirname(os.path.abspath(inspect.getsourcefile(lambda: None))) +RV32_ARCH_FLAGS = {"zba": 1 << 0} + def base_path(*p): return os.path.abspath(os.path.join(BASEPATH, *p)).replace("\\", "/") @@ -58,6 +60,23 @@ def base_path(*p): # Set PYTHONIOENCODING so that CPython will use utf-8 on systems which set another encoding in the locale os.environ["PYTHONIOENCODING"] = "utf-8" + +def normalize_newlines(data): + """Normalize newline variations to \\n. + + Only normalizes actual line endings, not literal \\r characters in strings. + Handles \\r\\r\\n and \\r\\n cases to ensure consistent comparison + across different platforms and terminals. + """ + if isinstance(data, bytes): + # Handle PTY double-newline issue first + data = data.replace(b"\r\r\n", b"\n") + # Then handle standard Windows line endings + data = data.replace(b"\r\n", b"\n") + # Don't convert standalone \r as it might be literal content + return data + + # Code to allow a target MicroPython to import an .mpy from RAM # Note: the module is named `__injected_test` but it needs to have `__name__` set to # `__main__` so that the test sees itself as the main module, eg so unittest works. @@ -88,31 +107,82 @@ def getcwd(self): return "" def stat(self, path): if path == '__injected_test.mpy': - return tuple(0 for _ in range(10)) + return (0,0,0,0,0,0,0,0,0,0) else: - raise OSError(-2) # ENOENT + raise OSError(2) # ENOENT def open(self, path, mode): + self.stat(path) return __File() vfs.mount(__FS(), '/__vfstest') os.chdir('/__vfstest') +{import_prologue} __import__('__injected_test') """ # Platforms associated with the unix port, values of `sys.platform`. PC_PLATFORMS = ("darwin", "linux", "win32") +# Mapping from `sys.platform` to the port name, for special cases. +# See `platform_to_port()` function. +platform_to_port_map = {"pyboard": "stm32", "WiPy": "cc3200"} +platform_to_port_map.update({p: "unix" for p in PC_PLATFORMS}) + +# Tests to skip for values of the `--via-mpy` argument. +via_mpy_tests_to_skip = { + # Skip the following when mpy is enabled. + True: ( + # These print out the filename and that's expected to match the .py name. + "import/import_file.py", + "io/argv.py", + "misc/sys_settrace_features.py", + "misc/sys_settrace_generator.py", + "misc/sys_settrace_loop.py", + ), +} + +# Tests to skip for specific emitters. +emitter_tests_to_skip = { + # Some tests are known to fail with native emitter. + # Remove them from the below when they work. + "native": ( + # These require raise_varargs. + "basics/gen_yield_from_close.py", + "basics/try_finally_return2.py", + "basics/try_reraise.py", + "basics/try_reraise2.py", + "misc/features.py", + # These require checking for unbound local. + "basics/annotate_var.py", + "basics/del_deref.py", + "basics/del_local.py", + "basics/scope_implicit.py", + "basics/unboundlocal.py", + # These require "raise from". + "basics/exception_chain.py", + # These require stack-allocated slice optimisation. + "micropython/heapalloc_slice.py", + # These require running the scheduler. + "micropython/schedule.py", + "extmod/asyncio_event_queue.py", + "extmod/asyncio_iterator_event.py", + # These require sys.exc_info(). + "misc/sys_exc_info.py", + # These require sys.settrace(). + "misc/sys_settrace_cov.py", + "misc/sys_settrace_features.py", + "misc/sys_settrace_generator.py", + "misc/sys_settrace_loop.py", + # These are bytecode-specific tests. + "stress/bytecode_limit.py", + ), +} + # Tests to skip on specific targets. # These are tests that are difficult to detect that they should not be run on the given target. platform_tests_to_skip = { - "esp8266": ( - "micropython/viper_args.py", # too large - "micropython/viper_binop_arith.py", # too large - "misc/rge_sm.py", # too large - ), "minimal": ( "basics/class_inplace_op.py", # all special methods not supported "basics/subclass_native_init.py", # native subclassing corner cases not support - "misc/rge_sm.py", # too large "micropython/opt_level.py", # don't assume line numbers are stored ), "nrf": ( @@ -140,14 +210,6 @@ def open(self, path, mode): "thread/thread_lock3.py", "thread/thread_shared2.py", ), - "qemu": ( - # Skip tests that require Cortex-M4. - "inlineasm/thumb/asmfpaddsub.py", - "inlineasm/thumb/asmfpcmp.py", - "inlineasm/thumb/asmfpldrstr.py", - "inlineasm/thumb/asmfpmuldiv.py", - "inlineasm/thumb/asmfpsqrt.py", - ), "webassembly": ( "basics/string_format_modulo.py", # can't print nulls to stdout "basics/string_strip.py", # can't print nulls to stdout @@ -162,6 +224,9 @@ def open(self, path, mode): "extmod/asyncio_new_event_loop.py", "extmod/asyncio_threadsafeflag.py", "extmod/asyncio_wait_for_fwd.py", + "extmod/asyncio_event_queue.py", + "extmod/asyncio_iterator_event.py", + "extmod/asyncio_wait_for_linked_task.py", "extmod/binascii_a2b_base64.py", "extmod/deflate_compress_memory_error.py", # tries to allocate unlimited memory "extmod/re_stack_overflow.py", @@ -184,6 +249,89 @@ def open(self, path, mode): ), } +# These tests don't test float explicitly but rather use it to perform the test. +tests_requiring_float = ( + "extmod/asyncio_basic.py", + "extmod/asyncio_basic2.py", + "extmod/asyncio_cancel_task.py", + "extmod/asyncio_event.py", + "extmod/asyncio_fair.py", + "extmod/asyncio_gather.py", + "extmod/asyncio_gather_notimpl.py", + "extmod/asyncio_get_event_loop.py", + "extmod/asyncio_iterator_event.py", + "extmod/asyncio_lock.py", + "extmod/asyncio_task_done.py", + "extmod/asyncio_wait_for.py", + "extmod/asyncio_wait_for_fwd.py", + "extmod/asyncio_wait_for_linked_task.py", + "extmod/asyncio_wait_task.py", + "extmod/json_dumps_float.py", + "extmod/json_loads_float.py", + "extmod/random_extra_float.py", + "extmod/select_poll_eintr.py", + "extmod/tls_threads.py", + "extmod/uctypes_le_float.py", + "extmod/uctypes_native_float.py", + "extmod/uctypes_sizeof_float.py", + "misc/rge_sm.py", + "ports/unix/ffi_float.py", + "ports/unix/ffi_float2.py", +) + +# These tests don't test slice explicitly but rather use it to perform the test. +tests_requiring_slice = ( + "basics/builtin_range.py", + "basics/bytearray1.py", + "basics/class_super.py", + "basics/containment.py", + "basics/errno1.py", + "basics/fun_str.py", + "basics/generator1.py", + "basics/globals_del.py", + "basics/memoryview1.py", + "basics/memoryview_gc.py", + "basics/object1.py", + "basics/python34.py", + "basics/struct_endian.py", + "extmod/btree1.py", + "extmod/deflate_decompress.py", + "extmod/framebuf16.py", + "extmod/framebuf4.py", + "extmod/machine1.py", + "extmod/time_mktime.py", + "extmod/time_res.py", + "extmod/tls_sslcontext_ciphers.py", + "extmod/vfs_fat_fileio1.py", + "extmod/vfs_fat_finaliser.py", + "extmod/vfs_fat_more.py", + "extmod/vfs_fat_ramdisk.py", + "extmod/vfs_fat_ramdisklarge.py", + "extmod/vfs_lfs.py", + "extmod/vfs_rom.py", + "float/string_format_modulo.py", + "micropython/builtin_execfile.py", + "micropython/extreme_exc.py", + "micropython/heapalloc_fail_bytearray.py", + "micropython/heapalloc_fail_list.py", + "micropython/heapalloc_fail_memoryview.py", + "micropython/import_mpy_invalid.py", + "micropython/import_mpy_native.py", + "micropython/import_mpy_native_gc.py", + "misc/non_compliant.py", + "misc/rge_sm.py", +) + +# Tests that require `import target_wiring` to work. +tests_requiring_target_wiring = ( + "extmod/machine_uart_irq_txidle.py", + "extmod/machine_uart_tx.py", + "extmod_hardware/machine_encoder.py", + "extmod_hardware/machine_uart_irq_break.py", + "extmod_hardware/machine_uart_irq_rx.py", + "extmod_hardware/machine_uart_irq_rxidle.py", +) + def rm_f(fname): if os.path.exists(fname): @@ -210,22 +358,31 @@ def convert_regex_escapes(line): return bytes("".join(cs), "utf8") +def platform_to_port(platform): + return platform_to_port_map.get(platform, platform) + + +def convert_device_shortcut_to_real_device(device): + if device.startswith("port:"): + return device.split(":", 1)[1] + elif device.startswith("a") and device[1:].isdigit(): + return "/dev/ttyACM" + device[1:] + elif device.startswith("u") and device[1:].isdigit(): + return "/dev/ttyUSB" + device[1:] + elif device.startswith("c") and device[1:].isdigit(): + return "COM" + device[1:] + else: + return device + + def get_test_instance(test_instance, baudrate, user, password): - if test_instance.startswith("port:"): - _, port = test_instance.split(":", 1) - elif test_instance == "unix": + if test_instance == "unix": return None elif test_instance == "webassembly": return PyboardNodeRunner() - elif test_instance.startswith("a") and test_instance[1:].isdigit(): - port = "/dev/ttyACM" + test_instance[1:] - elif test_instance.startswith("u") and test_instance[1:].isdigit(): - port = "/dev/ttyUSB" + test_instance[1:] - elif test_instance.startswith("c") and test_instance[1:].isdigit(): - port = "COM" + test_instance[1:] else: # Assume it's a device path. - port = test_instance + port = convert_device_shortcut_to_real_device(test_instance) global pyboard sys.path.append(base_path("../tools")) @@ -245,46 +402,114 @@ def detect_inline_asm_arch(pyb, args): return None +def map_rv32_arch_flags(flags): + mapped_flags = [] + for extension, bit in RV32_ARCH_FLAGS.items(): + if flags & bit: + mapped_flags.append(extension) + flags &= ~bit + if flags: + raise Exception("Unexpected flag bits set in value {}".format(flags)) + return mapped_flags + + def detect_test_platform(pyb, args): # Run a script to detect various bits of information about the target test instance. output = run_feature_check(pyb, args, "target_info.py") if output.endswith(b"CRASH"): raise ValueError("cannot detect platform: {}".format(output)) - platform, arch = str(output, "ascii").strip().split() + platform, arch, arch_flags, build, thread, float_prec, unicode = ( + str(output, "ascii").strip().split() + ) if arch == "None": arch = None inlineasm_arch = detect_inline_asm_arch(pyb, args) + if thread == "None": + thread = None + float_prec = int(float_prec) + unicode = unicode == "True" + if arch == "rv32imc": + arch_flags = map_rv32_arch_flags(int(arch_flags)) + else: + arch_flags = None args.platform = platform args.arch = arch + args.arch_flags = arch_flags if arch and not args.mpy_cross_flags: args.mpy_cross_flags = "-march=" + arch + if arch_flags: + args.mpy_cross_flags += " -march-flags=" + ",".join(arch_flags) args.inlineasm_arch = inlineasm_arch + args.build = build + args.thread = thread + args.float_prec = float_prec + args.unicode = unicode + # Print the detected information about the target. print("platform={}".format(platform), end="") if arch: print(" arch={}".format(arch), end="") + if arch_flags: + print(" arch_flags={}".format(",".join(arch_flags)), end="") if inlineasm_arch: print(" inlineasm={}".format(inlineasm_arch), end="") - print() - - -def prepare_script_for_target(args, *, script_filename=None, script_text=None, force_plain=False): + if thread: + print(" thread={}".format(thread), end="") + if float_prec: + print(" float={}-bit".format(float_prec), end="") + if unicode: + print(" unicode", end="") + + +def detect_target_wiring_script(pyb, args): + tw_data = b"" + tw_source = None + if args.target_wiring: + # A target_wiring path is explicitly provided, so use that. + tw_source = args.target_wiring + with open(tw_source, "rb") as f: + tw_data = f.read() + elif hasattr(pyb, "exec_raw") and pyb.exec_raw("import target_wiring") == (b"", b""): + # The board already has a target_wiring module available, so use that. + tw_source = "on-device" + else: + port = platform_to_port(args.platform) + build = args.build + tw_board_exact = None + tw_board_partial = None + tw_port = None + for file in os.listdir("target_wiring"): + file_base = file.removesuffix(".py") + if file_base == build: + # A file matching the target's board/build name. + tw_board_exact = file + elif file_base.endswith("x") and build.startswith(file_base.removesuffix("x")): + # A file with a partial match to the target's board/build name. + tw_board_partial = file + elif file_base == port: + # A file matching the target's port. + tw_port = file + tw_source = tw_board_exact or tw_board_partial or tw_port + if tw_source: + with open("target_wiring/" + tw_source, "rb") as f: + tw_data = f.read() + if tw_source: + print(" target_wiring={}".format(tw_source), end="") + pyb.target_wiring_script = tw_data + + +def prepare_script_for_target(args, *, script_text=None, force_plain=False): if force_plain or (not args.via_mpy and args.emit == "bytecode"): - if script_filename is not None: - with open(script_filename, "rb") as f: - script_text = f.read() + # A plain test to run as-is, no processing needed. + pass elif args.via_mpy: tempname = tempfile.mktemp(dir="") mpy_filename = tempname + ".mpy" - if script_filename is None: - script_filename = tempname + ".py" - cleanup_script_filename = True - with open(script_filename, "wb") as f: - f.write(script_text) - else: - cleanup_script_filename = False + script_filename = tempname + ".py" + with open(script_filename, "wb") as f: + f.write(script_text) try: subprocess.check_output( @@ -300,8 +525,7 @@ def prepare_script_for_target(args, *, script_filename=None, script_text=None, f script_text = b"__buf=" + bytes(repr(f.read()), "ascii") + b"\n" rm_f(mpy_filename) - if cleanup_script_filename: - rm_f(script_filename) + rm_f(script_filename) script_text += bytes(injected_import_hook_code, "ascii") else: @@ -312,34 +536,61 @@ def prepare_script_for_target(args, *, script_filename=None, script_text=None, f def run_script_on_remote_target(pyb, args, test_file, is_special): - had_crash, script = prepare_script_for_target( - args, script_filename=test_file, force_plain=is_special - ) + with open(test_file, "rb") as f: + script = f.read() + + # If the test is not a special test, prepend it with a print to indicate that it started. + # If the print does not execute this means that the test did not even start, eg it was + # too large for the target. + prepend_start_test = not is_special + if prepend_start_test: + if script.startswith(b"#"): + script = b"print('START TEST')" + script + else: + script = b"print('START TEST')\n" + script + + had_crash, script = prepare_script_for_target(args, script_text=script, force_plain=is_special) + if had_crash: return True, script try: had_crash = False pyb.enter_raw_repl() - output_mupy = pyb.exec_(script) + if test_file.endswith(tests_requiring_target_wiring) and pyb.target_wiring_script: + pyb.exec_( + "import sys;sys.modules['target_wiring']=__build_class__(lambda:exec(" + + repr(pyb.target_wiring_script) + + "),'target_wiring')" + ) + output_mupy = pyb.exec_(script, timeout=TEST_TIMEOUT) except pyboard.PyboardError as e: had_crash = True if not is_special and e.args[0] == "exception": - output_mupy = e.args[1] + e.args[2] + b"CRASH" + if prepend_start_test and e.args[1] == b"" and b"MemoryError" in e.args[2]: + output_mupy = b"SKIP-TOO-LARGE\n" + else: + output_mupy = e.args[1] + e.args[2] + b"CRASH" else: output_mupy = bytes(e.args[0], "ascii") + b"\nCRASH" + + if prepend_start_test: + if output_mupy.startswith(b"START TEST\r\n"): + output_mupy = output_mupy.removeprefix(b"START TEST\r\n") + else: + had_crash = True + return had_crash, output_mupy -special_tests = [ +tests_with_regex_output = [ base_path(file) for file in ( + # CIRCUITPY-CHANGE: removal and additions "micropython/meminfo.py", "basics/bytes_compare3.py", "basics/builtin_help.py", "thread/thread_exc2.py", - # CIRCUITPY-CHANGE: removal and additions - # REMOVE "esp32/partition_ota.py", "circuitpython/traceback_test.py", "circuitpython/traceback_test_chained.py", ) @@ -350,10 +601,7 @@ def run_micropython(pyb, args, test_file, test_file_abspath, is_special=False): had_crash = False if pyb is None: # run on PC - if ( - test_file_abspath.startswith((base_path("cmdline/"), base_path("feature_check/"))) - or test_file_abspath in special_tests - ): + if test_file_abspath.startswith((base_path("cmdline/"), base_path("feature_check/"))): # special handling for tests of the unix cmdline program is_special = True @@ -392,6 +640,10 @@ def get(required=False): return rv def send_get(what): + # Detect {\x00} pattern and convert to ctrl-key codes. + ctrl_code = lambda m: bytes([int(m.group(1))]) + what = re.sub(rb"{\\x(\d\d)}", ctrl_code, what) + os.write(master, what) return get() @@ -471,17 +723,17 @@ def send_get(what): ) # canonical form for all ports/platforms is to use \n for end-of-line - output_mupy = output_mupy.replace(b"\r\n", b"\n") + output_mupy = normalize_newlines(output_mupy) # don't try to convert the output if we should skip this test - if had_crash or output_mupy in (b"SKIP\n", b"CRASH"): + if had_crash or output_mupy in (b"SKIP\n", b"SKIP-TOO-LARGE\n", b"CRASH"): return output_mupy # skipped special tests will output "SKIP" surrounded by other interpreter debug output if is_special and not had_crash and b"\nSKIP\n" in output_mupy: return b"SKIP\n" - if is_special or test_file_abspath in special_tests: + if is_special or test_file_abspath in tests_with_regex_output: # convert parts of the output that are not stable across runs with open(test_file + ".exp", "rb") as f: lines_exp = [] @@ -603,30 +855,25 @@ def run_script_on_remote_target(self, args, test_file, is_special): def run_tests(pyb, tests, args, result_dir, num_threads=1): - test_count = ThreadSafeCounter() testcase_count = ThreadSafeCounter() - passed_count = ThreadSafeCounter() - failed_tests = ThreadSafeCounter([]) - skipped_tests = ThreadSafeCounter([]) + test_results = ThreadSafeCounter([]) skip_tests = set() skip_native = False skip_int_big = False + skip_int_64 = False skip_bytearray = False skip_set_type = False skip_slice = False skip_async = False skip_const = False skip_revops = False - skip_io_module = False skip_fstring = False skip_endian = False skip_inlineasm = False has_complex = True has_coverage = False - upy_float_precision = 32 - if True: # Even if we run completely different tests in a different directory, # we need to access feature_checks from the same directory as the @@ -642,6 +889,11 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): if output != b"1000000000000000000000000000000000000000000000\n": skip_int_big = True + # Check if 'long long' precision integers are supported, even if arbitrary precision is not + output = run_feature_check(pyb, args, "int_64.py") + if output != b"4611686018427387904\n": + skip_int_64 = True + # Check if bytearray is supported, and skip such tests if it's not output = run_feature_check(pyb, args, "bytearray.py") if output != b"bytearray\n": @@ -672,11 +924,6 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): if output == b"TypeError\n": skip_revops = True - # Check if io module exists, and skip such tests if it doesn't - output = run_feature_check(pyb, args, "io_module.py") - if output != b"io\n": - skip_io_module = True - # Check if fstring feature is enabled, and skip such tests if it doesn't output = run_feature_check(pyb, args, "fstring.py") if output != b"a=1\n": @@ -690,13 +937,20 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): skip_tests.add("inlineasm/thumb/asmbitops.py") skip_tests.add("inlineasm/thumb/asmconst.py") skip_tests.add("inlineasm/thumb/asmdiv.py") + skip_tests.add("inlineasm/thumb/asmit.py") + skip_tests.add("inlineasm/thumb/asmspecialregs.py") + if args.arch not in ("armv7emsp", "armv7emdp"): skip_tests.add("inlineasm/thumb/asmfpaddsub.py") skip_tests.add("inlineasm/thumb/asmfpcmp.py") skip_tests.add("inlineasm/thumb/asmfpldrstr.py") skip_tests.add("inlineasm/thumb/asmfpmuldiv.py") skip_tests.add("inlineasm/thumb/asmfpsqrt.py") - skip_tests.add("inlineasm/thumb/asmit.py") - skip_tests.add("inlineasm/thumb/asmspecialregs.py") + + if args.inlineasm_arch == "rv32": + # Check if @micropython.asm_rv32 supports Zba instructions, and skip such tests if it doesn't + output = run_feature_check(pyb, args, "inlineasm_rv32_zba.py") + if output != b"rv32_zba\n": + skip_tests.add("inlineasm/rv32/asmzba.py") # Check if emacs repl is supported, and skip such tests if it's not t = run_feature_check(pyb, args, "repl_emacs_check.py") @@ -709,11 +963,6 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): skip_tests.add("cmdline/repl_words_move.py") upy_byteorder = run_feature_check(pyb, args, "byteorder.py") - upy_float_precision = run_feature_check(pyb, args, "float.py") - try: - upy_float_precision = int(upy_float_precision) - except ValueError: - upy_float_precision = 0 has_complex = run_feature_check(pyb, args, "complex.py") == b"complex\n" has_coverage = run_feature_check(pyb, args, "coverage.py") == b"coverage\n" cpy_byteorder = subprocess.check_output( @@ -723,24 +972,6 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): skip_inlineasm = args.inlineasm_arch is None - # These tests don't test slice explicitly but rather use it to perform the test - misc_slice_tests = ( - "builtin_range", - "bytearray1", - "class_super", - "containment", - "errno1", - "fun_str", - "generator1", - "globals_del", - "memoryview1", - "memoryview_gc", - "object1", - "python34", - "string_format_modulo", - "struct_endian", - ) - # Some tests shouldn't be run on GitHub Actions if os.getenv("GITHUB_ACTIONS") == "true": skip_tests.add("thread/stress_schedule.py") # has reliability issues @@ -749,30 +980,31 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): # fails with stack overflow on Debug builds skip_tests.add("misc/sys_settrace_features.py") - if upy_float_precision == 0: - skip_tests.add("extmod/uctypes_le_float.py") - skip_tests.add("extmod/uctypes_native_float.py") - skip_tests.add("extmod/uctypes_sizeof_float.py") - skip_tests.add("extmod/json_dumps_float.py") - skip_tests.add("extmod/json_loads_float.py") - skip_tests.add("extmod/random_extra_float.py") - skip_tests.add("misc/rge_sm.py") - if upy_float_precision < 32: + if args.float_prec == 0: + skip_tests.update(tests_requiring_float) + if args.float_prec < 32: skip_tests.add( "float/float2int_intbig.py" ) # requires fp32, there's float2int_fp30_intbig.py instead skip_tests.add( - "float/string_format.py" - ) # requires fp32, there's string_format_fp30.py instead + "float/float_struct_e.py" + ) # requires fp32, there's float_struct_e_fp30.py instead skip_tests.add("float/bytes_construct.py") # requires fp32 skip_tests.add("float/bytearray_construct.py") # requires fp32 skip_tests.add("float/float_format_ints_power10.py") # requires fp32 - if upy_float_precision < 64: + if args.float_prec < 64: skip_tests.add("float/float_divmod.py") # tested by float/float_divmod_relaxed.py instead skip_tests.add("float/float2int_doubleprec_intbig.py") + skip_tests.add("float/float_struct_e_doubleprec.py") skip_tests.add("float/float_format_ints_doubleprec.py") skip_tests.add("float/float_parse_doubleprec.py") + if not args.unicode: + skip_tests.add("extmod/json_loads.py") # tests loading a utf-8 character + + if skip_slice: + skip_tests.update(tests_requiring_slice) + if not has_complex: skip_tests.add("float/complex1.py") skip_tests.add("float/complex1_intbig.py") @@ -788,8 +1020,8 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): skip_tests.add("cmdline/repl_sys_ps1_ps2.py") skip_tests.add("extmod/ssl_poll.py") - # Skip thread mutation tests on targets that don't have the GIL. - if args.platform in PC_PLATFORMS + ("rp2",): + # Skip thread mutation tests on targets that have unsafe threading behaviour. + if args.thread == "unsafe": for t in tests: if t.startswith("thread/mutate_"): skip_tests.add(t) @@ -798,6 +1030,15 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): if args.platform not in PC_PLATFORMS: skip_tests.add("basics/exception_chain.py") # warning is not printed skip_tests.add("micropython/meminfo.py") # output is very different to PC output + skip_tests.add("unicode/file1.py") # requires local file access + skip_tests.add("unicode/file2.py") # requires local file access + skip_tests.add("unicode/file_invalid.py") # requires local file access + + # Skip certain tests when going via a .mpy file. + skip_tests.update(via_mpy_tests_to_skip.get(args.via_mpy, ())) + + # Skip emitter-specific tests. + skip_tests.update(emitter_tests_to_skip.get(args.emit, ())) # Skip platform-specific tests. skip_tests.update(platform_tests_to_skip.get(args.platform, ())) @@ -812,49 +1053,6 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): # Works but CPython uses '\' path separator skip_tests.add("import/import_file.py") - # Some tests are known to fail with native emitter - # Remove them from the below when they work - if args.emit == "native": - skip_tests.add("basics/gen_yield_from_close.py") # require raise_varargs - skip_tests.update( - {"basics/%s.py" % t for t in "try_reraise try_reraise2".split()} - ) # require raise_varargs - skip_tests.add("basics/annotate_var.py") # requires checking for unbound local - skip_tests.add("basics/del_deref.py") # requires checking for unbound local - skip_tests.add("basics/del_local.py") # requires checking for unbound local - skip_tests.add("basics/exception_chain.py") # raise from is not supported - skip_tests.add("basics/scope_implicit.py") # requires checking for unbound local - skip_tests.add("basics/sys_tracebacklimit.py") # requires traceback info - skip_tests.add("basics/try_finally_return2.py") # requires raise_varargs - skip_tests.add("basics/unboundlocal.py") # requires checking for unbound local - skip_tests.add("misc/features.py") # requires raise_varargs - # CIRCUITPY-CHANGE - skip_tests.update( - ( - "basics/chained_exception.py", - "circuitpython/traceback_test.py", - "circuitpython/traceback_test_chained.py", - ) - ) # because native doesn't have proper traceback info - skip_tests.add( - "misc/print_exception.py" - ) # because native doesn't have proper traceback info - skip_tests.add("misc/sys_exc_info.py") # sys.exc_info() is not supported for native - skip_tests.add("misc/sys_settrace_features.py") # sys.settrace() not supported - skip_tests.add("misc/sys_settrace_generator.py") # sys.settrace() not supported - skip_tests.add("misc/sys_settrace_loop.py") # sys.settrace() not supported - skip_tests.add( - "micropython/emg_exc.py" - ) # because native doesn't have proper traceback info - skip_tests.add( - "micropython/heapalloc_traceback.py" - ) # because native doesn't have proper traceback info - skip_tests.add( - "micropython/opt_level_lineno.py" - ) # native doesn't have proper traceback info - skip_tests.add("micropython/schedule.py") # native code doesn't check pending events - skip_tests.add("stress/bytecode_limit.py") # bytecode specific test - def run_one_test(test_file): test_file = test_file.replace("\\", "/") test_file_abspath = os.path.abspath(test_file).replace("\\", "/") @@ -870,19 +1068,19 @@ def run_one_test(test_file): test_basename = test_file.replace("..", "_").replace("./", "").replace("/", "_") test_name = os.path.splitext(os.path.basename(test_file))[0] - is_native = ( - test_name.startswith("native_") - or test_name.startswith("viper_") - or args.emit == "native" - ) + is_native = test_name.startswith("native_") or test_name.startswith("viper_") is_endian = test_name.endswith("_endian") - is_int_big = test_name.startswith("int_big") or test_name.endswith("_intbig") + is_int_big = ( + test_name.startswith("int_big") + or test_name.endswith("_intbig") + or test_name.startswith("ffi_int") # these tests contain large integer literals + ) + is_int_64 = test_name.startswith("int_64") or test_name.endswith("_int64") is_bytearray = test_name.startswith("bytearray") or test_name.endswith("_bytearray") is_set_type = test_name.startswith(("set_", "frozenset")) or test_name.endswith("_set") - is_slice = test_name.find("slice") != -1 or test_name in misc_slice_tests - is_async = test_name.startswith(("async_", "asyncio_")) + is_slice = test_name.find("slice") != -1 + is_async = test_name.startswith(("async_", "asyncio_")) or test_name.endswith("_async") is_const = test_name.startswith("const") - is_io_module = test_name.startswith("io_") is_fstring = test_name.startswith("string_fstring") is_inlineasm = test_name.startswith("asm") @@ -890,19 +1088,19 @@ def run_one_test(test_file): skip_it |= skip_native and is_native skip_it |= skip_endian and is_endian skip_it |= skip_int_big and is_int_big + skip_it |= skip_int_64 and is_int_64 skip_it |= skip_bytearray and is_bytearray skip_it |= skip_set_type and is_set_type skip_it |= skip_slice and is_slice skip_it |= skip_async and is_async skip_it |= skip_const and is_const skip_it |= skip_revops and "reverse_op" in test_name - skip_it |= skip_io_module and is_io_module skip_it |= skip_fstring and is_fstring skip_it |= skip_inlineasm and is_inlineasm if skip_it: print("skip ", test_file) - skipped_tests.append(test_name) + test_results.append((test_file, "skip", "")) return # Run the test on the MicroPython target. @@ -917,7 +1115,11 @@ def run_one_test(test_file): # start-up code (eg boot.py) when preparing to run the next test. pyb.read_until(1, b"raw REPL; CTRL-B to exit\r\n") print("skip ", test_file) - skipped_tests.append(test_name) + test_results.append((test_file, "skip", "")) + return + elif output_mupy == b"SKIP-TOO-LARGE\n": + print("lrge ", test_file) + test_results.append((test_file, "skip", "too large")) return # Look at the output of the test to see if unittest was used. @@ -954,7 +1156,11 @@ def run_one_test(test_file): # Expected output is result of running unittest. output_expected = None else: - test_file_expected = test_file + ".exp" + # Prefer emitter-specific expected output. + test_file_expected = test_file + "." + args.emit + ".exp" + if not os.path.isfile(test_file_expected): + # Fall back to generic expected output. + test_file_expected = test_file + ".exp" if os.path.isfile(test_file_expected): # Expected output given by a file, so read that in. with open(test_file_expected, "rb") as f: @@ -1012,7 +1218,7 @@ def run_one_test(test_file): # Print test summary, update counters, and save .exp/.out files if needed. if test_passed: print("pass ", test_file, extra_info) - passed_count.increment() + test_results.append((test_file, "pass", "")) rm_f(filename_expected) rm_f(filename_mupy) else: @@ -1024,9 +1230,7 @@ def run_one_test(test_file): rm_f(filename_expected) # in case left over from previous failed run with open(filename_mupy, "wb") as f: f.write(output_mupy) - failed_tests.append((test_name, test_file)) - - test_count.increment() + test_results.append((test_file, "fail", "")) # Print a note if this looks like it might have been a misfired unittest if not uses_unittest and not test_passed: @@ -1053,17 +1257,49 @@ def run_one_test(test_file): print(line) sys.exit(1) - print( - "{} tests performed ({} individual testcases)".format( - test_count.value, testcase_count.value - ) + # Return test results. + return test_results.value, testcase_count.value + + +# Print a summary of the results and save them to a JSON file. +# Returns True if everything succeeded, False otherwise. +def create_test_report(args, test_results, testcase_count=None): + passed_tests = list(r for r in test_results if r[1] == "pass") + skipped_tests = list(r for r in test_results if r[1] == "skip" and r[2] != "too large") + skipped_tests_too_large = list( + r for r in test_results if r[1] == "skip" and r[2] == "too large" ) - print("{} tests passed".format(passed_count.value)) + failed_tests = list(r for r in test_results if r[1] == "fail") + + num_tests_performed = len(passed_tests) + len(failed_tests) + + testcase_count_info = "" + if testcase_count is not None: + testcase_count_info = " ({} individual testcases)".format(testcase_count) + print("{} tests performed{}".format(num_tests_performed, testcase_count_info)) + + print("{} tests passed".format(len(passed_tests))) - skipped_tests = sorted(skipped_tests.value) if len(skipped_tests) > 0: - print("{} tests skipped: {}".format(len(skipped_tests), " ".join(skipped_tests))) - failed_tests = sorted(failed_tests.value) + print( + "{} tests skipped: {}".format( + len(skipped_tests), " ".join(test[0] for test in skipped_tests) + ) + ) + + if len(skipped_tests_too_large) > 0: + print( + "{} tests skipped because they are too large: {}".format( + len(skipped_tests_too_large), " ".join(test[0] for test in skipped_tests_too_large) + ) + ) + + if len(failed_tests) > 0: + print( + "{} tests failed: {}".format( + len(failed_tests), " ".join(test[0] for test in failed_tests) + ) + ) # Serialize regex added by append_filter. def to_json(obj): @@ -1071,23 +1307,22 @@ def to_json(obj): return obj.pattern return obj - with open(os.path.join(result_dir, RESULTS_FILE), "w") as f: + with open(os.path.join(args.result_dir, RESULTS_FILE), "w") as f: json.dump( - {"args": vars(args), "failed_tests": [test[1] for test in failed_tests]}, + { + # The arguments passed on the command-line. + "args": vars(args), + # A list of all results of the form [(test, result, reason), ...]. + "results": list(test for test in test_results), + # A list of failed tests. This is deprecated, use the "results" above instead. + "failed_tests": [test[0] for test in failed_tests], + }, f, default=to_json, ) - if len(failed_tests) > 0: - print( - "{} tests failed: {}".format( - len(failed_tests), " ".join(test[0] for test in failed_tests) - ) - ) - return False - - # all tests succeeded - return True + # Return True only if all tests succeeded. + return len(failed_tests) == 0 class append_filter(argparse.Action): @@ -1104,27 +1339,11 @@ def __call__(self, parser, args, value, option): args.filters.append((option, re.compile(value))) -def main(): - cmd_parser = argparse.ArgumentParser( - formatter_class=argparse.RawDescriptionHelpFormatter, - description="""Run and manage tests for MicroPython. - +test_instance_description = """\ By default the tests are run against the unix port of MicroPython. To run it against something else, use the -t option. See below for details. - -Tests are discovered by scanning test directories for .py files or using the -specified test files. If test files nor directories are specified, the script -expects to be ran in the tests directory (where this file is located) and the -builtin tests suitable for the target platform are ran. - -When running tests, run-tests.py compares the MicroPython output of the test with the output -produced by running the test through CPython unless a .exp file is found, in which -case it is used as comparison. - -If a test fails, run-tests.py produces a pair of .out and .exp files in the result -directory with the MicroPython output and the expectations, respectively. -""", - epilog="""\ +""" +test_instance_epilog = """\ The -t option accepts the following for the test instance: - unix - use the unix port of MicroPython, specified by the MICROPY_MICROPYTHON environment variable (which defaults to the standard variant of either the unix @@ -1140,7 +1359,35 @@ def main(): - execpty: - execute a command and attach to the printed /dev/pts/ device - ... - connect to the given IPv4 address - anything else specifies a serial port +""" + +test_directory_description = """\ +Tests are discovered by scanning test directories for .py files or using the +specified test files. If test files nor directories are specified, the script +expects to be ran in the tests directory (where this file is located) and the +builtin tests suitable for the target platform are ran. +""" + +def main(): + global injected_import_hook_code + + cmd_parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description=f"""Run and manage tests for MicroPython. + +{test_instance_description} +{test_directory_description} + +When running tests, run-tests.py compares the MicroPython output of the test with the output +produced by running the test through CPython unless a .exp file is found (or a +.native.exp file when using the native emitter), in which case it is used as comparison. + +If a test fails, run-tests.py produces a pair of .out and .exp files in the result +directory with the MicroPython output and the expectations, respectively. +""", + epilog=f"""\ +{test_instance_epilog} Options -i and -e can be multiple and processed in the order given. Regex "search" (vs "match") operation is used. An action (include/exclude) of the last matching regex is used: @@ -1214,8 +1461,25 @@ def main(): action="store_true", help="re-run only the failed tests", ) + cmd_parser.add_argument( + "--begin", + metavar="PROLOGUE", + default=None, + help="prologue python file to execute before module import", + ) + cmd_parser.add_argument( + "--target-wiring", + default=None, + help="force the given script to be used as target_wiring.py", + ) args = cmd_parser.parse_args() + prologue = "" + if args.begin: + with open(args.begin, "rt") as source: + prologue = source.read() + injected_import_hook_code = injected_import_hook_code.replace("{import_prologue}", prologue) + if args.print_failures: for out in glob(os.path.join(args.result_dir, "*.out")): testbase = out[:-4] @@ -1256,7 +1520,7 @@ def main(): results_file = os.path.join(args.result_dir, RESULTS_FILE) if os.path.exists(results_file): with open(results_file, "r") as f: - tests = json.load(f)["failed_tests"] + tests = list(test[0] for test in json.load(f)["results"] if test[1] == "fail") else: tests = [] elif len(args.files) == 0: @@ -1267,47 +1531,27 @@ def main(): if args.test_dirs is None: test_dirs = ( "basics", - "circuitpython", # CIRCUITPY-CHANGE "micropython", "misc", "extmod", + "stress", ) if args.inlineasm_arch is not None: test_dirs += ("inlineasm/{}".format(args.inlineasm_arch),) - if args.platform == "pyboard": - # run pyboard tests - test_dirs += ("float", "stress", "ports/stm32") - elif args.platform == "mimxrt": - test_dirs += ("float", "stress") - elif args.platform == "renesas-ra": - test_dirs += ("float", "ports/renesas-ra") - elif args.platform == "rp2": - test_dirs += ("float", "stress", "thread", "ports/rp2") - elif args.platform == "esp32": - test_dirs += ("float", "stress", "thread") - elif args.platform in ("esp8266", "minimal", "samd", "nrf"): + if args.thread is not None: + test_dirs += ("thread",) + if args.float_prec > 0: test_dirs += ("float",) - elif args.platform == "WiPy": - # run WiPy tests - test_dirs += ("ports/cc3200",) - elif args.platform in PC_PLATFORMS: + if args.unicode: + test_dirs += ("unicode",) + port_specific_test_dir = "ports/{}".format(platform_to_port(args.platform)) + if os.path.isdir(port_specific_test_dir): + test_dirs += (port_specific_test_dir,) + if args.platform in PC_PLATFORMS: # run PC tests - test_dirs += ( - "float", - "import", - "io", - "stress", - "unicode", - "cmdline", - "ports/unix", - ) - elif args.platform == "qemu": - test_dirs += ( - "float", - "ports/qemu", - ) - elif args.platform == "webassembly": - test_dirs += ("float", "ports/webassembly") + test_dirs += ("import",) + if args.build != "minimal": + test_dirs += ("cmdline", "io") else: # run tests from these directories test_dirs = args.test_dirs @@ -1322,6 +1566,13 @@ def main(): # tests explicitly given tests = args.files + # If any tests need it, prepare the target_wiring script for the target. + if pyb and any(test.endswith(tests_requiring_target_wiring) for test in tests): + detect_target_wiring_script(pyb, args) + + # End the target information line. + print() + if not args.keep_path: # Clear search path to make sure tests use only builtin modules, those in # extmod, and a path to unittest in case it's needed. @@ -1342,7 +1593,8 @@ def main(): try: os.makedirs(args.result_dir, exist_ok=True) - res = run_tests(pyb, tests, args, args.result_dir, args.jobs) + test_results, testcase_count = run_tests(pyb, tests, args, args.result_dir, args.jobs) + res = create_test_report(args, test_results, testcase_count) finally: if pyb: pyb.close() diff --git a/tests/serial_test.py b/tests/serial_test.py new file mode 100755 index 0000000000000..3b5940d91a907 --- /dev/null +++ b/tests/serial_test.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python +# +# Performance and reliability test for serial port communication. +# +# Basic usage: +# serial_test.py [-t serial-device] +# +# The `serial-device` will default to /dev/ttyACM0. + +import argparse +import random +import serial +import sys +import time + +run_tests_module = __import__("run-tests") + +echo_test_script = """ +import sys +bytes_min=%u +bytes_max=%u +repeat=%u +b=memoryview(bytearray(bytes_max)) +for n in range(bytes_min,bytes_max+1): + for _ in range(repeat): + n2 = sys.stdin.readinto(b[:n]) + sys.stdout.write(b[:n2]) +""" + +read_test_script = """ +bin = True +try: + wr=__import__("pyb").USB_VCP(0).send +except: + import sys + if hasattr(sys.stdout,'buffer'): + wr=sys.stdout.buffer.write + else: + wr=sys.stdout.write + bin = False +b=bytearray(%u) +if bin: + wr('BIN') + for i in range(len(b)): + b[i] = i & 0xff +else: + wr('TXT') + for i in range(len(b)): + b[i] = 0x20 + (i & 0x3f) +for _ in range(%d): + wr(b) +""" + + +write_test_script_verified = """ +import sys +try: + rd=__import__("pyb").USB_VCP(0).recv +except: + rd=sys.stdin.readinto +b=bytearray(%u) +for _ in range(%u): + n = rd(b) + fail = 0 + for i in range(n): + if b[i] != 32 + (i & 0x3f): + fail += 1 + if fail: + sys.stdout.write(b'ER%%05u' %% fail) + else: + sys.stdout.write(b'OK%%05u' %% n) +""" + +write_test_script_unverified = """ +import sys +try: + rd=__import__("pyb").USB_VCP(0).recv +except: + rd=sys.stdin.readinto +b=bytearray(%u) +for _ in range(%u): + n = rd(b) + if n != len(b): + sys.stdout.write(b'ER%%05u' %% n) + else: + sys.stdout.write(b'OK%%05u' %% n) +""" + + +class TestError(Exception): + pass + + +def drain_input(ser): + time.sleep(0.1) + while ser.inWaiting() > 0: + data = ser.read(ser.inWaiting()) + time.sleep(0.1) + + +def send_script(ser, script): + ser.write(b"\x03\x01\x04") # break, raw-repl, soft-reboot + drain_input(ser) + chunk_size = 32 + for i in range(0, len(script), chunk_size): + ser.write(script[i : i + chunk_size]) + time.sleep(0.01) + ser.write(b"\x04") # eof + ser.flush() + response = ser.read(2) + if response != b"OK": + response += ser.read(ser.inWaiting()) + raise TestError("could not send script", response) + + +def echo_test(ser_repl, ser_data): + global test_passed + + # Make the test data deterministic. + random.seed(0) + + # Set parameters for the test. + # Go just a bit above the size of a USB high-speed packet. + bytes_min = 1 + bytes_max = 520 + num_repeat = 1 + + # Load and run the write_test_script. + script = bytes(echo_test_script % (bytes_min, bytes_max, num_repeat), "ascii") + send_script(ser_repl, script) + + # A selection of printable bytes for echo data. + printable_bytes = list(range(48, 58)) + list(range(65, 91)) + list(range(97, 123)) + + # Write data to the device and record the echo'd data. + # Use a different selection of random printable characters for each + # echo, to make it easier to debug when the echo doesn't match. + num_errors = 0 + echo_results = [] + for num_bytes in range(bytes_min, bytes_max + 1): + print(f"DATA ECHO: {num_bytes} / {bytes_max}", end="\r") + for repeat in range(num_repeat): + rand_bytes = list(random.choice(printable_bytes) for _ in range(8)) + buf = bytes(random.choice(rand_bytes) for _ in range(num_bytes)) + ser_data.write(buf) + buf2 = ser_data.read(len(buf)) + match = buf == buf2 + num_errors += not match + echo_results.append((match, buf, buf2)) + if num_errors > 8: + # Stop early if there are too many errors. + break + ser_repl.write(b"\x03") + + # Print results. + if all(match for match, _, _ in echo_results): + print("DATA ECHO: OK for {}-{} bytes at a time".format(bytes_min, bytes_max)) + else: + test_passed = False + print("DATA ECHO: FAIL ") + for match, buf, buf2 in echo_results: + print(" sent", len(buf), buf) + if match: + print(" echo match") + else: + print(" echo", len(buf), buf2) + + +def read_test(ser_repl, ser_data, bufsize, nbuf): + global test_passed + + assert bufsize % 256 == 0 # for verify to work + + # how long to wait for data from device + # (if UART TX is also enabled then it can take 1.4s to send + # out a 16KB butter at 115200bps) + READ_TIMEOUT_S = 2 + + # Load and run the read_test_script. + script = bytes(read_test_script % (bufsize, nbuf), "ascii") + send_script(ser_repl, script) + + # Read from the device the type of data that it will send (BIN or TXT). + data_type = ser_data.read(3) + + # Read data from the device, check it is correct, and measure throughput. + n = 0 + last_byte = None + t_start = time.time() + remain = nbuf * bufsize + total_data = bytearray(remain) + while remain: + t0 = time.monotonic_ns() + while ser_data.inWaiting() == 0: + if time.monotonic_ns() - t0 > READ_TIMEOUT_S * 1e9: + # timeout waiting for data from device + break + time.sleep(0.0001) + if not ser_data.inWaiting(): + test_passed = False + print("ERROR: timeout waiting for data") + print(total_data[:n]) + return 0 + to_read = min(ser_data.inWaiting(), remain) + data = ser_data.read(to_read) + remain -= len(data) + print(f"{n} / {nbuf * bufsize}", end="\r") + total_data[n : n + len(data)] = data + n += len(data) + t_end = time.time() + for i in range(0, len(total_data)): + if data_type == b"BIN": + wanted = i & 0xFF + else: + wanted = 0x20 + (i & 0x3F) + if total_data[i] != wanted: + test_passed = False + print("ERROR: data mismatch:", i, wanted, total_data[i]) + ser_repl.write(b"\x03") # break + t = t_end - t_start + + # Print results. + print( + "DATA IN: bufsize=%u, read %u bytes in %.2f msec = %.2f kibytes/sec = %.2f MBits/sec" + % (bufsize, n, t * 1000, n / 1024 / t, n * 8 / 1000000 / t) + ) + + return n / t + + +def write_test(ser_repl, ser_data, bufsize, nbuf, verified): + global test_passed + + # Load and run the write_test_script. + if verified: + script = write_test_script_verified + else: + script = write_test_script_unverified + script = bytes(script % (bufsize, nbuf), "ascii") + send_script(ser_repl, script) + drain_input(ser_repl) + + # Write data to the device, check it is correct, and measure throughput. + n = 0 + t_start = time.time() + buf = bytearray(bufsize) + for i in range(len(buf)): + buf[i] = 32 + (i & 0x3F) # don't want to send ctrl chars! + for i in range(nbuf): + ser_data.write(buf) + n += len(buf) + print(f"{n} / {nbuf * bufsize}", end="\r") + response = ser_repl.read(7) + if response != b"OK%05u" % bufsize: + test_passed = False + print("ERROR: bad response, expecting OK%05u, got %r" % (bufsize, response)) + t_end = time.time() + ser_repl.write(b"\x03") # break + t = t_end - t_start + + # Print results. + print( + "DATA OUT: verify=%d, bufsize=%u, wrote %u bytes in %.2f msec = %.2f kibytes/sec = %.2f MBits/sec" + % (verified, bufsize, n, t * 1000, n / 1024 / t, n * 8 / 1000000 / t) + ) + + return n / t + + +def do_test(dev_repl, dev_data=None, time_per_subtest=1): + if dev_data is None: + print("REPL and data on", dev_repl) + ser_repl = serial.Serial(dev_repl, baudrate=115200, timeout=1) + ser_data = ser_repl + else: + print("REPL on", dev_repl) + print("data on", dev_data) + ser_repl = serial.Serial(dev_repl, baudrate=115200, timeout=1) + ser_data = serial.Serial(dev_data, baudrate=115200, timeout=1) + + # Do echo test first, and abort if it doesn't pass. + echo_test(ser_repl, ser_data) + if not test_passed: + return + + # Do read and write throughput test. + for test_func, test_args, bufsize in ( + (read_test, (), 256), + (write_test, (True,), 128), + (write_test, (False,), 128), + ): + nbuf = 128 + while bufsize <= 16384: + rate = test_func(ser_repl, ser_data, bufsize, nbuf, *test_args) + bufsize *= 2 + if rate: + # Adjust the amount of data based on the rate, to keep each subtest + # at around time_per_subtest seconds long. + nbuf = max(min(128, int(rate * time_per_subtest / bufsize)), 1) + + ser_repl.close() + ser_data.close() + + +def main(): + global test_passed + + cmd_parser = argparse.ArgumentParser( + description="Test performance and reliability of serial port communication.", + epilog=run_tests_module.test_instance_epilog, + formatter_class=argparse.RawTextHelpFormatter, + ) + cmd_parser.add_argument( + "-t", + "--test-instance", + default="a0", + help="MicroPython instance to test", + ) + cmd_parser.add_argument( + "--time-per-subtest", default="1", help="approximate time to take per subtest (in seconds)" + ) + args = cmd_parser.parse_args() + + dev_repl = run_tests_module.convert_device_shortcut_to_real_device(args.test_instance) + + test_passed = True + try: + do_test(dev_repl, None, float(args.time_per_subtest)) + except TestError as er: + test_passed = False + print("ERROR:", er) + + if not test_passed: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/tests/stress/bytecode_limit.py b/tests/stress/bytecode_limit.py index 948d7668da551..0a72b66fa05b6 100644 --- a/tests/stress/bytecode_limit.py +++ b/tests/stress/bytecode_limit.py @@ -1,19 +1,29 @@ # Test the limits of bytecode generation. +import sys + +# Tune the test parameters based on the target's bytecode generator. +if hasattr(sys.implementation, "_mpy"): + # Target can load .mpy files so generated bytecode uses 1 byte per qstr. + number_of_body_copies = (433, 432, 431, 399) +else: + # Target can't load .mpy files so generated bytecode uses 2 bytes per qstr. + number_of_body_copies = (401, 400, 399, 398) + body = " with f()()() as a:\n try:\n f()()()\n except Exception:\n pass\n" # Test overflow of jump offset. # Print results at the end in case an intermediate value of n fails with MemoryError. results = [] -for n in (433, 432, 431, 430): +for n in number_of_body_copies: try: exec("cond = 0\nif cond:\n" + body * n + "else:\n print('cond false')\n") - results.append((n, "ok")) + results.append("ok") except MemoryError: print("SKIP") raise SystemExit - except RuntimeError: - results.append((n, "RuntimeError")) + except RuntimeError as er: + results.append(repr(er)) print(results) # Test changing size of code info (source line/bytecode mapping) due to changing diff --git a/tests/stress/bytecode_limit.py.exp b/tests/stress/bytecode_limit.py.exp index cda52b1b97348..50511665f0076 100644 --- a/tests/stress/bytecode_limit.py.exp +++ b/tests/stress/bytecode_limit.py.exp @@ -1,4 +1,4 @@ cond false cond false -[(433, 'RuntimeError'), (432, 'RuntimeError'), (431, 'ok'), (430, 'ok')] +["RuntimeError('bytecode overflow',)", "RuntimeError('bytecode overflow',)", 'ok', 'ok'] [123] diff --git a/tests/stress/dict_copy.py b/tests/stress/dict_copy.py index 73d3a5b51d601..f9b742e20f716 100644 --- a/tests/stress/dict_copy.py +++ b/tests/stress/dict_copy.py @@ -1,6 +1,11 @@ # copying a large dictionary -a = {i: 2 * i for i in range(1000)} +try: + a = {i: 2 * i for i in range(1000)} +except MemoryError: + print("SKIP") + raise SystemExit + b = a.copy() for i in range(1000): print(i, b[i]) diff --git a/tests/stress/dict_create.py b/tests/stress/dict_create.py index e9db40a8e6c5b..91a83a12f9d85 100644 --- a/tests/stress/dict_create.py +++ b/tests/stress/dict_create.py @@ -3,6 +3,10 @@ d = {} x = 1 while x < 1000: - d[x] = x + try: + d[x] = x + except MemoryError: + print("SKIP") + raise SystemExit x += 1 print(d[500]) diff --git a/tests/stress/fun_call_limit.py b/tests/stress/fun_call_limit.py index b802aadd558c0..69f8aa5aec413 100644 --- a/tests/stress/fun_call_limit.py +++ b/tests/stress/fun_call_limit.py @@ -16,14 +16,16 @@ def test(n): # If the port has at least 32-bits then this test should pass. -print(test(29)) +print(test(28)) # This test should fail on all ports (overflows a small int). print(test(70)) -# Check that there is a correct transition to the limit of too many args before *args. +# 28 is the biggest number that will pass on a 32-bit port using object +# representation B, which has 1<<28 still fitting in a positive small int. reached_limit = False -for i in range(30, 70): +any_test_succeeded = False +for i in range(28, 70): result = test(i) if reached_limit: if result != "SyntaxError": @@ -34,3 +36,5 @@ def test(n): else: if result != i + 4: print("FAIL") + any_test_succeeded = True +assert any_test_succeeded # At least one iteration must have passed diff --git a/tests/stress/fun_call_limit.py.exp b/tests/stress/fun_call_limit.py.exp index 53d2b28043015..491abbfa8be72 100644 --- a/tests/stress/fun_call_limit.py.exp +++ b/tests/stress/fun_call_limit.py.exp @@ -1,2 +1,2 @@ -33 +32 SyntaxError diff --git a/tests/stress/qstr_limit.py b/tests/stress/qstr_limit.py index 08b10a039f509..c7bd437f3adc1 100644 --- a/tests/stress/qstr_limit.py +++ b/tests/stress/qstr_limit.py @@ -12,8 +12,8 @@ def make_id(n, base="a"): var = make_id(l) try: exec(var + "=1", g) - except RuntimeError: - print("RuntimeError", l) + except RuntimeError as er: + print("RuntimeError", er, l) continue print(var in g) @@ -26,16 +26,16 @@ def f(**k): for l in range(254, 259): try: exec("f({}=1)".format(make_id(l))) - except RuntimeError: - print("RuntimeError", l) + except RuntimeError as er: + print("RuntimeError", er, l) # type construction for l in range(254, 259): id = make_id(l) try: - print(type(id, (), {}).__name__) - except RuntimeError: - print("RuntimeError", l) + print(type(id, (), {})) + except RuntimeError as er: + print("RuntimeError", er, l) # hasattr, setattr, getattr @@ -48,28 +48,20 @@ class A: a = A() try: setattr(a, id, 123) - except RuntimeError: - print("RuntimeError", l) + except RuntimeError as er: + print("RuntimeError", er, l) try: print(hasattr(a, id), getattr(a, id)) - except RuntimeError: - print("RuntimeError", l) + except RuntimeError as er: + print("RuntimeError", er, l) # format with keys for l in range(254, 259): id = make_id(l) try: print(("{" + id + "}").format(**{id: l})) - except RuntimeError: - print("RuntimeError", l) - -# modulo format with keys -for l in range(254, 259): - id = make_id(l) - try: - print(("%(" + id + ")d") % {id: l}) - except RuntimeError: - print("RuntimeError", l) + except RuntimeError as er: + print("RuntimeError", er, l) # import module # (different OS's have different results so only run those that are consistent) @@ -78,8 +70,8 @@ class A: __import__(make_id(l)) except ImportError: print("ok", l) - except RuntimeError: - print("RuntimeError", l) + except RuntimeError as er: + print("RuntimeError", er, l) # import package for l in (100, 101, 102, 128, 129): @@ -87,5 +79,5 @@ class A: exec("import " + make_id(l) + "." + make_id(l, "A")) except ImportError: print("ok", l) - except RuntimeError: - print("RuntimeError", l) + except RuntimeError as er: + print("RuntimeError", er, l) diff --git a/tests/stress/qstr_limit.py.exp b/tests/stress/qstr_limit.py.exp index 455761bc71e4a..2349adf220f98 100644 --- a/tests/stress/qstr_limit.py.exp +++ b/tests/stress/qstr_limit.py.exp @@ -1,43 +1,38 @@ True True -RuntimeError 256 -RuntimeError 257 -RuntimeError 258 +RuntimeError name too long 256 +RuntimeError name too long 257 +RuntimeError name too long 258 {'abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrst': 1} {'abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstu': 1} -RuntimeError 256 -RuntimeError 257 -RuntimeError 258 -abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrst -abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstu -RuntimeError 256 -RuntimeError 257 -RuntimeError 258 +RuntimeError name too long 256 +RuntimeError name too long 257 +RuntimeError name too long 258 + + +RuntimeError name too long 256 +RuntimeError name too long 257 +RuntimeError name too long 258 True 123 True 123 -RuntimeError 256 -RuntimeError 256 -RuntimeError 257 -RuntimeError 257 -RuntimeError 258 -RuntimeError 258 +RuntimeError name too long 256 +RuntimeError name too long 256 +RuntimeError name too long 257 +RuntimeError name too long 257 +RuntimeError name too long 258 +RuntimeError name too long 258 254 255 -RuntimeError 256 -RuntimeError 257 -RuntimeError 258 -254 -255 -RuntimeError 256 -RuntimeError 257 -RuntimeError 258 +RuntimeError name too long 256 +RuntimeError name too long 257 +RuntimeError name too long 258 ok 100 ok 101 -RuntimeError 256 -RuntimeError 257 -RuntimeError 258 +RuntimeError name too long 256 +RuntimeError name too long 257 +RuntimeError name too long 258 ok 100 ok 101 ok 102 -RuntimeError 128 -RuntimeError 129 +RuntimeError name too long 128 +RuntimeError name too long 129 diff --git a/tests/stress/qstr_limit_str_modulo.py b/tests/stress/qstr_limit_str_modulo.py new file mode 100644 index 0000000000000..90b9f4364ec6f --- /dev/null +++ b/tests/stress/qstr_limit_str_modulo.py @@ -0,0 +1,21 @@ +# Test interning qstrs that go over the qstr length limit (255 bytes in default configuration). +# The tests here are specifically for str formatting with %. + +try: + "" % () +except TypeError: + print("SKIP") + raise SystemExit + + +def make_id(n, base="a"): + return "".join(chr(ord(base) + i % 26) for i in range(n)) + + +# modulo format with keys +for l in range(254, 259): + id = make_id(l) + try: + print(("%(" + id + ")d") % {id: l}) + except RuntimeError as er: + print("RuntimeError", er, l) diff --git a/tests/stress/qstr_limit_str_modulo.py.exp b/tests/stress/qstr_limit_str_modulo.py.exp new file mode 100644 index 0000000000000..3632c85bffea3 --- /dev/null +++ b/tests/stress/qstr_limit_str_modulo.py.exp @@ -0,0 +1,5 @@ +254 +255 +RuntimeError name too long 256 +RuntimeError name too long 257 +RuntimeError name too long 258 diff --git a/tests/stress/recursive_iternext.py b/tests/stress/recursive_iternext.py index bbc389e726237..c737f1e36d70a 100644 --- a/tests/stress/recursive_iternext.py +++ b/tests/stress/recursive_iternext.py @@ -1,4 +1,8 @@ # This tests that recursion with iternext doesn't lead to segfault. +# +# This test segfaults CPython, but that's not a bug as CPython doesn't enforce +# limits on C recursion - see +# https://github.com/python/cpython/issues/58218#issuecomment-1093570209 try: enumerate filter @@ -9,49 +13,25 @@ print("SKIP") raise SystemExit -# We need to pick an N that is large enough to hit the recursion -# limit, but not too large that we run out of heap memory. -try: - # large stack/heap, eg unix - [0] * 80000 - N = 5000 -except: - try: - # medium, eg pyboard - [0] * 10000 - N = 1000 - except: - # small, eg esp8266 - N = 100 - -try: - x = (1, 2) - for i in range(N): - x = enumerate(x) - tuple(x) -except RuntimeError: - print("RuntimeError") -try: +# Progressively build a bigger nested iterator structure (10 at a time for speed), +# and then try to evaluate it via tuple(x) which makes deep recursive function calls. +# +# Eventually this should raise a RuntimeError as MicroPython runs out of stack. +# It shouldn't ever raise a MemoryError, if it does then somehow MicroPython has +# run out of heap (for the nested structure) before running out of stack. +def recurse_iternext(nested_fn): x = (1, 2) - for i in range(N): - x = filter(None, x) - tuple(x) -except RuntimeError: - print("RuntimeError") + while True: + for _ in range(10): + x = nested_fn(x) + try: + tuple(x) + except RuntimeError: + print("RuntimeError") + break -try: - x = (1, 2) - for i in range(N): - x = map(max, x, ()) - tuple(x) -except RuntimeError: - print("RuntimeError") -try: - x = (1, 2) - for i in range(N): - x = zip(x) - tuple(x) -except RuntimeError: - print("RuntimeError") +# Test on various nested iterator structures +for nested_fn in [enumerate, lambda x: filter(None, x), lambda x: map(max, x, ()), zip]: + recurse_iternext(nested_fn) diff --git a/tests/target_wiring/EK_RA6M2.py b/tests/target_wiring/EK_RA6M2.py new file mode 100644 index 0000000000000..7d4a8cbbd6484 --- /dev/null +++ b/tests/target_wiring/EK_RA6M2.py @@ -0,0 +1,8 @@ +# Target wiring for EK_RA6M2. +# +# Connect: +# - P601 to P602 + +# UART(9) is on P602/P601. +uart_loopback_args = (9,) +uart_loopback_kwargs = {} diff --git a/tests/target_wiring/NUCLEO_WB55.py b/tests/target_wiring/NUCLEO_WB55.py new file mode 100644 index 0000000000000..ad7c120d3770e --- /dev/null +++ b/tests/target_wiring/NUCLEO_WB55.py @@ -0,0 +1,8 @@ +# Target wiring for NUCLEO_WB55. +# +# Connect: +# - PA2 to PA3 + +# LPUART(1) is on PA2/PA3. +uart_loopback_args = ("LP1",) +uart_loopback_kwargs = {} diff --git a/tests/target_wiring/PYBx.py b/tests/target_wiring/PYBx.py new file mode 100644 index 0000000000000..10ce520ef0af4 --- /dev/null +++ b/tests/target_wiring/PYBx.py @@ -0,0 +1,8 @@ +# Target wiring for PYBV10, PYBV11, PYBLITEV10, PYBD_SF2, PYBD_SF3, PYBD_SF6. +# +# Connect: +# - X1 to X2 + +# UART("XA") is on X1/X2 (usually UART(4) on PA0/PA1). +uart_loopback_args = ("XA",) +uart_loopback_kwargs = {} diff --git a/tests/target_wiring/ZEPHYR_NUCLEO_WB55RG.py b/tests/target_wiring/ZEPHYR_NUCLEO_WB55RG.py new file mode 100644 index 0000000000000..c7ca2ac26f794 --- /dev/null +++ b/tests/target_wiring/ZEPHYR_NUCLEO_WB55RG.py @@ -0,0 +1,7 @@ +# Target wiring for zephyr nucleo_wb55rg. +# +# Connect: +# - TX=PC0 to RX=PC1 + +uart_loopback_args = ("lpuart1",) +uart_loopback_kwargs = {} diff --git a/tests/target_wiring/alif.py b/tests/target_wiring/alif.py new file mode 100644 index 0000000000000..18f3cbe7e5e0d --- /dev/null +++ b/tests/target_wiring/alif.py @@ -0,0 +1,7 @@ +# Target wiring for general alif board. +# +# Connect: +# - UART1 TX and RX, usually P0_5 and P0_4 + +uart_loopback_args = (1,) +uart_loopback_kwargs = {} diff --git a/tests/target_wiring/esp32.py b/tests/target_wiring/esp32.py new file mode 100644 index 0000000000000..2767cd5acb2e8 --- /dev/null +++ b/tests/target_wiring/esp32.py @@ -0,0 +1,12 @@ +# Target wiring for general esp32 board. +# +# Connect: +# - GPIO4 to GPIO5 +# - GPIO12 to GPIO13 + +uart_loopback_args = (1,) +uart_loopback_kwargs = {"tx": 4, "rx": 5} + +encoder_loopback_id = 0 +encoder_loopback_out_pins = (4, 12) +encoder_loopback_in_pins = (5, 13) diff --git a/tests/target_wiring/mimxrt.py b/tests/target_wiring/mimxrt.py new file mode 100644 index 0000000000000..669e9095990db --- /dev/null +++ b/tests/target_wiring/mimxrt.py @@ -0,0 +1,7 @@ +# Target wiring for general mimxrt board. +# +# Connect: +# - UART1 TX and RX, usually D0 and D1 + +uart_loopback_args = (1,) +uart_loopback_kwargs = {} diff --git a/tests/target_wiring/nrf.py b/tests/target_wiring/nrf.py new file mode 100644 index 0000000000000..6979dd28ee595 --- /dev/null +++ b/tests/target_wiring/nrf.py @@ -0,0 +1,7 @@ +# Target wiring for general nrf board. +# +# Connect: +# - UART0 TX and RX + +uart_loopback_args = (0,) +uart_loopback_kwargs = {} diff --git a/tests/target_wiring/rp2.py b/tests/target_wiring/rp2.py new file mode 100644 index 0000000000000..cb0fa0d626339 --- /dev/null +++ b/tests/target_wiring/rp2.py @@ -0,0 +1,7 @@ +# Target wiring for general rp2 board. +# +# Connect: +# - GPIO0 to GPIO1 + +uart_loopback_args = (0,) +uart_loopback_kwargs = {"tx": "GPIO0", "rx": "GPIO1"} diff --git a/tests/target_wiring/samd.py b/tests/target_wiring/samd.py new file mode 100644 index 0000000000000..887c43a242f9b --- /dev/null +++ b/tests/target_wiring/samd.py @@ -0,0 +1,7 @@ +# Target wiring for general samd board. +# +# Connect: +# - D0 to D1 + +uart_loopback_args = () +uart_loopback_kwargs = {"tx": "D1", "rx": "D0"} diff --git a/tests/thread/disable_irq.py b/tests/thread/disable_irq.py index 3f1ac74f30877..596e3e477b9c8 100644 --- a/tests/thread/disable_irq.py +++ b/tests/thread/disable_irq.py @@ -1,8 +1,14 @@ # Ensure that disabling IRQs creates mutual exclusion between threads # (also tests nesting of disable_irq across threads) -import machine -import time -import _thread + +# CIRCUITPY-CHANGE: no machine +try: + import machine + import time + import _thread +except ImportError: + print("SKIP") + raise SystemExit if not hasattr(machine, "disable_irq"): print("SKIP") diff --git a/tests/thread/mutate_bytearray.py b/tests/thread/mutate_bytearray.py index b4664781a1522..7116d291cfeec 100644 --- a/tests/thread/mutate_bytearray.py +++ b/tests/thread/mutate_bytearray.py @@ -2,6 +2,7 @@ # # MIT license; Copyright (c) 2016 Damien P. George on behalf of Pycom Ltd +import time import _thread # the shared bytearray @@ -36,7 +37,7 @@ def th(n, lo, hi): # busy wait for threads to finish while n_finished < n_thread: - pass + time.sleep(0) # check bytearray has correct contents print(len(ba)) diff --git a/tests/thread/mutate_dict.py b/tests/thread/mutate_dict.py index 3777af66248c6..dd5f69e6c5d66 100644 --- a/tests/thread/mutate_dict.py +++ b/tests/thread/mutate_dict.py @@ -2,6 +2,7 @@ # # MIT license; Copyright (c) 2016 Damien P. George on behalf of Pycom Ltd +import time import _thread # the shared dict @@ -38,7 +39,7 @@ def th(n, lo, hi): # busy wait for threads to finish while n_finished < n_thread: - pass + time.sleep(0) # check dict has correct contents print(sorted(di.items())) diff --git a/tests/thread/mutate_instance.py b/tests/thread/mutate_instance.py index 306ad91c95cfa..63f7fb1e23df8 100644 --- a/tests/thread/mutate_instance.py +++ b/tests/thread/mutate_instance.py @@ -2,6 +2,7 @@ # # MIT license; Copyright (c) 2016 Damien P. George on behalf of Pycom Ltd +import time import _thread @@ -40,7 +41,7 @@ def th(n, lo, hi): # busy wait for threads to finish while n_finished < n_thread: - pass + time.sleep(0) # check user instance has correct contents print(user.a, user.b, user.c) diff --git a/tests/thread/mutate_list.py b/tests/thread/mutate_list.py index 6f1e8812541a0..d7398a2f1e0b4 100644 --- a/tests/thread/mutate_list.py +++ b/tests/thread/mutate_list.py @@ -2,6 +2,7 @@ # # MIT license; Copyright (c) 2016 Damien P. George on behalf of Pycom Ltd +import time import _thread # the shared list @@ -39,7 +40,7 @@ def th(n, lo, hi): # busy wait for threads to finish while n_finished < n_thread: - pass + time.sleep(0) # check list has correct contents li.sort() diff --git a/tests/thread/mutate_set.py b/tests/thread/mutate_set.py index 2d9a3e0ce9efd..7dcefa1d113f9 100644 --- a/tests/thread/mutate_set.py +++ b/tests/thread/mutate_set.py @@ -2,6 +2,7 @@ # # MIT license; Copyright (c) 2016 Damien P. George on behalf of Pycom Ltd +import time import _thread # the shared set @@ -33,7 +34,7 @@ def th(n, lo, hi): # busy wait for threads to finish while n_finished < n_thread: - pass + time.sleep(0) # check set has correct contents print(sorted(se)) diff --git a/tests/thread/stress_aes.py b/tests/thread/stress_aes.py index d8d0acd568a7a..ca25f8ad2fd57 100644 --- a/tests/thread/stress_aes.py +++ b/tests/thread/stress_aes.py @@ -277,7 +277,7 @@ def thread_entry(n_loop): n_thread = 2 n_loop = 2 else: - n_thread = 20 + n_thread = 10 n_loop = 5 for i in range(n_thread): _thread.start_new_thread(thread_entry, (n_loop,)) diff --git a/tests/thread/stress_recurse.py b/tests/thread/stress_recurse.py index 73b3a40f33daa..ec8b43fe8fc22 100644 --- a/tests/thread/stress_recurse.py +++ b/tests/thread/stress_recurse.py @@ -2,6 +2,7 @@ # # MIT license; Copyright (c) 2016 Damien P. George on behalf of Pycom Ltd +import time import _thread @@ -24,5 +25,5 @@ def thread_entry(): # busy wait for thread to finish while not finished: - pass + time.sleep(0) print("done") diff --git a/tests/thread/stress_schedule.py b/tests/thread/stress_schedule.py index 97876f0f77ca8..362d71aa12e38 100644 --- a/tests/thread/stress_schedule.py +++ b/tests/thread/stress_schedule.py @@ -27,6 +27,8 @@ def task(x): n += 1 +# This function must always use the bytecode emitter so it bounces the GIL when running. +@micropython.bytecode def thread(): while thread_run: try: @@ -46,7 +48,7 @@ def thread(): # Wait up to 10 seconds for 10000 tasks to be scheduled. t = time.ticks_ms() while n < _NUM_TASKS and time.ticks_diff(time.ticks_ms(), t) < _TIMEOUT_MS: - pass + time.sleep(0) # Stop all threads. thread_run = False diff --git a/tests/thread/thread_coop.py b/tests/thread/thread_coop.py index aefc4af074db5..85cda789c936e 100644 --- a/tests/thread/thread_coop.py +++ b/tests/thread/thread_coop.py @@ -7,6 +7,7 @@ import _thread import sys from time import ticks_ms, ticks_diff, sleep_ms +import micropython done = False @@ -21,6 +22,8 @@ MAX_DELTA = 100 +# This function must always use the bytecode emitter so the VM can bounce the GIL when running. +@micropython.bytecode def busy_thread(): while not done: pass diff --git a/tests/thread/thread_exc1.py b/tests/thread/thread_exc1.py index cd87740929103..cd6599983c141 100644 --- a/tests/thread/thread_exc1.py +++ b/tests/thread/thread_exc1.py @@ -2,6 +2,7 @@ # # MIT license; Copyright (c) 2016 Damien P. George on behalf of Pycom Ltd +import time import _thread @@ -34,5 +35,5 @@ def thread_entry(): # busy wait for threads to finish while n_finished < n_thread: - pass + time.sleep(0) print("done") diff --git a/tests/thread/thread_exc2.py.native.exp b/tests/thread/thread_exc2.py.native.exp new file mode 100644 index 0000000000000..9b2e715ef8dfc --- /dev/null +++ b/tests/thread/thread_exc2.py.native.exp @@ -0,0 +1,3 @@ +Unhandled exception in thread started by +ValueError: +done diff --git a/tests/thread/thread_gc1.py b/tests/thread/thread_gc1.py index b36ea9d4c8421..45c17cc17bed3 100644 --- a/tests/thread/thread_gc1.py +++ b/tests/thread/thread_gc1.py @@ -2,6 +2,7 @@ # # MIT license; Copyright (c) 2016 Damien P. George on behalf of Pycom Ltd +import time import gc import _thread @@ -44,6 +45,6 @@ def thread_entry(n): # busy wait for threads to finish while n_finished < n_thread: - pass + time.sleep(0) print(n_correct == n_finished) diff --git a/tests/thread/thread_ident1.py b/tests/thread/thread_ident1.py index 2a3732eff53dc..08cfd3eb36e8a 100644 --- a/tests/thread/thread_ident1.py +++ b/tests/thread/thread_ident1.py @@ -2,6 +2,7 @@ # # MIT license; Copyright (c) 2016 Damien P. George on behalf of Pycom Ltd +import time import _thread @@ -27,6 +28,6 @@ def thread_entry(): new_tid = _thread.start_new_thread(thread_entry, ()) while not finished: - pass + time.sleep(0) print("done", type(new_tid) == int, new_tid == tid) diff --git a/tests/thread/thread_lock3.py b/tests/thread/thread_lock3.py index a927dc6829e15..c5acfa21b7dc6 100644 --- a/tests/thread/thread_lock3.py +++ b/tests/thread/thread_lock3.py @@ -2,6 +2,7 @@ # # MIT license; Copyright (c) 2016 Damien P. George on behalf of Pycom Ltd +import time import _thread lock = _thread.allocate_lock() @@ -26,4 +27,4 @@ def thread_entry(idx): # busy wait for threads to finish while n_finished < n_thread: - pass + time.sleep(0) diff --git a/tests/thread/thread_lock4.py b/tests/thread/thread_lock4_intbig.py similarity index 100% rename from tests/thread/thread_lock4.py rename to tests/thread/thread_lock4_intbig.py diff --git a/tests/thread/thread_shared1.py b/tests/thread/thread_shared1.py index 251e26fae6ca3..c2e33abcec7ee 100644 --- a/tests/thread/thread_shared1.py +++ b/tests/thread/thread_shared1.py @@ -2,6 +2,7 @@ # # MIT license; Copyright (c) 2016 Damien P. George on behalf of Pycom Ltd +import time import _thread @@ -40,5 +41,5 @@ def thread_entry(n, tup): # busy wait for threads to finish while n_finished < n_thread: - pass + time.sleep(0) print(tup) diff --git a/tests/thread/thread_shared2.py b/tests/thread/thread_shared2.py index a1223c2b94f40..4ce9057ca017e 100644 --- a/tests/thread/thread_shared2.py +++ b/tests/thread/thread_shared2.py @@ -3,6 +3,7 @@ # # MIT license; Copyright (c) 2016 Damien P. George on behalf of Pycom Ltd +import time import _thread @@ -31,5 +32,5 @@ def thread_entry(n, lst, idx): # busy wait for threads to finish while n_finished < n_thread: - pass + time.sleep(0) print(lst) diff --git a/tests/thread/thread_stacksize1.py b/tests/thread/thread_stacksize1.py index 140d165cb3497..75e1da9642f95 100644 --- a/tests/thread/thread_stacksize1.py +++ b/tests/thread/thread_stacksize1.py @@ -3,6 +3,7 @@ # MIT license; Copyright (c) 2016 Damien P. George on behalf of Pycom Ltd import sys +import time import _thread # different implementations have different minimum sizes @@ -51,5 +52,5 @@ def thread_entry(): # busy wait for threads to finish while n_finished < n_thread: - pass + time.sleep(0) print("done") diff --git a/tests/thread/thread_stdin.py b/tests/thread/thread_stdin.py index a469933f19b55..498b0a3a27022 100644 --- a/tests/thread/thread_stdin.py +++ b/tests/thread/thread_stdin.py @@ -5,6 +5,7 @@ # This is a regression test for https://github.com/micropython/micropython/issues/15230 # on rp2, but doubles as a general property to test across all ports. import sys +import time import _thread try: @@ -38,7 +39,7 @@ def is_done(self): # have run yet. The actual delay is <20ms but spinning here instead of # sleep(0.1) means the test can run on MP builds without float support. while not thread_waiter.is_done(): - pass + time.sleep(0) # The background thread should have completed its wait by now. print(thread_waiter.is_done()) diff --git a/tests/unix/extra_coverage.py b/tests/unix/extra_coverage.py index 0ea8f7886bfff..ec10a44ff39b3 100644 --- a/tests/unix/extra_coverage.py +++ b/tests/unix/extra_coverage.py @@ -6,6 +6,16 @@ import errno import io +import uctypes + +# create an int-like variable used for coverage of `mp_obj_get_ll` +buf = bytearray(b"\xde\xad\xbe\xef") +struct = uctypes.struct( + uctypes.addressof(buf), + {"f32": uctypes.UINT32 | 0}, + uctypes.BIG_ENDIAN, +) +deadbeef = struct.f32 data = extra_coverage() @@ -23,6 +33,7 @@ print(stream.read(1)) # read 1 byte encounters non-blocking error print(stream.readline()) # readline encounters non-blocking error print(stream.readinto(bytearray(10))) # readinto encounters non-blocking error +print(stream.readinto1(bytearray(10))) # readinto1 encounters non-blocking error print(stream.write(b"1")) # write encounters non-blocking error print(stream.write1(b"1")) # write1 encounters non-blocking error stream.set_buf(b"123") @@ -38,6 +49,28 @@ stream.set_error(0) print(stream.ioctl(0, bytearray(10))) # successful ioctl call +print("# stream.readinto") + +# stream.readinto will read 3 bytes then try to read more to fill the buffer, +# but on the second attempt will encounter EIO and should raise that error. +stream.set_error(errno.EIO) +stream.set_buf(b"123") +buf = bytearray(4) +try: + stream.readinto(buf) +except OSError as er: + print("OSError", er.errno == errno.EIO) +print(buf) + +# stream.readinto1 will read 3 bytes then should return them immediately, and +# not encounter the EIO. +stream.set_error(errno.EIO) +stream.set_buf(b"123") +buf = bytearray(4) +print(stream.readinto1(buf), buf) + +print("# stream textio") + stream2 = data[3] # is textio print(stream2.read(1)) # read 1 byte encounters non-blocking error with textio stream @@ -92,7 +125,7 @@ print(returns_NULL()) -# test for freeze_mpy +# test for freeze_mpy (importing prints several lines) import frozentest print(frozentest.__file__) diff --git a/tests/unix/extra_coverage.py.exp b/tests/unix/extra_coverage.py.exp index 4cd8b8666d270..cdb1145569497 100644 --- a/tests/unix/extra_coverage.py.exp +++ b/tests/unix/extra_coverage.py.exp @@ -2,25 +2,41 @@ -123 +123 123 -0123 123 -123 -1ABCDEF +123f +123F +7fffffffffffffff +7FFFFFFFFFFFFFFF +18446744073709551615 +789f +789F ab abc ' abc' ' True' 'Tru' false true (null) -2147483648 2147483648 -80000000 -80000000 +8000000f +8000000F abc % +.a . +<%> + + +<43690> +<43690> +<43690> + +<1000.000000> + +<9223372036854775807> # GC -0x0 -0x0 +0 +0 # GC part 2 pass # tracked allocation -m_tracked_head = 0x0 +m_tracked_head = 0 0 1 1 1 2 1 @@ -37,7 +53,7 @@ m_tracked_head = 0x0 5 1 6 1 7 1 -m_tracked_head = 0x0 +m_tracked_head = 0 # vstr tests sts @@ -91,6 +107,16 @@ data 12345 6 -1 +0 +1 +0 +0.000000 +deadbeef +c0ffee777c0ffee +deadbeef +0deadbeef +c0ffee +000c0ffee # runtime utils TypeError: unsupported type for __abs__: 'str' TypeError: unsupported types for __divmod__: 'str', 'str' @@ -99,12 +125,10 @@ TypeError: unsupported types for __divmod__: 'str', 'str' 2 OverflowError: overflow converting long int to machine word OverflowError: overflow converting long int to machine word +TypeError: can't convert NoneType to int +TypeError: can't convert NoneType to int ValueError: Warning: test -# format float -? -+1e+00 -+1e+00 # binary 123 456 @@ -124,6 +148,13 @@ unlocked KeyboardInterrupt: KeyboardInterrupt: 10 +loop +scheduled function +loop +scheduled function +loop +scheduled function +scheduled function # ringbuf 99 0 98 1 @@ -185,11 +216,17 @@ None None None None +None b'123' b'123' b'123' OSError 0 +# stream.readinto +OSError True +bytearray(b'123\x00') +3 bytearray(b'123\x00') +# stream textio None None cpp None @@ -215,7 +252,7 @@ b'\x00\xff' frzmpy4 1 frzmpy4 2 NULL -uPy +interned a long string that is not interned a string that has unicode αβγ chars b'bytes 1234\x01' diff --git a/tests/unix/ffi_float2.py b/tests/unix/ffi_float2.py index bbed6966627e1..eac6cd106cf43 100644 --- a/tests/unix/ffi_float2.py +++ b/tests/unix/ffi_float2.py @@ -29,4 +29,5 @@ def ffi_open(names): for fun in (tgammaf,): for val in (0.5, 1, 1.0, 1.5, 4, 4.0): - print("%.6f" % fun(val)) + # limit to 5 decimals in order to pass with REPR_C with FORMAT_IMPL_BASIC + print("%.5f" % fun(val)) diff --git a/tests/unix/ffi_float2.py.exp b/tests/unix/ffi_float2.py.exp index 58fc6a01acb07..4c750e223a3a6 100644 --- a/tests/unix/ffi_float2.py.exp +++ b/tests/unix/ffi_float2.py.exp @@ -1,6 +1,6 @@ -1.772454 -1.000000 -1.000000 -0.886227 -6.000000 -6.000000 +1.77245 +1.00000 +1.00000 +0.88623 +6.00000 +6.00000 diff --git a/tests/unix/mod_os.py b/tests/unix/mod_os.py index f69fa45b2b22a..468f6badd1e75 100644 --- a/tests/unix/mod_os.py +++ b/tests/unix/mod_os.py @@ -1,6 +1,9 @@ # This module is not entirely compatible with CPython import os +if not hasattr(os, "getenv"): + print("SKIP") + raise SystemExit os.putenv("TEST_VARIABLE", "TEST_VALUE") diff --git a/tests/unix/time_mktime_localtime.py b/tests/unix/time_mktime_localtime.py index d1c03c103d704..df5d6cda690a2 100644 --- a/tests/unix/time_mktime_localtime.py +++ b/tests/unix/time_mktime_localtime.py @@ -1,4 +1,8 @@ -import time +try: + import time +except ImportError: + print("SKIP") + raise SystemExit DAYS_PER_MONTH = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] diff --git a/tools/analyze_heap_dump.py b/tools/analyze_heap_dump.py index ac1bb0c8ce592..a7be95db7d9f3 100755 --- a/tools/analyze_heap_dump.py +++ b/tools/analyze_heap_dump.py @@ -82,7 +82,7 @@ help="Draw the ownership graph of blocks on the heap", ) @click.option("--analyze-snapshots", default="last", type=click.Choice(["all", "last"])) -def do_all_the_things( # noqa: C901: too complex +def do_all_the_things( # noqa: C901 too complex ram_filename, bin_filename, map_filename, diff --git a/tools/boardgen.py b/tools/boardgen.py index 39bedf71cd0c8..3723e7ce31b83 100644 --- a/tools/boardgen.py +++ b/tools/boardgen.py @@ -108,6 +108,10 @@ def add_board_pin_name(self, board_pin_name, hidden=False): ) ) + # Iterate over board pin names in consistent sorted order. + def board_pin_names(self): + return sorted(self._board_pin_names, key=lambda x: x[0]) + # Override this to handle an af specified in af.csv. def add_af(self, af_idx, af_name, af): raise NotImplementedError @@ -295,7 +299,7 @@ def print_board_locals_dict(self, out_source): file=out_source, ) for pin in self.available_pins(): - for board_pin_name, board_hidden in pin._board_pin_names: + for board_pin_name, board_hidden in pin.board_pin_names(): if board_hidden: # Don't include hidden pins in Pins.board. continue @@ -389,7 +393,7 @@ def print_defines(self, out_header, cpu=True, board=True): # #define pin_BOARDNAME (pin_CPUNAME) if board: - for board_pin_name, _board_hidden in pin._board_pin_names: + for board_pin_name, _board_hidden in pin.board_pin_names(): # Note: Hidden board pins are still available to C via the macro. # Note: The RHS isn't wrapped in (), which is necessary to make the # STATIC_AF_ macro work on STM32. diff --git a/tools/cc1 b/tools/cc1 index 827d5886a04f5..aa2534f01e7bb 100755 --- a/tools/cc1 +++ b/tools/cc1 @@ -23,8 +23,8 @@ import re # TODO somehow make them externally configurable # this is the path to the true C compiler -cc1_path = '/usr/lib/gcc/x86_64-unknown-linux-gnu/5.3.0/cc1' -#cc1_path = '/usr/lib/gcc/arm-none-eabi/5.3.0/cc1' +cc1_path = "/usr/lib/gcc/x86_64-unknown-linux-gnu/5.3.0/cc1" +# cc1_path = '/usr/lib/gcc/arm-none-eabi/5.3.0/cc1' # this must be the same as MICROPY_QSTR_BYTES_IN_HASH bytes_in_qstr_hash = 2 @@ -41,11 +41,16 @@ print_debug = False ################################################################################ # precompile regexs -re_preproc_line = re.compile(r'# [0-9]+ ') -re_map_entry = re.compile(r'\{.+?\(MP_QSTR_([A-Za-z0-9_]+)\).+\},') -re_mp_obj_dict_t = re.compile(r'(?P(static )?const mp_obj_dict_t (?P[a-z0-9_]+) = \{ \.base = \{&mp_type_dict\}, \.map = \{ \.all_keys_are_qstrs = 1, \.is_fixed = 1, \.is_ordered = )1(?P, \.used = .+ };)$') -re_mp_map_t = re.compile(r'(?P(static )?const mp_map_t (?P[a-z0-9_]+) = \{ \.all_keys_are_qstrs = 1, \.is_fixed = 1, \.is_ordered = )1(?P, \.used = .+ };)$') -re_mp_rom_map_elem_t = re.compile(r'static const mp_rom_map_elem_t [a-z_0-9]+\[\] = {$') +re_preproc_line = re.compile(r"# [0-9]+ ") +re_map_entry = re.compile(r"\{.+?\(MP_QSTR_([A-Za-z0-9_]+)\).+\},") +re_mp_obj_dict_t = re.compile( + r"(?P(static )?const mp_obj_dict_t (?P[a-z0-9_]+) = \{ \.base = \{&mp_type_dict\}, \.map = \{ \.all_keys_are_qstrs = 1, \.is_fixed = 1, \.is_ordered = )1(?P, \.used = .+ };)$" +) +re_mp_map_t = re.compile( + r"(?P(static )?const mp_map_t (?P[a-z0-9_]+) = \{ \.all_keys_are_qstrs = 1, \.is_fixed = 1, \.is_ordered = )1(?P, \.used = .+ };)$" +) +re_mp_rom_map_elem_t = re.compile(r"static const mp_rom_map_elem_t [a-z_0-9]+\[\] = {$") + # this must match the equivalent function in qstr.c def compute_hash(qstr): @@ -55,18 +60,19 @@ def compute_hash(qstr): # Make sure that valid hash is never zero, zero means "hash not computed" return (hash & ((1 << (8 * bytes_in_qstr_hash)) - 1)) or 1 + # this algo must match the equivalent in map.c def hash_insert(map, key, value): hash = compute_hash(key) pos = hash % len(map) start_pos = pos if print_debug: - print(' insert %s: start at %u/%u -- ' % (key, pos, len(map)), end='') + print(" insert %s: start at %u/%u -- " % (key, pos, len(map)), end="") while True: if map[pos] is None: # found empty slot, so key is not in table if print_debug: - print('put at %u' % pos) + print("put at %u" % pos) map[pos] = (key, value) return else: @@ -76,6 +82,7 @@ def hash_insert(map, key, value): pos = (pos + 1) % len(map) assert pos != start_pos + def hash_find(map, key): hash = compute_hash(key) pos = hash % len(map) @@ -92,6 +99,7 @@ def hash_find(map, key): if pos == start_pos: return attempts, None + def process_map_table(file, line, output): output.append(line) @@ -101,7 +109,7 @@ def process_map_table(file, line, output): while True: line = file.readline() if len(line) == 0: - print('unexpected end of input') + print("unexpected end of input") sys.exit(1) line = line.strip() if len(line) == 0: @@ -110,38 +118,38 @@ def process_map_table(file, line, output): if re_preproc_line.match(line): # preprocessor line number comment continue - if line == '};': + if line == "};": # end of table (we assume it appears on a single line) break table_contents.append(line) # make combined string of entries - entries_str = ''.join(table_contents) + entries_str = "".join(table_contents) # split into individual entries entries = [] while entries_str: # look for single entry, by matching nested braces match = None - if entries_str[0] == '{': + if entries_str[0] == "{": nested_braces = 0 for i in range(len(entries_str)): - if entries_str[i] == '{': + if entries_str[i] == "{": nested_braces += 1 - elif entries_str[i] == '}': + elif entries_str[i] == "}": nested_braces -= 1 if nested_braces == 0: - match = re_map_entry.match(entries_str[:i + 2]) + match = re_map_entry.match(entries_str[: i + 2]) break if not match: - print('unknown line in table:', entries_str) + print("unknown line in table:", entries_str) sys.exit(1) # extract single entry line = match.group(0) qstr = match.group(1) - entries_str = entries_str[len(line):].lstrip() + entries_str = entries_str[len(line) :].lstrip() # add the qstr and the whole line to list of all entries entries.append((qstr, line)) @@ -164,28 +172,28 @@ def process_map_table(file, line, output): attempts, line = hash_find(map, qstr) assert line is not None if print_debug: - print(' %s lookup took %u attempts' % (qstr, attempts)) + print(" %s lookup took %u attempts" % (qstr, attempts)) total_attempts += attempts - if len(entries): + if entries: stats = len(map), len(entries) / len(map), total_attempts / len(entries) else: stats = 0, 0, 0 if print_debug: - print(' table stats: size=%d, load=%.2f, avg_lookups=%.1f' % stats) + print(" table stats: size=%d, load=%.2f, avg_lookups=%.1f" % stats) # output hash table for row in map: if row is None: - output.append('{ 0, 0 },\n') + output.append("{ 0, 0 },\n") else: - output.append(row[1] + '\n') - output.append('};\n') + output.append(row[1] + "\n") + output.append("};\n") # skip to next non-blank line while True: line = file.readline() if len(line) == 0: - print('unexpected end of input') + print("unexpected end of input") sys.exit(1) line = line.strip() if len(line) == 0: @@ -197,19 +205,20 @@ def process_map_table(file, line, output): if match is None: match = re_mp_map_t.match(line) if match is None: - print('expecting mp_obj_dict_t or mp_map_t definition') + print("expecting mp_obj_dict_t or mp_map_t definition") print(output[0]) print(line) sys.exit(1) - line = match.group('head') + '0' + match.group('tail') + '\n' + line = match.group("head") + "0" + match.group("tail") + "\n" output.append(line) - return (match.group('id'),) + stats + return (match.group("id"),) + stats + def process_file(filename): output = [] file_changed = False - with open(filename, 'rt') as f: + with open(filename, "rt") as f: while True: line = f.readline() if not line: @@ -218,39 +227,41 @@ def process_file(filename): file_changed = True stats = process_map_table(f, line, output) if print_stats: - print(' [%s: size=%d, load=%.2f, avg_lookups=%.1f]' % stats) + print(" [%s: size=%d, load=%.2f, avg_lookups=%.1f]" % stats) else: output.append(line) if file_changed: if print_debug: - print(' modifying static maps in', output[0].strip()) - with open(filename, 'wt') as f: + print(" modifying static maps in", output[0].strip()) + with open(filename, "wt") as f: for line in output: f.write(line) + def main(): # run actual C compiler # need to quote args that have special characters in them def quote(s): - if s.find('<') != -1 or s.find('>') != -1: + if s.find("<") != -1 or s.find(">") != -1: return "'" + s + "'" else: return s - ret = os.system(cc1_path + ' ' + ' '.join(quote(s) for s in sys.argv[1:])) + + ret = os.system(cc1_path + " " + " ".join(quote(s) for s in sys.argv[1:])) if ret != 0: - ret = (ret & 0x7f) or 127 # make it in range 0-127, but non-zero + ret = (ret & 0x7F) or 127 # make it in range 0-127, but non-zero sys.exit(ret) - if sys.argv[1] == '-E': + if sys.argv[1] == "-E": # CPP has been run, now do our processing stage for i, arg in enumerate(sys.argv): - if arg == '-o': + if arg == "-o": return process_file(sys.argv[i + 1]) print('%s: could not find "-o" option' % (sys.argv[0],)) sys.exit(1) - elif sys.argv[1] == '-fpreprocessed': + elif sys.argv[1] == "-fpreprocessed": # compiler has been run, nothing more to do return else: @@ -258,5 +269,6 @@ def main(): print('%s: unknown first option "%s"' % (sys.argv[0], sys.argv[1])) sys.exit(1) -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/tools/chart_code_size.py b/tools/chart_code_size.py index cd62074fd0d83..f3d9c5bdc40f5 100644 --- a/tools/chart_code_size.py +++ b/tools/chart_code_size.py @@ -24,7 +24,7 @@ def parse_hex(h): @click.command() @click.argument("elf_filename") -def do_all_the_things(elf_filename): # noqa: C901: too complex +def do_all_the_things(elf_filename): # noqa: C901 too complex symbol = None last_address = 0 all_symbols = {} diff --git a/tools/ci.sh b/tools/ci.sh index cfc9754837f76..132fbd8f81cf3 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -9,15 +9,20 @@ fi # Ensure known OPEN_MAX (NO_FILES) limit. ulimit -n 1024 +# Fail on some things which are warnings otherwise +export MICROPY_MAINTAINER_BUILD=1 + ######################################################################################## # general helper functions function ci_gcc_arm_setup { + sudo apt-get update sudo apt-get install gcc-arm-none-eabi libnewlib-arm-none-eabi arm-none-eabi-gcc --version } function ci_gcc_riscv_setup { + sudo apt-get update sudo apt-get install gcc-riscv64-unknown-elf picolibc-riscv64-unknown-elf riscv64-unknown-elf-gcc --version } @@ -35,6 +40,7 @@ function ci_picotool_setup { # c code formatting function ci_c_code_formatting_setup { + sudo apt-get update sudo apt-get install uncrustify uncrustify --version } @@ -74,42 +80,69 @@ function ci_code_size_setup { ci_picotool_setup } +function _ci_is_git_merge { + [[ $(git log -1 --format=%P "$1" | wc -w) > 1 ]] +} + function ci_code_size_build { # check the following ports for the change in their code size - PORTS_TO_CHECK=bmusxpdv + # Override the list by setting PORTS_TO_CHECK in the environment before invoking ci. + : ${PORTS_TO_CHECK:=bmusxpdv} + SUBMODULES="lib/asf4 lib/berkeley-db-1.xx lib/btstack lib/cyw43-driver lib/lwip lib/mbedtls lib/micropython-lib lib/nxp_driver lib/pico-sdk lib/stm32lib lib/tinyusb" # Default GitHub pull request sets HEAD to a generated merge commit # between PR branch (HEAD^2) and base branch (i.e. master) (HEAD^1). # # We want to compare this generated commit with the base branch, to see what - # the code size impact would be if we merged this PR. - REFERENCE=$(git rev-parse --short HEAD^1) - COMPARISON=$(git rev-parse --short HEAD) + # the code size impact would be if we merged this PR. During CI we are at a merge commit, + # so this tests the merged PR against its merge base. + # Override the refs by setting REFERENCE and/or COMPARISON in the environment before invoking ci. + : ${COMPARISON:=$(git rev-parse --short HEAD)} + : ${REFERENCE:=$(git rev-parse --short ${COMPARISON}^1)} echo "Comparing sizes of reference ${REFERENCE} to ${COMPARISON}..." git log --oneline $REFERENCE..$COMPARISON - function code_size_build_step { - COMMIT=$1 - OUTFILE=$2 - IGNORE_ERRORS=$3 - - echo "Building ${COMMIT}..." - git checkout --detach $COMMIT - git submodule update --init $SUBMODULES - git show -s - tools/metrics.py clean $PORTS_TO_CHECK - tools/metrics.py build $PORTS_TO_CHECK | tee $OUTFILE || $IGNORE_ERRORS - } - - # build reference, save to size0 - # ignore any errors with this build, in case master is failing - code_size_build_step $REFERENCE ~/size0 true - # build PR/branch, save to size1 - code_size_build_step $COMPARISON ~/size1 false - - unset -f code_size_build_step + OLD_BRANCH="$(git rev-parse --abbrev-ref HEAD)" + + ( # Execute in a subshell so the trap & code_size_build_step doesn't leak + function code_size_build_step { + if [ ! -z "$OLD_BRANCH" ]; then + trap 'git checkout "$OLD_BRANCH"' RETURN EXIT ERR + fi + + COMMIT=$1 + OUTFILE=$2 + IGNORE_ERRORS=$3 + + git checkout --detach $COMMIT + git submodule update --init $SUBMODULES + git show -s + tools/metrics.py clean "$PORTS_TO_CHECK" + # Allow errors from tools/metrics.py to propagate out of the pipe below. + set -o pipefail + tools/metrics.py build "$PORTS_TO_CHECK" | tee -a $OUTFILE || $IGNORE_ERRORS + return $? + } + + # build reference, save to size0 + # ignore any errors with this build, in case master is failing + echo "BUILDING $(git log --format='%s [%h]' -1 ${REFERENCE})" > ~/size0 + code_size_build_step $REFERENCE ~/size0 true + # build PR/branch, save to size1 + if _ci_is_git_merge "$COMPARISON"; then + echo "BUILDING $(git log --oneline -1 --format='%s [merge of %h]' ${COMPARISON}^2)" + else + echo "BUILDING $(git log --oneline -1 --formta='%s [%h]' ${COMPARISON})" + fi > ~/size1 + code_size_build_step $COMPARISON ~/size1 false + ) +} + +function ci_code_size_report { + # Allow errors from tools/metrics.py to propagate out of the pipe above. + (set -o pipefail; tools/metrics.py diff ~/size0 ~/size1 | tee diff) } ######################################################################################## @@ -117,15 +150,12 @@ function ci_code_size_build { function ci_mpy_format_setup { sudo apt-get update - sudo apt-get install python2.7 sudo pip3 install pyelftools - python2.7 --version python3 --version } function ci_mpy_format_test { # Test mpy-tool.py dump feature on bytecode - python2.7 ./tools/mpy-tool.py -xd tests/frozen/frozentest.mpy python3 ./tools/mpy-tool.py -xd tests/frozen/frozentest.mpy # Build MicroPython @@ -143,6 +173,15 @@ function ci_mpy_format_test { $micropython ./tools/mpy-tool.py -x -d examples/natmod/features1/features1.mpy } +function ci_mpy_cross_debug_emitter { + make ${MAKEOPTS} -C mpy-cross + mpy_cross=./mpy-cross/build/mpy-cross + + # Make sure the debug emitter does not crash or fail for simple files + $mpy_cross -X emit=native -march=debug ./tests/basics/0prelim.py | \ + grep -E "ENTRY|EXIT" | wc -l | grep "^2$" +} + ######################################################################################## # ports/cc3200 @@ -158,14 +197,17 @@ function ci_cc3200_build { ######################################################################################## # ports/esp32 -# GitHub tag of ESP-IDF to use for CI (note: must be a tag or a branch) -IDF_VER=v5.2.2 +# GitHub tag of ESP-IDF to use for CI, extracted from the esp32 dependency lockfile +# This should end up as a tag name like vX.Y.Z +# (note: This hacky parsing can be replaced with 'yq' once Ubuntu >=24.04 is in use) +IDF_VER=v$(grep -A10 "idf:" ports/esp32/lockfiles/dependencies.lock.esp32 | grep "version:" | head -n1 | sed -E 's/ +version: //') PYTHON=$(command -v python3 2> /dev/null) PYTHON_VER=$(${PYTHON:-python} --version | cut -d' ' -f2) export IDF_CCACHE_ENABLE=1 function ci_esp32_idf_setup { + echo "Using ESP-IDF version $IDF_VER" git clone --depth 1 --branch $IDF_VER https://github.com/espressif/esp-idf.git # doing a treeless clone isn't quite as good as --shallow-submodules, but it # is smaller than full clones and works when the submodule commit isn't a head. @@ -204,13 +246,28 @@ function ci_esp32_build_s3_c3 { make ${MAKEOPTS} -C ports/esp32 BOARD=ESP32_GENERIC_C3 } +function ci_esp32_build_c2_c5_c6 { + ci_esp32_build_common + + make ${MAKEOPTS} -C ports/esp32 BOARD=ESP32_GENERIC_C2 + make ${MAKEOPTS} -C ports/esp32 BOARD=ESP32_GENERIC_C5 + make ${MAKEOPTS} -C ports/esp32 BOARD=ESP32_GENERIC_C6 +} + +function ci_esp32_build_p4 { + ci_esp32_build_common + + make ${MAKEOPTS} -C ports/esp32 BOARD=ESP32_GENERIC_P4 + make ${MAKEOPTS} -C ports/esp32 BOARD=ESP32_GENERIC_P4 BOARD_VARIANT=C6_WIFI +} + ######################################################################################## # ports/esp8266 function ci_esp8266_setup { sudo pip3 install pyserial esptool==3.3.1 pyelftools ar - wget https://github.com/jepler/esp-open-sdk/releases/download/2018-06-10/xtensa-lx106-elf-standalone.tar.gz - zcat xtensa-lx106-elf-standalone.tar.gz | tar x + wget https://micropython.org/resources/xtensa-lx106-elf-standalone.tar.gz + (set -o pipefail; zcat xtensa-lx106-elf-standalone.tar.gz | tar x) # Remove this esptool.py so pip version is used instead rm xtensa-lx106-elf/bin/esptool.py } @@ -317,17 +374,52 @@ function ci_qemu_setup_rv32 { qemu-system-riscv32 --version } -function ci_qemu_build_arm { +function ci_qemu_setup_rv64 { + ci_gcc_riscv_setup + sudo apt-get update + sudo apt-get install qemu-system + qemu-system-riscv64 --version +} + +function ci_qemu_build_arm_prepare { make ${MAKEOPTS} -C mpy-cross make ${MAKEOPTS} -C ports/qemu submodules +} + +function ci_qemu_build_arm_bigendian { + ci_qemu_build_arm_prepare make ${MAKEOPTS} -C ports/qemu CFLAGS_EXTRA=-DMP_ENDIANNESS_BIG=1 - make ${MAKEOPTS} -C ports/qemu clean - make ${MAKEOPTS} -C ports/qemu test_full +} + +function ci_qemu_build_arm_sabrelite { + ci_qemu_build_arm_prepare make ${MAKEOPTS} -C ports/qemu BOARD=SABRELITE test_full +} - # Test building and running native .mpy with armv7m architecture. +function ci_qemu_build_arm_thumb_softfp { + ci_qemu_build_arm_prepare + make BOARD=MPS2_AN385 ${MAKEOPTS} -C ports/qemu test_full + + # Test building native .mpy with ARM-M softfp architectures. + ci_native_mpy_modules_build armv6m ci_native_mpy_modules_build armv7m - make ${MAKEOPTS} -C ports/qemu test_natmod + + # Test running native .mpy with all ARM-M architectures. + make BOARD=MPS2_AN385 ${MAKEOPTS} -C ports/qemu test_natmod RUN_TESTS_EXTRA="--arch armv6m" + make BOARD=MPS2_AN385 ${MAKEOPTS} -C ports/qemu test_natmod RUN_TESTS_EXTRA="--arch armv7m" +} + +function ci_qemu_build_arm_thumb_hardfp { + ci_qemu_build_arm_prepare + make BOARD=MPS2_AN500 ${MAKEOPTS} -C ports/qemu test_full + + # Test building native .mpy with all ARM-M hardfp architectures. + ci_native_mpy_modules_build armv7emsp + ci_native_mpy_modules_build armv7emdp + + # Test running native .mpy with all ARM-M hardfp architectures. + make BOARD=MPS2_AN500 ${MAKEOPTS} -C ports/qemu test_natmod RUN_TESTS_EXTRA="--arch armv7emsp" + make BOARD=MPS2_AN500 ${MAKEOPTS} -C ports/qemu test_natmod RUN_TESTS_EXTRA="--arch armv7emdp" } function ci_qemu_build_rv32 { @@ -340,6 +432,12 @@ function ci_qemu_build_rv32 { make ${MAKEOPTS} -C ports/qemu BOARD=VIRT_RV32 test_natmod } +function ci_qemu_build_rv64 { + make ${MAKEOPTS} -C mpy-cross + make ${MAKEOPTS} -C ports/qemu BOARD=VIRT_RV64 submodules + make ${MAKEOPTS} -C ports/qemu BOARD=VIRT_RV64 test +} + ######################################################################################## # ports/renesas-ra @@ -403,13 +501,22 @@ function ci_samd_build { # ports/stm32 function ci_stm32_setup { - ci_gcc_arm_setup + # Use a recent version of the ARM toolchain, to work with Cortex-M55. + wget https://developer.arm.com/-/media/Files/downloads/gnu/14.3.rel1/binrel/arm-gnu-toolchain-14.3.rel1-x86_64-arm-none-eabi.tar.xz + xzcat arm-gnu-toolchain-14.3.rel1-x86_64-arm-none-eabi.tar.xz | tar x + pip3 install pyelftools pip3 install ar pip3 install pyhy } +function ci_stm32_path { + echo $(pwd)/arm-gnu-toolchain-14.3.rel1-x86_64-arm-none-eabi/bin +} + function ci_stm32_pyb_build { + # This function builds the following MCU families: F4, F7. + make ${MAKEOPTS} -C mpy-cross make ${MAKEOPTS} -C ports/stm32 MICROPY_PY_NETWORK_WIZNET5K=5200 submodules make ${MAKEOPTS} -C ports/stm32 BOARD=PYBD_SF2 submodules @@ -421,20 +528,18 @@ function ci_stm32_pyb_build { make ${MAKEOPTS} -C ports/stm32/mboot BOARD=PYBV10 CFLAGS_EXTRA='-DMBOOT_FSLOAD=1 -DMBOOT_VFS_LFS2=1' make ${MAKEOPTS} -C ports/stm32/mboot BOARD=PYBD_SF6 make ${MAKEOPTS} -C ports/stm32/mboot BOARD=STM32F769DISC CFLAGS_EXTRA='-DMBOOT_ADDRESS_SPACE_64BIT=1 -DMBOOT_SDCARD_ADDR=0x100000000ULL -DMBOOT_SDCARD_BYTE_SIZE=0x400000000ULL -DMBOOT_FSLOAD=1 -DMBOOT_VFS_FAT=1' - - # Test building native .mpy with armv7emsp architecture. - git submodule update --init lib/berkeley-db-1.xx - ci_native_mpy_modules_build armv7emsp } function ci_stm32_nucleo_build { + # This function builds the following MCU families: F0, H5, H7, L0, L4, WB. + make ${MAKEOPTS} -C mpy-cross make ${MAKEOPTS} -C ports/stm32 BOARD=NUCLEO_H743ZI submodules git submodule update --init lib/mynewt-nimble # Test building various MCU families, some with additional options. make ${MAKEOPTS} -C ports/stm32 BOARD=NUCLEO_F091RC - make ${MAKEOPTS} -C ports/stm32 BOARD=STM32H573I_DK + make ${MAKEOPTS} -C ports/stm32 BOARD=STM32H573I_DK CFLAGS_EXTRA='-DMICROPY_HW_TINYUSB_STACK=1' make ${MAKEOPTS} -C ports/stm32 BOARD=NUCLEO_H743ZI COPT=-O2 CFLAGS_EXTRA='-DMICROPY_PY_THREAD=1' make ${MAKEOPTS} -C ports/stm32 BOARD=NUCLEO_L073RZ make ${MAKEOPTS} -C ports/stm32 BOARD=NUCLEO_L476RG DEBUG=1 @@ -454,21 +559,22 @@ function ci_stm32_nucleo_build { } function ci_stm32_misc_build { + # This function builds the following MCU families: G0, G4, H7, L1, N6, U5, WL. + make ${MAKEOPTS} -C mpy-cross make ${MAKEOPTS} -C ports/stm32 BOARD=ARDUINO_GIGA submodules make ${MAKEOPTS} -C ports/stm32 BOARD=ARDUINO_GIGA + make ${MAKEOPTS} -C ports/stm32 BOARD=NUCLEO_G0B1RE + make ${MAKEOPTS} -C ports/stm32 BOARD=NUCLEO_G474RE + make ${MAKEOPTS} -C ports/stm32 BOARD=NUCLEO_L152RE + make ${MAKEOPTS} -C ports/stm32 BOARD=NUCLEO_N657X0 + make ${MAKEOPTS} -C ports/stm32 BOARD=NUCLEO_U5A5ZJ_Q + make ${MAKEOPTS} -C ports/stm32 BOARD=NUCLEO_WL55 } ######################################################################################## # ports/unix -CI_UNIX_OPTS_SYS_SETTRACE=( - MICROPY_PY_BTREE=0 - MICROPY_PY_FFI=0 - MICROPY_PY_SSL=0 - CFLAGS_EXTRA="-DMICROPY_PY_SYS_SETTRACE=1" -) - CI_UNIX_OPTS_SYS_SETTRACE_STACKLESS=( MICROPY_PY_BTREE=0 MICROPY_PY_FFI=0 @@ -494,6 +600,26 @@ CI_UNIX_OPTS_QEMU_RISCV64=( MICROPY_STANDALONE=1 ) +CI_UNIX_OPTS_SANITIZE_ADDRESS=( + # Macro MP_ASAN allows detecting ASan on gcc<=13 + CFLAGS_EXTRA="-fsanitize=address --param asan-use-after-return=0 -DMP_ASAN=1" + LDFLAGS_EXTRA="-fsanitize=address --param asan-use-after-return=0" +) + +CI_UNIX_OPTS_SANITIZE_UNDEFINED=( + # Macro MP_UBSAN allows detecting UBSan on gcc<=13 + CFLAGS_EXTRA="-fsanitize=undefined -fno-sanitize=nonnull-attribute -DMP_UBSAN=1" + LDFLAGS_EXTRA="-fsanitize=undefined -fno-sanitize=nonnull-attribute" +) + +CI_UNIX_OPTS_REPR_B=( + VARIANT=standard + CFLAGS_EXTRA="-DMICROPY_OBJ_REPR=MICROPY_OBJ_REPR_B -DMICROPY_PY_UCTYPES=0 -Dmp_int_t=int32_t -Dmp_uint_t=uint32_t" + MICROPY_FORCE_32BIT=1 + RUN_TESTS_MPY_CROSS_FLAGS="--mpy-cross-flags=\"-march=x86 -msmall-int-bits=30\"" + +) + function ci_unix_build_helper { make ${MAKEOPTS} -C mpy-cross make ${MAKEOPTS} -C ports/unix "$@" submodules @@ -509,13 +635,26 @@ function ci_unix_run_tests_helper { make -C ports/unix "$@" test } +function ci_unix_run_tests_full_extra { + micropython=$1 + (cd tests && MICROPY_CPYTHON3=python3 MICROPY_MICROPYTHON=$micropython ./run-multitests.py multi_net/*.py) + (cd tests && MICROPY_CPYTHON3=python3 MICROPY_MICROPYTHON=$micropython ./run-perfbench.py --average 1 1000 1000) +} + +function ci_unix_run_tests_full_no_native_helper { + variant=$1 + shift + micropython=../ports/unix/build-$variant/micropython + make -C ports/unix VARIANT=$variant "$@" test_full_no_native + ci_unix_run_tests_full_extra $micropython +} + function ci_unix_run_tests_full_helper { variant=$1 shift micropython=../ports/unix/build-$variant/micropython make -C ports/unix VARIANT=$variant "$@" test_full - (cd tests && MICROPY_CPYTHON3=python3 MICROPY_MICROPYTHON=$micropython ./run-multitests.py multi_net/*.py) - (cd tests && MICROPY_CPYTHON3=python3 MICROPY_MICROPYTHON=$micropython ./run-perfbench.py 1000 1000) + ci_unix_run_tests_full_extra $micropython } function ci_native_mpy_modules_build { @@ -524,40 +663,19 @@ function ci_native_mpy_modules_build { else arch=$1 fi - for natmod in features1 features3 features4 heapq re + for natmod in btree deflate features1 features3 features4 framebuf heapq random re do - make -C examples/natmod/$natmod clean + make -C examples/natmod/$natmod ARCH=$arch clean make -C examples/natmod/$natmod ARCH=$arch done - # deflate, framebuf, and random currently cannot build on xtensa due to - # some symbols that have been removed from the compiler's runtime, in - # favour of being provided from ROM. - if [ $arch != "xtensa" ]; then - for natmod in deflate framebuf random - do - make -C examples/natmod/$natmod clean - make -C examples/natmod/$natmod ARCH=$arch - done - fi - - # features2 requires soft-float on armv7m, rv32imc, and xtensa. On armv6m - # the compiler generates absolute relocations in the object file - # referencing soft-float functions, which is not supported at the moment. - make -C examples/natmod/features2 clean - if [ $arch = "rv32imc" ] || [ $arch = "armv7m" ] || [ $arch = "xtensa" ]; then + # features2 requires soft-float on rv32imc and xtensa. + make -C examples/natmod/features2 ARCH=$arch clean + if [ $arch = "rv32imc" ] || [ $arch = "xtensa" ]; then make -C examples/natmod/features2 ARCH=$arch MICROPY_FLOAT_IMPL=float - elif [ $arch != "armv6m" ]; then + else make -C examples/natmod/features2 ARCH=$arch fi - - # btree requires thread local storage support on rv32imc, whilst on xtensa - # it relies on symbols that are provided from ROM but not exposed to - # natmods at the moment. - if [ $arch != "rv32imc" ] && [ $arch != "xtensa" ]; then - make -C examples/natmod/btree clean - make -C examples/natmod/btree ARCH=$arch - fi } function ci_native_mpy_modules_32bit_build { @@ -569,7 +687,7 @@ function ci_unix_minimal_build { } function ci_unix_minimal_run_tests { - (cd tests && MICROPY_CPYTHON3=python3 MICROPY_MICROPYTHON=../ports/unix/build-minimal/micropython ./run-tests.py -e exception_chain -e self_type_check -e subclass_native_init -d basics) + make -C ports/unix VARIANT=minimal test } function ci_unix_standard_build { @@ -591,9 +709,9 @@ function ci_unix_standard_v2_run_tests { } function ci_unix_coverage_setup { - sudo pip3 install setuptools - sudo pip3 install pyelftools - sudo pip3 install ar + pip3 install setuptools + pip3 install pyelftools + pip3 install ar gcc --version python3 --version } @@ -604,7 +722,7 @@ function ci_unix_coverage_build { } function ci_unix_coverage_run_tests { - ci_unix_run_tests_full_helper coverage + MICROPY_TEST_TIMEOUT=60 ci_unix_run_tests_full_helper coverage } function ci_unix_coverage_run_mpy_merge_tests { @@ -639,12 +757,11 @@ function ci_unix_coverage_run_native_mpy_tests { function ci_unix_32bit_setup { sudo dpkg --add-architecture i386 sudo apt-get update - sudo apt-get install gcc-multilib g++-multilib libffi-dev:i386 python2.7 + sudo apt-get install gcc-multilib g++-multilib libffi-dev:i386 sudo pip3 install setuptools sudo pip3 install pyelftools sudo pip3 install ar gcc --version - python2.7 --version python3 --version } @@ -662,13 +779,20 @@ function ci_unix_coverage_32bit_run_native_mpy_tests { } function ci_unix_nanbox_build { - # Use Python 2 to check that it can run the build scripts - ci_unix_build_helper PYTHON=python2.7 VARIANT=nanbox CFLAGS_EXTRA="-DMICROPY_PY_MATH_CONSTANTS=1" + ci_unix_build_helper VARIANT=nanbox CFLAGS_EXTRA="-DMICROPY_PY_MATH_CONSTANTS=1" ci_unix_build_ffi_lib_helper gcc -m32 } function ci_unix_nanbox_run_tests { - ci_unix_run_tests_full_helper nanbox PYTHON=python2.7 + ci_unix_run_tests_full_no_native_helper nanbox +} + +function ci_unix_longlong_build { + ci_unix_build_helper VARIANT=longlong "${CI_UNIX_OPTS_SANITIZE_UNDEFINED[@]}" +} + +function ci_unix_longlong_run_tests { + ci_unix_run_tests_full_helper longlong } function ci_unix_float_build { @@ -681,7 +805,17 @@ function ci_unix_float_run_tests { ci_unix_run_tests_helper CFLAGS_EXTRA="-DMICROPY_FLOAT_IMPL=MICROPY_FLOAT_IMPL_FLOAT" } +function ci_unix_gil_enabled_build { + ci_unix_build_helper VARIANT=standard MICROPY_PY_THREAD_GIL=1 + ci_unix_build_ffi_lib_helper gcc +} + +function ci_unix_gil_enabled_run_tests { + ci_unix_run_tests_full_helper standard MICROPY_PY_THREAD_GIL=1 +} + function ci_unix_clang_setup { + sudo apt-get update sudo apt-get install clang clang --version } @@ -693,7 +827,8 @@ function ci_unix_stackless_clang_build { } function ci_unix_stackless_clang_run_tests { - ci_unix_run_tests_helper CC=clang + # Timeout needs to be increased for thread/stress_aes.py test. + MICROPY_TEST_TIMEOUT=90 ci_unix_run_tests_helper CC=clang } function ci_unix_float_clang_build { @@ -706,24 +841,36 @@ function ci_unix_float_clang_run_tests { ci_unix_run_tests_helper CC=clang } -function ci_unix_settrace_build { +function ci_unix_settrace_stackless_build { + make ${MAKEOPTS} -C mpy-cross + make ${MAKEOPTS} -C ports/unix submodules + make ${MAKEOPTS} -C ports/unix "${CI_UNIX_OPTS_SYS_SETTRACE_STACKLESS[@]}" +} + +function ci_unix_settrace_stackless_run_tests { + ci_unix_run_tests_full_helper standard "${CI_UNIX_OPTS_SYS_SETTRACE_STACKLESS[@]}" +} + +function ci_unix_sanitize_undefined_build { make ${MAKEOPTS} -C mpy-cross make ${MAKEOPTS} -C ports/unix submodules - make ${MAKEOPTS} -C ports/unix "${CI_UNIX_OPTS_SYS_SETTRACE[@]}" + make ${MAKEOPTS} -C ports/unix VARIANT=coverage "${CI_UNIX_OPTS_SANITIZE_UNDEFINED[@]}" + ci_unix_build_ffi_lib_helper gcc } -function ci_unix_settrace_run_tests { - ci_unix_run_tests_full_helper standard "${CI_UNIX_OPTS_SYS_SETTRACE[@]}" +function ci_unix_sanitize_undefined_run_tests { + MICROPY_TEST_TIMEOUT=60 ci_unix_run_tests_full_helper coverage VARIANT=coverage "${CI_UNIX_OPTS_SANITIZE_UNDEFINED[@]}" } -function ci_unix_settrace_stackless_build { +function ci_unix_sanitize_address_build { make ${MAKEOPTS} -C mpy-cross make ${MAKEOPTS} -C ports/unix submodules - make ${MAKEOPTS} -C ports/unix "${CI_UNIX_OPTS_SYS_SETTRACE_STACKLESS[@]}" + make ${MAKEOPTS} -C ports/unix VARIANT=coverage "${CI_UNIX_OPTS_SANITIZE_ADDRESS[@]}" + ci_unix_build_ffi_lib_helper gcc } -function ci_unix_settrace_stackless_run_tests { - ci_unix_run_tests_full_helper standard "${CI_UNIX_OPTS_SYS_SETTRACE_STACKLESS[@]}" +function ci_unix_sanitize_address_run_tests { + MICROPY_TEST_TIMEOUT=60 ci_unix_run_tests_full_helper coverage VARIANT=coverage "${CI_UNIX_OPTS_SANITIZE_ADDRESS[@]}" } function ci_unix_macos_build { @@ -740,7 +887,9 @@ function ci_unix_macos_run_tests { # Issues with macOS tests: # - float_parse and float_parse_doubleprec parse/print floats out by a few mantissa bits # - ffi_callback crashes for an unknown reason - (cd tests && MICROPY_MICROPYTHON=../ports/unix/build-standard/micropython ./run-tests.py --exclude '(float_parse|float_parse_doubleprec|ffi_callback).py') + # - thread/stress_heap.py is flaky + # - thread/thread_gc1.py is flaky + (cd tests && MICROPY_MICROPYTHON=../ports/unix/build-standard/micropython ./run-tests.py --exclude '(float_parse|float_parse_doubleprec|ffi_callback|thread/stress_heap|thread/thread_gc1).py') } function ci_unix_qemu_mips_setup { @@ -758,8 +907,12 @@ function ci_unix_qemu_mips_build { } function ci_unix_qemu_mips_run_tests { + # Issues with MIPS tests: + # - thread/stress_aes.py takes around 50 seconds + # - thread/stress_recurse.py is flaky + # - thread/thread_gc1.py is flaky file ./ports/unix/build-coverage/micropython - (cd tests && MICROPY_MICROPYTHON=../ports/unix/build-coverage/micropython ./run-tests.py) + (cd tests && MICROPY_MICROPYTHON=../ports/unix/build-coverage/micropython MICROPY_TEST_TIMEOUT=90 ./run-tests.py --exclude 'thread/stress_recurse.py|thread/thread_gc1.py') } function ci_unix_qemu_arm_setup { @@ -779,8 +932,11 @@ function ci_unix_qemu_arm_build { function ci_unix_qemu_arm_run_tests { # Issues with ARM tests: # - (i)listdir does not work, it always returns the empty list (it's an issue with the underlying C call) + # - thread/stress_aes.py takes around 70 seconds + # - thread/stress_recurse.py is flaky + # - thread/thread_gc1.py is flaky file ./ports/unix/build-coverage/micropython - (cd tests && MICROPY_MICROPYTHON=../ports/unix/build-coverage/micropython ./run-tests.py --exclude 'vfs_posix.*\.py') + (cd tests && MICROPY_MICROPYTHON=../ports/unix/build-coverage/micropython MICROPY_TEST_TIMEOUT=90 ./run-tests.py --exclude 'vfs_posix.*\.py|thread/stress_recurse.py|thread/thread_gc1.py') } function ci_unix_qemu_riscv64_setup { @@ -798,14 +954,30 @@ function ci_unix_qemu_riscv64_build { } function ci_unix_qemu_riscv64_run_tests { + # Issues with RISCV-64 tests: + # - thread/stress_aes.py takes around 140 seconds + # - thread/stress_recurse.py is flaky + # - thread/thread_gc1.py is flaky file ./ports/unix/build-coverage/micropython - (cd tests && MICROPY_MICROPYTHON=../ports/unix/build-coverage/micropython ./run-tests.py) + (cd tests && MICROPY_MICROPYTHON=../ports/unix/build-coverage/micropython MICROPY_TEST_TIMEOUT=180 ./run-tests.py --exclude 'thread/stress_recurse.py|thread/thread_gc1.py') +} + +function ci_unix_repr_b_build { + ci_unix_build_helper "${CI_UNIX_OPTS_REPR_B[@]}" + ci_unix_build_ffi_lib_helper gcc -m32 +} + +function ci_unix_repr_b_run_tests { + # ci_unix_run_tests_full_no_native_helper is not used due to + # https://github.com/micropython/micropython/issues/18105 + ci_unix_run_tests_helper "${CI_UNIX_OPTS_REPR_B[@]}" } ######################################################################################## # ports/windows function ci_windows_setup { + sudo apt-get update sudo apt-get install gcc-mingw-w64 } @@ -813,14 +985,15 @@ function ci_windows_build { make ${MAKEOPTS} -C mpy-cross make ${MAKEOPTS} -C ports/windows submodules make ${MAKEOPTS} -C ports/windows CROSS_COMPILE=i686-w64-mingw32- + make ${MAKEOPTS} -C ports/windows CROSS_COMPILE=x86_64-w64-mingw32- BUILD=build-standard-w64 } ######################################################################################## # ports/zephyr -ZEPHYR_DOCKER_VERSION=v0.26.13 -ZEPHYR_SDK_VERSION=0.16.8 -ZEPHYR_VERSION=v3.7.0 +ZEPHYR_DOCKER_VERSION=v0.28.1 +ZEPHYR_SDK_VERSION=0.17.2 +ZEPHYR_VERSION=v4.2.0 function ci_zephyr_setup { IMAGE=ghcr.io/zephyrproject-rtos/ci:${ZEPHYR_DOCKER_VERSION} @@ -859,6 +1032,7 @@ function ci_zephyr_install { } function ci_zephyr_build { + git submodule update --init lib/micropython-lib docker exec zephyr-ci west build -p auto -b qemu_x86 -- -DCONF_FILE=prj_minimal.conf docker exec zephyr-ci west build -p auto -b frdm_k64f docker exec zephyr-ci west build -p auto -b mimxrt1050_evk @@ -867,9 +1041,7 @@ function ci_zephyr_build { function ci_zephyr_run_tests { docker exec zephyr-ci west build -p auto -b qemu_cortex_m3 -- -DCONF_FILE=prj_minimal.conf - # Issues with zephyr tests: - # - inf_nan_arith fails pow(-1, nan) test - (cd tests && ./run-tests.py -t execpty:"qemu-system-arm -cpu cortex-m3 -machine lm3s6965evb -nographic -monitor null -serial pty -kernel ../ports/zephyr/build/zephyr/zephyr.elf" -d basics float --exclude inf_nan_arith) + (cd tests && ./run-tests.py -t execpty:"qemu-system-arm -cpu cortex-m3 -machine lm3s6965evb -nographic -monitor null -serial pty -kernel ../ports/zephyr/build/zephyr/zephyr.elf") } ######################################################################################## @@ -886,3 +1058,86 @@ function ci_alif_ae3_build { make ${MAKEOPTS} -C ports/alif BOARD=OPENMV_AE3 MCU_CORE=M55_DUAL make ${MAKEOPTS} -C ports/alif BOARD=ALIF_ENSEMBLE MCU_CORE=M55_DUAL } + +function _ci_help { + # Note: these lines must be indented with tab characters (required by bash <<-EOF) + cat <<-EOF + ci.sh: Script fragments used during CI + + When invoked as a script, runs a sequence of ci steps, + stopping after the first error. + + Usage: + ${BASH_SOURCE} step1 step2... + + Steps: + EOF + if type -path column > /dev/null 2>&1; then + grep '^function ci_' $0 | awk '{print $2}' | sed 's/^ci_//' | column + else + grep '^function ci_' $0 | awk '{print $2}' | sed 's/^ci_//' + fi + exit +} + +function _ci_bash_completion { + echo "alias ci=\"$(readlink -f "$0")\"; complete -W '$(grep '^function ci_' $0 | awk '{print $2}' | sed 's/^ci_//')' ci" +} + +function _ci_zsh_completion { + echo "alias ci=\"$(readlink -f "$0"\"); _complete_mpy_ci_zsh() { compadd $(grep '^function ci_' $0 | awk '{sub(/^ci_/,"",$2); print $2}' | tr '\n' ' ') }; autoload -Uz compinit; compinit; compdef _complete_mpy_ci_zsh $(readlink -f "$0")" +} + +function _ci_fish_completion { + echo "alias ci=\"$(readlink -f "$0"\"); complete -c ci -p $(readlink -f "$0") -f -a '$(grep '^function ci_' $(readlink -f "$0") | awk '{sub(/^ci_/,"",$2); print $2}' | tr '\n' ' ')'" +} + +function _ci_main { + case "$1" in + (-h|-?|--help) + _ci_help + ;; + (--bash-completion) + _ci_bash_completion + ;; + (--zsh-completion) + _ci_zsh_completion + ;; + (--fish-completion) + _ci_fish_completion + ;; + (-*) + echo "Unknown option: $1" 1>&2 + exit 1 + ;; + (*) + set -e + cd $(dirname "$0")/.. + while [ $# -ne 0 ]; do + ci_$1 + shift + done + ;; + esac +} + +# https://stackoverflow.com/questions/2683279/how-to-detect-if-a-script-is-being-sourced +sourced=0 +if [ -n "$ZSH_VERSION" ]; then + case $ZSH_EVAL_CONTEXT in *:file) sourced=1;; esac +elif [ -n "$KSH_VERSION" ]; then + [ "$(cd -- "$(dirname -- "$0")" && pwd -P)/$(basename -- "$0")" != "$(cd -- "$(dirname -- "${.sh.file}")" && pwd -P)/$(basename -- "${.sh.file}")" ] && sourced=1 +elif [ -n "$BASH_VERSION" ]; then + (return 0 2>/dev/null) && sourced=1 +else # All other shells: examine $0 for known shell binary filenames. + # Detects `sh` and `dash`; add additional shell filenames as needed. + case ${0##*/} in sh|-sh|dash|-dash) sourced=1;; esac +fi + +if [ $sourced -eq 0 ]; then + # invoked as a command + if [ "$#" -eq 0 ]; then + set -- --help + fi + _ci_main "$@" +fi diff --git a/tools/codeformat.py b/tools/codeformat.py index a648d401ec312..cf91049a73162 100644 --- a/tools/codeformat.py +++ b/tools/codeformat.py @@ -231,7 +231,7 @@ def batch(cmd, files, N=200, check=False): command = ["python3", "tools/ruff_bindings.py"] batch(command, bindings_files(), check=True) - # Format Python files with black. + # Format Python files with "ruff format" (using config in pyproject.toml). if format_py: command = ["ruff", "format"] if args.v: diff --git a/tools/file2h.py b/tools/file2h.py index df9cc02fdabba..2707d4a16e5cf 100644 --- a/tools/file2h.py +++ b/tools/file2h.py @@ -9,8 +9,6 @@ # ; # This script simply prints the escaped string straight to stdout -from __future__ import print_function - import sys # Can either be set explicitly, or left blank to auto-detect diff --git a/tools/insert-usb-ids.py b/tools/insert-usb-ids.py index 4691d5a710c1e..1b043e1fef04f 100644 --- a/tools/insert-usb-ids.py +++ b/tools/insert-usb-ids.py @@ -6,8 +6,6 @@ # inserts those values into the template file specified by sys.argv[2], # printing the result to stdout -from __future__ import print_function - import sys import re import string diff --git a/tools/makemanifest.py b/tools/makemanifest.py index e076a03e0be3c..860935397af14 100644 --- a/tools/makemanifest.py +++ b/tools/makemanifest.py @@ -24,7 +24,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -from __future__ import print_function import sys import os import subprocess diff --git a/tools/manifestfile.py b/tools/manifestfile.py index beaa36d0f5fc2..9c7a6e140f968 100644 --- a/tools/manifestfile.py +++ b/tools/manifestfile.py @@ -25,7 +25,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -from __future__ import print_function import contextlib import os import sys diff --git a/tools/metrics.py b/tools/metrics.py index f6189e65abb4d..8bb96ba119a23 100755 --- a/tools/metrics.py +++ b/tools/metrics.py @@ -43,20 +43,23 @@ """ -import collections, sys, re, subprocess +import collections, os, sys, re, shlex, subprocess, multiprocessing -MAKE_FLAGS = ["-j3", "CFLAGS_EXTRA=-DNDEBUG"] +MAKE_FLAGS = ["-j{}".format(multiprocessing.cpu_count()), "CFLAGS_EXTRA=-DNDEBUG"] class PortData: - def __init__(self, name, dir, output, make_flags=None): + def __init__(self, name, dir, output, make_flags=None, pre_cmd=None): self.name = name self.dir = dir self.output = output self.make_flags = make_flags self.needs_mpy_cross = dir not in ("bare-arm", "minimal") + self.pre_cmd = pre_cmd +mpy_cross_output = "mpy-cross/build/mpy-cross" + port_data = { "b": PortData("bare-arm", "bare-arm", "build/firmware.elf"), "m": PortData("minimal x86", "minimal", "build/firmware.elf"), @@ -65,7 +68,12 @@ def __init__(self, name, dir, output, make_flags=None): "s": PortData("stm32", "stm32", "build-PYBV10/firmware.elf", "BOARD=PYBV10"), "c": PortData("cc3200", "cc3200", "build/WIPY/release/application.axf", "BTARGET=application"), "8": PortData("esp8266", "esp8266", "build-ESP8266_GENERIC/firmware.elf"), - "3": PortData("esp32", "esp32", "build-ESP32_GENERIC/micropython.elf"), + "3": PortData( + "esp32", + "esp32", + "build-ESP32_GENERIC/micropython.elf", + pre_cmd=". esp-idf/export.sh", + ), "x": PortData("mimxrt", "mimxrt", "build-TEENSY40/firmware.elf"), "e": PortData("renesas-ra", "renesas-ra", "build-EK_RA6M2/firmware.elf"), "r": PortData("nrf", "nrf", "build-PCA10040/firmware.elf"), @@ -74,8 +82,15 @@ def __init__(self, name, dir, output, make_flags=None): "v": PortData("qemu rv32", "qemu", "build-VIRT_RV32/firmware.elf", "BOARD=VIRT_RV32"), } +for port_letter, port in port_data.items(): + port.pre_cmd = os.environ.get(f"PRE_CMD_{port_letter}", port.pre_cmd) + + +def quoted(args): + return " ".join(shlex.quote(word) for word in args) -def syscmd(*args): + +def syscmd(*args, pre_cmd=None): sys.stdout.flush() a2 = [] for a in args: @@ -83,6 +98,10 @@ def syscmd(*args): a2.append(a) elif a: a2.extend(a) + if pre_cmd is not None: + a2_quoted = quoted(a2) + a2 = ["bash", "-c", "{} && {}".format(pre_cmd, a2_quoted)] + print(a2) subprocess.check_call(a2) @@ -108,6 +127,8 @@ def read_build_log(filename): with open(filename) as f: for line in f: line = line.strip() + if line.startswith("BUILDING ") and "_ref" not in data: + data["_ref"] = line.removeprefix("BUILDING ") if line.strip() == "COMPUTING SIZES": found_sizes = True elif found_sizes: @@ -139,9 +160,15 @@ def do_diff(args): data1 = read_build_log(args[0]) data2 = read_build_log(args[1]) + ref1 = data1.pop("_ref", "(unknown ref)") + ref2 = data2.pop("_ref", "(unknown ref)") + print(f"Reference: {ref1}") + print(f"Comparison: {ref2}") max_delta = None for key, value1 in data1.items(): value2 = data2[key] + if key == mpy_cross_output: + name = "mpy-cross" for port in port_data.values(): if key == "ports/{}/{}".format(port.dir, port.output): name = port.name @@ -181,8 +208,19 @@ def do_clean(args): ports = parse_port_list(args) print("CLEANING") + + if any(port.needs_mpy_cross for port in ports): + syscmd("make", "-C", "mpy-cross", "clean") + for port in ports: - syscmd("make", "-C", "ports/{}".format(port.dir), port.make_flags, "clean") + syscmd( + "make", + "-C", + "ports/{}".format(port.dir), + port.make_flags, + "clean", + pre_cmd=port.pre_cmd, + ) def do_build(args): @@ -196,7 +234,14 @@ def do_build(args): print("BUILDING PORTS") for port in ports: - syscmd("make", "-C", "ports/{}".format(port.dir), MAKE_FLAGS, port.make_flags) + syscmd( + "make", + "-C", + "ports/{}".format(port.dir), + MAKE_FLAGS, + port.make_flags, + pre_cmd=port.pre_cmd, + ) do_sizes(args) @@ -207,6 +252,10 @@ def do_sizes(args): ports = parse_port_list(args) print("COMPUTING SIZES") + + if any(port.needs_mpy_cross for port in ports): + syscmd("size", mpy_cross_output) + for port in ports: syscmd("size", "ports/{}/{}".format(port.dir, port.output)) diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py index 8849f2f1e5956..bf0b89018e4e7 100755 --- a/tools/mpy-tool.py +++ b/tools/mpy-tool.py @@ -24,40 +24,21 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -# Python 2/3/MicroPython compatibility code -from __future__ import print_function +import io +import struct import sys +from binascii import hexlify -if sys.version_info[0] == 2: - from binascii import hexlify as hexlify_py2 - - str_cons = lambda val, enc=None: str(val) - bytes_cons = lambda val, enc=None: bytearray(val) - is_str_type = lambda o: isinstance(o, str) - is_bytes_type = lambda o: type(o) is bytearray - is_int_type = lambda o: isinstance(o, int) or isinstance(o, long) # noqa: F821 - - def hexlify_to_str(b): - x = hexlify_py2(b) - return ":".join(x[i : i + 2] for i in range(0, len(x), 2)) - -elif sys.version_info[0] == 3: # Also handles MicroPython - from binascii import hexlify - - str_cons = str - bytes_cons = bytes - is_str_type = lambda o: isinstance(o, str) - is_bytes_type = lambda o: isinstance(o, bytes) - is_int_type = lambda o: isinstance(o, int) - - def hexlify_to_str(b): - return str(hexlify(b, ":"), "ascii") +str_cons = str +bytes_cons = bytes +is_str_type = lambda o: isinstance(o, str) +is_bytes_type = lambda o: isinstance(o, bytes) +is_int_type = lambda o: isinstance(o, int) -# end compatibility code +def hexlify_to_str(b): + return str(hexlify(b, ":"), "ascii") -import sys -import struct sys.path.append(sys.path[0] + "/../py") import makeqstrdata as qstrutil @@ -114,6 +95,23 @@ class Config: MP_NATIVE_ARCH_XTENSA = 9 MP_NATIVE_ARCH_XTENSAWIN = 10 MP_NATIVE_ARCH_RV32IMC = 11 +MP_NATIVE_ARCH_RV64IMC = 12 + +MP_NATIVE_ARCH_NAMES = [ + "NONE", + "X86", + "X64", + "ARMV6", + "ARMV6M", + "ARMV7M", + "ARMV7EM", + "ARMV7EMSP", + "ARMV7EMDP", + "XTENSA", + "XTENSAWIN", + "RV32IMC", + "RV64IMC", +] MP_PERSISTENT_OBJ_FUN_TABLE = 0 MP_PERSISTENT_OBJ_NONE = 1 @@ -142,6 +140,8 @@ class Config: MP_BC_FORMAT_VAR_UINT = 2 MP_BC_FORMAT_OFFSET = 3 +MP_NATIVE_ARCH_FLAGS_PRESENT = 0x40 + mp_unary_op_method_name = ( "__pos__", "__neg__", @@ -306,6 +306,25 @@ class Opcode: MP_BC_POP_JUMP_IF_TRUE, MP_BC_POP_JUMP_IF_FALSE, ) + ALL_OFFSET = ( + MP_BC_UNWIND_JUMP, + MP_BC_JUMP, + MP_BC_POP_JUMP_IF_TRUE, + MP_BC_POP_JUMP_IF_FALSE, + MP_BC_JUMP_IF_TRUE_OR_POP, + MP_BC_JUMP_IF_FALSE_OR_POP, + MP_BC_SETUP_WITH, + MP_BC_SETUP_EXCEPT, + MP_BC_SETUP_FINALLY, + MP_BC_POP_EXCEPT_JUMP, + MP_BC_FOR_ITER, + ) + ALL_WITH_CHILD = ( + MP_BC_MAKE_FUNCTION, + MP_BC_MAKE_FUNCTION_DEFARGS, + MP_BC_MAKE_CLOSURE, + MP_BC_MAKE_CLOSURE_DEFARGS, + ) # Create a dict mapping opcode value to opcode name. mapping = ["unknown" for _ in range(256)] @@ -562,6 +581,7 @@ def __init__( mpy_source_file, mpy_segments, header, + arch_flags, qstr_table, obj_table, raw_code, @@ -574,6 +594,7 @@ def __init__( self.mpy_segments = mpy_segments self.source_file = qstr_table[0] self.header = header + self.arch_flags = arch_flags self.qstr_table = qstr_table self.obj_table = obj_table self.raw_code = raw_code @@ -651,6 +672,14 @@ def disassemble(self): print("mpy_source_file:", self.mpy_source_file) print("source_file:", self.source_file.str) print("header:", hexlify_to_str(self.header)) + arch_index = (self.header[2] >> 2) & 0x2F + if arch_index >= len(MP_NATIVE_ARCH_NAMES): + arch_name = "UNKNOWN" + else: + arch_name = MP_NATIVE_ARCH_NAMES[arch_index] + print("arch:", arch_name) + if self.header[2] & MP_NATIVE_ARCH_FLAGS_PRESENT != 0: + print("arch_flags:", hex(self.arch_flags)) print("qstr_table[%u]:" % len(self.qstr_table)) for q in self.qstr_table: print(" %s" % q.str) @@ -888,7 +917,7 @@ def __init__(self, parent_name, qstr_table, fun_data, prelude_offset, code_kind) self.escaped_name = unique_escaped_name def disassemble_children(self): - print(" children:", [rc.simple_name.str for rc in self.children]) + self.print_children_annotated() for rc in self.children: rc.disassemble() @@ -980,6 +1009,75 @@ def freeze_raw_code(self, prelude_ptr=None, type_sig=0): raw_code_count += 1 raw_code_content += 4 * 4 + @staticmethod + def decode_lineinfo(line_info: memoryview) -> "tuple[int, int, memoryview]": + c = line_info[0] + if (c & 0x80) == 0: + # 0b0LLBBBBB encoding + return (c & 0x1F), (c >> 5), line_info[1:] + else: + # 0b1LLLBBBB 0bLLLLLLLL encoding (l's LSB in second byte) + return (c & 0xF), (((c << 4) & 0x700) | line_info[1]), line_info[2:] + + def get_source_annotation(self, ip: int, file=None) -> dict: + bc_offset = ip - self.offset_opcodes + try: + line_info = memoryview(self.fun_data)[self.offset_line_info : self.offset_opcodes] + except AttributeError: + return {"file": file, "line": None} + + source_line = 1 + while line_info: + bc_increment, line_increment, line_info = self.decode_lineinfo(line_info) + if bc_offset >= bc_increment: + bc_offset -= bc_increment + source_line += line_increment + else: + break + + return {"file": file, "line": source_line} + + def get_label(self, ip: "int | None" = None, child_num: "int | None" = None) -> str: + if ip is not None: + assert child_num is None + return "%s.%d" % (self.escaped_name, ip) + elif child_num is not None: + return "%s.child%d" % (self.escaped_name, child_num) + else: + return "%s" % self.escaped_name + + def print_children_annotated(self) -> None: + """ + Equivalent to `print(" children:", [child.simple_name.str for child in self.children])`, + but also includes json markers for the start and end of each one's name in that line. + """ + + labels = ["%s.children" % self.escaped_name] + annotation_labels = [] + output = io.StringIO() + output.write(" children: [") + sep = ", " + for i, child in enumerate(self.children): + if i != 0: + output.write(sep) + start_col = output.tell() + 1 + output.write(child.simple_name.str) + end_col = output.tell() + 1 + labels.append(self.get_label(child_num=i)) + annotation_labels.append( + { + "name": self.get_label(child_num=i), + "target": child.get_label(), + "range": { + "startCol": start_col, + "endCol": end_col, + }, + }, + ) + output.write("]") + + print(output.getvalue(), annotations={"labels": annotation_labels}, labels=labels) + class RawCodeBytecode(RawCode): def __init__(self, parent_name, qstr_table, obj_table, fun_data): @@ -988,9 +1086,58 @@ def __init__(self, parent_name, qstr_table, obj_table, fun_data): parent_name, qstr_table, fun_data, 0, MP_CODE_BYTECODE ) + def get_opcode_annotations_labels( + self, opcode: int, ip: int, arg: int, sz: int, arg_pos: int, arg_len: int + ) -> "tuple[dict, list[str]]": + annotations = { + "source": self.get_source_annotation(ip), + "disassembly": Opcode.mapping[opcode], + } + labels = [self.get_label(ip)] + + if opcode in Opcode.ALL_OFFSET: + annotations["link"] = { + "offset": arg_pos, + "length": arg_len, + "to": ip + arg + sz, + } + annotations["labels"] = [ + { + "name": self.get_label(ip), + "target": self.get_label(ip + arg + sz), + "range": { + "startCol": arg_pos + 1, + "endCol": arg_pos + arg_len + 1, + }, + }, + ] + + elif opcode in Opcode.ALL_WITH_CHILD: + try: + child = self.children[arg] + except IndexError: + # link out-of-range child to the child array itself + target = "%s.children" % self.escaped_name + else: + # link resolvable child to the actual child + target = child.get_label() + + annotations["labels"] = [ + { + "name": self.get_label(ip), + "target": target, + "range": { + "startCol": arg_pos + 1, + "endCol": arg_pos + arg_len + 1, + }, + }, + ] + + return annotations, labels + def disassemble(self): bc = self.fun_data - print("simple_name:", self.simple_name.str) + print("simple_name:", self.simple_name.str, labels=[self.get_label()]) print(" raw bytecode:", len(bc), hexlify_to_str(bc)) print(" prelude:", self.prelude_signature) print(" args:", [self.qstr_table[i].str for i in self.names[1:]]) @@ -1006,9 +1153,22 @@ def disassemble(self): pass else: arg = "" - print( - " %-11s %s %s" % (hexlify_to_str(bc[ip : ip + sz]), Opcode.mapping[bc[ip]], arg) + + pre_arg_part = " %-11s %s" % ( + hexlify_to_str(bc[ip : ip + sz]), + Opcode.mapping[bc[ip]], + ) + arg_part = "%s" % arg + annotations, labels = self.get_opcode_annotations_labels( + opcode=bc[ip], + ip=ip, + arg=arg, + sz=sz, + arg_pos=len(pre_arg_part) + 1, + arg_len=len(arg_part), ) + + print(pre_arg_part, arg_part, annotations=annotations, labels=labels) ip += sz self.disassemble_children() @@ -1085,6 +1245,7 @@ def __init__( MP_NATIVE_ARCH_XTENSA, MP_NATIVE_ARCH_XTENSAWIN, MP_NATIVE_ARCH_RV32IMC, + MP_NATIVE_ARCH_RV64IMC, ): self.fun_data_attributes = '__attribute__((section(".text,\\"ax\\",@progbits # ")))' else: @@ -1102,13 +1263,13 @@ def __init__( self.fun_data_attributes += " __attribute__ ((aligned (4)))" elif ( MP_NATIVE_ARCH_ARMV6M <= config.native_arch <= MP_NATIVE_ARCH_ARMV7EMDP - ) or config.native_arch == MP_NATIVE_ARCH_RV32IMC: - # ARMVxxM or RV32IMC -- two byte align. + ) or MP_NATIVE_ARCH_RV32IMC <= config.native_arch <= MP_NATIVE_ARCH_RV64IMC: + # ARMVxxM or RV{32,64}IMC -- two byte align. self.fun_data_attributes += " __attribute__ ((aligned (2)))" def disassemble(self): fun_data = self.fun_data - print("simple_name:", self.simple_name.str) + print("simple_name:", self.simple_name.str, labels=[self.get_label()]) print( " raw data:", len(fun_data), @@ -1362,7 +1523,7 @@ def read_mpy(filename): if header[1] != config.MPY_VERSION: raise MPYReadError(filename, "incompatible .mpy version") feature_byte = header[2] - mpy_native_arch = feature_byte >> 2 + mpy_native_arch = (feature_byte >> 2) & 0x2F if mpy_native_arch != MP_NATIVE_ARCH_NONE: mpy_sub_version = feature_byte & 3 if mpy_sub_version != config.MPY_SUB_VERSION: @@ -1373,6 +1534,11 @@ def read_mpy(filename): raise MPYReadError(filename, "native architecture mismatch") config.mp_small_int_bits = header[3] + arch_flags = 0 + # Read the architecture-specific flag bits if present. + if (feature_byte & MP_NATIVE_ARCH_FLAGS_PRESENT) != 0: + arch_flags = reader.read_uint() + # Read number of qstrs, and number of objects. n_qstr = reader.read_uint() n_obj = reader.read_uint() @@ -1401,6 +1567,7 @@ def read_mpy(filename): filename, segments, header, + arch_flags, qstr_table, obj_table, raw_code, @@ -1696,26 +1863,40 @@ def merge_mpy(compiled_modules, output_file): merged_mpy.extend(f.read()) else: main_cm_idx = None + arch_flags = 0 for idx, cm in enumerate(compiled_modules): feature_byte = cm.header[2] - mpy_native_arch = feature_byte >> 2 + mpy_native_arch = (feature_byte >> 2) & 0x2F if mpy_native_arch: # Must use qstr_table and obj_table from this raw_code if main_cm_idx is not None: raise Exception("can't merge files when more than one contains native code") main_cm_idx = idx + arch_flags = cm.arch_flags if main_cm_idx is not None: # Shift main_cm to front of list. compiled_modules.insert(0, compiled_modules.pop(main_cm_idx)) + if config.arch_flags is not None: + arch_flags = config.arch_flags + header = bytearray(4) ## CIRCUITPY-CHANGE: "C" is used for CircuitPython header[0] = ord("C") header[1] = config.MPY_VERSION - header[2] = config.native_arch << 2 | config.MPY_SUB_VERSION if config.native_arch else 0 + header[2] = ( + (MP_NATIVE_ARCH_FLAGS_PRESENT if arch_flags != 0 else 0) + | config.native_arch << 2 + | config.MPY_SUB_VERSION + if config.native_arch + else 0 + ) header[3] = config.mp_small_int_bits merged_mpy.extend(header) + if arch_flags != 0: + merged_mpy.extend(mp_encode_uint(arch_flags)) + n_qstr = 0 n_obj = 0 for cm in compiled_modules: @@ -1771,6 +1952,138 @@ def copy_section(file, offset, offset2): f.write(merged_mpy) +def extract_segments(compiled_modules, basename, kinds_arg): + import re + + kind_str = ("META", "QSTR", "OBJ", "CODE") + kinds = set() + if kinds_arg is not None: + for kind in kinds_arg.upper().split(","): + if kind in kind_str: + kinds.add(kind) + else: + raise Exception('unknown segment kind "%s"' % (kind,)) + segments = [] + for module in compiled_modules: + for segment in module.mpy_segments: + if not kinds or kind_str[segment.kind] in kinds: + segments.append((module.mpy_source_file, module.source_file.str, segment)) + count_len = len(str(len(segments))) + sanitiser = re.compile("[^a-zA-Z0-9_.-]") + for counter, entry in enumerate(segments): + file_name, source_file, segment = entry + output_name = ( + basename + + "_" + + str(counter).rjust(count_len, "0") + + "_" + + sanitiser.sub("_", source_file) + + "_" + + kind_str[segment.kind] + + "_" + + sanitiser.sub("_", str(segment.name)) + + ".bin" + ) + with open(file_name, "rb") as source: + with open(output_name, "wb") as output: + source.seek(segment.start) + output.write(source.read(segment.end - segment.start)) + + +class PrintShim: + """Base class for interposing extra functionality onto the global `print` method.""" + + def __init__(self): + self.wrapped_print = None + + def __enter__(self): + global print + + if self.wrapped_print is not None: + raise RecursionError + + self.wrapped_print = print + print = self + + return self + + def __exit__(self, exc_type, exc_value, traceback): + global print + + if self.wrapped_print is None: + return + + print = self.wrapped_print + self.wrapped_print = None + + self.on_exit() + + def on_exit(self): + pass + + def __call__(self, *a, **k): + return self.wrapped_print(*a, **k) + + +class PrintIgnoreExtraArgs(PrintShim): + """Just strip the `annotations` and `labels` kwargs and pass down to the underlying print.""" + + def __call__(self, *a, annotations: dict = {}, labels: "list[str]" = (), **k): + return super().__call__(*a, **k) + + +class PrintJson(PrintShim): + """Output lines as godbolt-compatible JSON with extra annotation info from `annotations` and `labels`, rather than plain text.""" + + def __init__(self, fp=sys.stdout, language_id: str = "mpy"): + super().__init__() + self.fp = fp + self.asm = { + "asm": [], + "labelDefinitions": {}, + "languageId": language_id, + } + self.line_number: int = 0 + self.buf: "io.StringIO | None" = None + + def on_exit(self): + import json + + if self.buf is not None: + # flush last partial line + self.__call__() + + json.dump(self.asm, self.fp) + + def __call__(self, *a, annotations: dict = {}, labels: "list[str]" = (), **k): + # ignore prints directed to an explicit output + if "file" in k: + return super().__call__(*a, **k) + + if self.buf is None: + self.buf = io.StringIO() + + super().__call__(*a, file=sys.stderr, **k) + + if "end" in k: + # buffer partial-line prints to collect into a single AsmResultLine + return super().__call__(*a, file=self.buf, **k) + else: + retval = super().__call__(*a, file=self.buf, end="", **k) + output = self.buf.getvalue() + self.buf = None + + asm_line = {"text": output} + asm_line.update(annotations) + self.asm["asm"].append(asm_line) + + self.line_number += 1 + for label in labels: + self.asm["labelDefinitions"][label] = self.line_number + + return retval + + def main(args=None): global global_qstrs @@ -1784,9 +2097,23 @@ def main(args=None): "-d", "--disassemble", action="store_true", help="output disassembled contents of files" ) cmd_parser.add_argument("-f", "--freeze", action="store_true", help="freeze files") + cmd_parser.add_argument( + "-j", + "--json", + action="store_true", + help="output hexdump, disassembly, and frozen code as JSON with extra metadata", + ) cmd_parser.add_argument( "--merge", action="store_true", help="merge multiple .mpy files into one" ) + cmd_parser.add_argument( + "-e", "--extract", metavar="BASE", type=str, help="write segments into separate files" + ) + cmd_parser.add_argument( + "--extract-only", + metavar="KIND[,...]", + help="extract only segments of the given type (meta, qstr, obj, code)", + ) cmd_parser.add_argument("-q", "--qstr-header", help="qstr header file to freeze against") cmd_parser.add_argument( "-mlongint-impl", @@ -1801,6 +2128,12 @@ def main(args=None): default=16, help="mpz digit size used by target (default 16)", ) + cmd_parser.add_argument( + "-march-flags", + metavar="F", + type=int, + help="architecture flags value to set in the output file (strips existing flags if not present)", + ) cmd_parser.add_argument("-o", "--output", default=None, help="output file") cmd_parser.add_argument("files", nargs="+", help="input .mpy files") args = cmd_parser.parse_args(args) @@ -1813,6 +2146,7 @@ def main(args=None): }[args.mlongint_impl] config.MPZ_DIG_SIZE = args.mmpz_dig_size config.native_arch = MP_NATIVE_ARCH_NONE + config.arch_flags = args.march_flags # set config values for qstrs, and get the existing base set of qstrs # already in the firmware @@ -1836,24 +2170,40 @@ def main(args=None): print(er, file=sys.stderr) sys.exit(1) - if args.hexdump: - hexdump_mpy(compiled_modules) + if args.json: + if args.freeze: + print_shim = PrintJson(sys.stdout, language_id="c") + elif args.hexdump: + print_shim = PrintJson(sys.stdout, language_id="stderr") + elif args.disassemble: + print_shim = PrintJson(sys.stdout, language_id="mpy") + else: + print_shim = PrintJson(sys.stdout) + else: + print_shim = PrintIgnoreExtraArgs() - if args.disassemble: + with print_shim: if args.hexdump: - print() - disassemble_mpy(compiled_modules) + hexdump_mpy(compiled_modules) - if args.freeze: - try: - freeze_mpy(firmware_qstr_idents, compiled_modules) - except FreezeError as er: - print(er, file=sys.stderr) - sys.exit(1) + if args.disassemble: + if args.hexdump: + print() + disassemble_mpy(compiled_modules) + + if args.freeze: + try: + freeze_mpy(firmware_qstr_idents, compiled_modules) + except FreezeError as er: + print(er, file=sys.stderr) + sys.exit(1) if args.merge: merge_mpy(compiled_modules, args.output) + if args.extract: + extract_segments(compiled_modules, args.extract, args.extract_only) + if __name__ == "__main__": main() diff --git a/tools/mpy_ld.py b/tools/mpy_ld.py index 219cd1a7468bc..26db07261631c 100755 --- a/tools/mpy_ld.py +++ b/tools/mpy_ld.py @@ -402,6 +402,7 @@ def __init__(self, arch): self.known_syms = {} # dict of symbols that are defined self.unresolved_syms = [] # list of unresolved symbols self.mpy_relocs = [] # list of relocations needed in the output .mpy file + self.externs = {} # dict of externally-defined symbols def check_arch(self, arch_name): if arch_name != self.arch.name: @@ -491,10 +492,14 @@ def populate_got(env): sym = got_entry.sym if hasattr(sym, "resolved"): sym = sym.resolved - sec = sym.section - addr = sym["st_value"] - got_entry.sec_name = sec.name - got_entry.link_addr += sec.addr + addr + if sym.name in env.externs: + got_entry.sec_name = ".external.fixed_addr" + got_entry.link_addr = env.externs[sym.name] + else: + sec = sym.section + addr = sym["st_value"] + got_entry.sec_name = sec.name + got_entry.link_addr += sec.addr + addr # Get sorted GOT, sorted by external, text, rodata, bss so relocations can be combined got_list = sorted( @@ -520,6 +525,9 @@ def populate_got(env): dest = int(got_entry.name.split("+")[1], 16) // env.arch.word_size elif got_entry.sec_name == ".external.mp_fun_table": dest = got_entry.sym.mp_fun_table_offset + elif got_entry.sec_name == ".external.fixed_addr": + # Fixed-address symbols should not be relocated. + continue elif got_entry.sec_name.startswith(".text"): dest = ".text" elif got_entry.sec_name.startswith(".rodata"): @@ -703,8 +711,9 @@ def do_relocation_text(env, text_addr, r): (addr, value) = process_riscv32_relocation(env, text_addr, r) elif env.arch.name == "EM_ARM" and r_info_type == R_ARM_ABS32: - # happens for soft-float on armv6m - raise ValueError("Absolute relocations not supported on ARM") + # Absolute relocation, handled as a data relocation. + do_relocation_data(env, text_addr, r) + return else: # Unknown/unsupported relocation @@ -773,9 +782,9 @@ def do_relocation_data(env, text_addr, r): ): # Relocation in data.rel.ro to internal/external symbol if env.arch.word_size == 4: - struct_type = "/). + + symbols = {} + + LINE_REGEX = re.compile( + r"^(?PPROVIDE\()?" # optional weak marker start + r"(?P[a-zA-Z_]\w*)" # symbol name + r"=0x(?P
[\da-fA-F]{1,8})*" # symbol address + r"(?(weak)\));$", # optional weak marker end and line terminator + re.ASCII, + ) + + inside_comment = False + for line in (line.strip() for line in source.readlines()): + if line.startswith("/*") and not inside_comment: + if not line.endswith("*/"): + inside_comment = True + continue + if inside_comment: + if line.endswith("*/"): + inside_comment = False + continue + if line.startswith("//"): + continue + match = LINE_REGEX.match("".join(line.split())) + if not match: + continue + tokens = match.groupdict() + symbol = tokens["symbol"] + address = int(tokens["address"], 16) + if symbol in symbols: + raise ValueError(f"Symbol {symbol} already defined") + symbols[symbol] = address + return symbols + + def main(): import argparse @@ -1501,6 +1569,13 @@ def main(): cmd_parser.add_argument( "--output", "-o", default=None, help="output .mpy file (default to input with .o->.mpy)" ) + cmd_parser.add_argument( + "--externs", + "-e", + type=argparse.FileType("rt"), + default=None, + help="linkerscript providing fixed-address symbols to augment symbol resolution", + ) cmd_parser.add_argument("files", nargs="+", help="input files") args = cmd_parser.parse_args() diff --git a/tools/pyboard.py b/tools/pyboard.py index 20310ba7081c0..c9f65d5d873c2 100755 --- a/tools/pyboard.py +++ b/tools/pyboard.py @@ -248,14 +248,22 @@ def inWaiting(self): class Pyboard: def __init__( - self, device, baudrate=115200, user="micro", password="python", wait=0, exclusive=True + self, + device, + baudrate=115200, + user="micro", + password="python", + wait=0, + exclusive=True, + timeout=None, + write_timeout=5, ): self.in_raw_repl = False self.use_raw_paste = True if device.startswith("exec:"): self.serial = ProcessToSerial(device[len("exec:") :]) elif device.startswith("execpty:"): - self.serial = ProcessPtyToTerminal(device[len("qemupty:") :]) + self.serial = ProcessPtyToTerminal(device[len("execpty:") :]) elif device and device[0].isdigit() and device[-1].isdigit() and device.count(".") == 3: # device looks like an IP address self.serial = TelnetToSerial(device, user, password, read_timeout=10) @@ -264,7 +272,12 @@ def __init__( import serial.tools.list_ports # Set options, and exclusive if pyserial supports it - serial_kwargs = {"baudrate": baudrate, "interCharTimeout": 1} + serial_kwargs = { + "baudrate": baudrate, + "timeout": timeout, + "write_timeout": write_timeout, + "interCharTimeout": 1, + } if serial.__version__ >= "3.3": serial_kwargs["exclusive"] = exclusive @@ -304,14 +317,25 @@ def __init__( def close(self): self.serial.close() - def read_until(self, min_num_bytes, ending, timeout=10, data_consumer=None): - # if data_consumer is used then data is not accumulated and the ending must be 1 byte long + def read_until( + self, min_num_bytes, ending, timeout=10, data_consumer=None, timeout_overall=None + ): + """ + min_num_bytes: Obsolete. + ending: Return if 'ending' matches. + timeout [s]: Return if timeout between characters. None: Infinite timeout. + timeout_overall [s]: Return not later than timeout_overall. None: Infinite timeout. + data_consumer: Use callback for incoming characters. + If data_consumer is used then data is not accumulated and the ending must be 1 byte long + + It is not visible to the caller why the function returned. It could be ending or timeout. + """ assert data_consumer is None or len(ending) == 1 + assert isinstance(timeout, (type(None), int, float)) + assert isinstance(timeout_overall, (type(None), int, float)) - data = self.serial.read(min_num_bytes) - if data_consumer: - data_consumer(data) - timeout_count = 0 + data = b"" + begin_overall_s = begin_char_s = time.monotonic() while True: if data.endswith(ending): break @@ -322,15 +346,25 @@ def read_until(self, min_num_bytes, ending, timeout=10, data_consumer=None): data = new_data else: data = data + new_data - timeout_count = 0 + begin_char_s = time.monotonic() else: - timeout_count += 1 - if timeout is not None and timeout_count >= 100 * timeout: + if timeout is not None and time.monotonic() >= begin_char_s + timeout: + break + if ( + timeout_overall is not None + and time.monotonic() >= begin_overall_s + timeout_overall + ): break time.sleep(0.01) return data - def enter_raw_repl(self, soft_reset=True): + def enter_raw_repl(self, soft_reset=True, timeout_overall=10): + try: + self._enter_raw_repl_unprotected(soft_reset, timeout_overall) + except OSError as er: + raise PyboardError("could not enter raw repl: {}".format(er)) + + def _enter_raw_repl_unprotected(self, soft_reset, timeout_overall): self.serial.write(b"\r\x03") # ctrl-C: interrupt any running program # flush input (without relying on serial.flushInput()) @@ -342,7 +376,9 @@ def enter_raw_repl(self, soft_reset=True): self.serial.write(b"\r\x01") # ctrl-A: enter raw REPL if soft_reset: - data = self.read_until(1, b"raw REPL; CTRL-B to exit\r\n>") + data = self.read_until( + 1, b"raw REPL; CTRL-B to exit\r\n>", timeout_overall=timeout_overall + ) if not data.endswith(b"raw REPL; CTRL-B to exit\r\n>"): print(data) raise PyboardError("could not enter raw repl") @@ -352,12 +388,12 @@ def enter_raw_repl(self, soft_reset=True): # Waiting for "soft reboot" independently to "raw REPL" (done below) # allows boot.py to print, which will show up after "soft reboot" # and before "raw REPL". - data = self.read_until(1, b"soft reboot\r\n") + data = self.read_until(1, b"soft reboot\r\n", timeout_overall=timeout_overall) if not data.endswith(b"soft reboot\r\n"): print(data) raise PyboardError("could not enter raw repl") - data = self.read_until(1, b"raw REPL; CTRL-B to exit\r\n") + data = self.read_until(1, b"raw REPL; CTRL-B to exit\r\n", timeout_overall=timeout_overall) if not data.endswith(b"raw REPL; CTRL-B to exit\r\n"): print(data) raise PyboardError("could not enter raw repl") @@ -475,8 +511,8 @@ def eval(self, expression, parse=False): return ret # In Python3, call as pyboard.exec(), see the setattr call below. - def exec_(self, command, data_consumer=None): - ret, ret_err = self.exec_raw(command, data_consumer=data_consumer) + def exec_(self, command, timeout=10, data_consumer=None): + ret, ret_err = self.exec_raw(command, timeout, data_consumer) if ret_err: raise PyboardError("exception", ret, ret_err) return ret diff --git a/tools/pydfu.py b/tools/pydfu.py index cd7354818cdea..376c697cbd5eb 100755 --- a/tools/pydfu.py +++ b/tools/pydfu.py @@ -11,8 +11,6 @@ See document UM0391 for a description of the DFuse file. """ -from __future__ import print_function - import argparse import collections import inspect @@ -75,11 +73,7 @@ # USB DFU interface __DFU_INTERFACE = 0 -# Python 3 deprecated getargspec in favour of getfullargspec, but -# Python 2 doesn't have the latter, so detect which one to use -getargspec = getattr(inspect, "getfullargspec", getattr(inspect, "getargspec", None)) - -if "length" in getargspec(usb.util.get_string).args: +if "length" in inspect.getfullargspec(usb.util.get_string).args: # PyUSB 1.0.0.b1 has the length argument def get_string(dev, index): return usb.util.get_string(dev, 255, index) diff --git a/tools/verifygitlog.py b/tools/verifygitlog.py index 67215d5c5d066..dba6ebd6de59a 100755 --- a/tools/verifygitlog.py +++ b/tools/verifygitlog.py @@ -96,20 +96,47 @@ def verify_message_body(raw_body, err): if len(subject_line) >= 73: err.error("Subject line must be 72 or fewer characters: " + subject_line) + # Do additional checks on the prefix of the subject line. + verify_subject_line_prefix(subject_line.split(": ")[0], err) + # Second one divides subject and body. if len(raw_body) > 1 and raw_body[1]: err.error("Second message line must be empty: " + raw_body[1]) # Message body lines. for line in raw_body[2:]: - # Long lines with URLs are exempt from the line length rule. - if len(line) >= 76 and "://" not in line: + # Long lines with URLs or human names are exempt from the line length rule. + if len(line) >= 76 and not ( + "://" in line + or line.startswith("Co-authored-by: ") + or line.startswith("Signed-off-by: ") + ): err.error("Message lines should be 75 or less characters: " + line) if not raw_body[-1].startswith("Signed-off-by: ") or "@" not in raw_body[-1]: err.error('Message must be signed-off. Use "git commit -s".') +def verify_subject_line_prefix(prefix, err): + ext = (".c", ".h", ".cpp", ".js", ".rst", ".md") + + if prefix.startswith((".", "/")): + err.error('Subject prefix cannot begin with "." or "/".') + + if prefix.endswith("/"): + err.error('Subject prefix cannot end with "/".') + + if prefix.startswith("ports/"): + err.error( + 'Subject prefix cannot begin with "ports/", start with the name of the port instead.' + ) + + if prefix.endswith(ext): + err.error( + "Subject prefix cannot end with a file extension, use the main part of the filename without the extension." + ) + + def run(args): verbose("run", *args)