Make SD cards available over web workflow

This changes storage.mount() to require that a mount point exist
on the parent file system.

A bug in background tasks is also fixed where the function
parameter is cleared on pending callbacks during "reset".

Disk usage is shown on the directory listing and changes based on
the mounted file system. Writable is also loaded per-directory.

Fixes micropython#8108. Fixes micropython#8690. Fixes micropython#8107.

Author: tannewt

extmod/vfs_blockdev.c

@@ -30,12 +30,46 @@
 #include "py/mperrno.h"
 #include "extmod/vfs.h"
 
+#if CIRCUITPY_SDCARDIO
+#include "shared-bindings/sdcardio/SDCard.h"
+#endif
+#if CIRCUITPY_SDIOIO
+#include "shared-bindings/sdioio/SDCard.h"
+#endif
+
+#include "esp_log.h"
+
+static const char *TAG = "vfs_blockdev";
+
 #if MICROPY_VFS
 
 void mp_vfs_blockdev_init(mp_vfs_blockdev_t *self, mp_obj_t bdev) {
     mp_load_method(bdev, MP_QSTR_readblocks, self->readblocks);
     mp_load_method_maybe(bdev, MP_QSTR_writeblocks, self->writeblocks);
     mp_load_method_maybe(bdev, MP_QSTR_ioctl, self->u.ioctl);
+    ESP_LOGI(TAG, "new vfs blockdev %p around %p", self, MP_OBJ_TO_PTR(bdev));
+
+    // CIRCUITPY-CHANGE: Support native SD cards.
+    #if CIRCUITPY_SDCARDIO
+    if (mp_obj_get_type(bdev) == &sdcardio_SDCard_type) {
+        ESP_LOGI(TAG, "sdcardio.SDCard");
+        self->flags |= MP_BLOCKDEV_FLAG_NATIVE | MP_BLOCKDEV_FLAG_HAVE_IOCTL;
+        self->readblocks[0] = mp_const_none;
+        self->readblocks[1] = bdev;
+        self->readblocks[2] = (mp_obj_t)sdcardio_sdcard_readblocks; // native version
+        self->writeblocks[0] = mp_const_none;
+        self->writeblocks[1] = bdev;
+        self->writeblocks[2] = (mp_obj_t)sdcardio_sdcard_writeblocks; // native version
+        self->u.ioctl[0] = mp_const_none;
+        self->u.ioctl[1] = bdev;
+        self->u.ioctl[2] = (mp_obj_t)sdcardio_sdcard_ioctl; // native version
+    }
+    #endif
+    #if CIRCUITPY_SDIOIO
+    if (mp_obj_get_type(bdev) == &sdioio_SDCard_type) {
+        // TODO: Enable native blockdev for SDIO too.
+    }
+    #endif
     if (self->u.ioctl[0] != MP_OBJ_NULL) {
         // Device supports new block protocol, so indicate it
         self->flags |= MP_BLOCKDEV_FLAG_HAVE_IOCTL;
@@ -48,8 +82,8 @@ void mp_vfs_blockdev_init(mp_vfs_blockdev_t *self, mp_obj_t bdev) {
 
 int mp_vfs_blockdev_read(mp_vfs_blockdev_t *self, size_t block_num, size_t num_blocks, uint8_t *buf) {
     if (self->flags & MP_BLOCKDEV_FLAG_NATIVE) {
-        mp_uint_t (*f)(uint8_t *, uint32_t, uint32_t) = (void *)(uintptr_t)self->readblocks[2];
-        return f(buf, block_num, num_blocks);
+        mp_uint_t (*f)(mp_obj_t self, uint8_t *, uint32_t, uint32_t) = (void *)(uintptr_t)self->readblocks[2];
+        return f(self->readblocks[1], buf, block_num, num_blocks);
     } else {
         mp_obj_array_t ar = {{&mp_type_bytearray}, BYTEARRAY_TYPECODE, 0, num_blocks *self->block_size, buf};
         self->readblocks[2] = MP_OBJ_NEW_SMALL_INT(block_num);
@@ -80,8 +114,8 @@ int mp_vfs_blockdev_write(mp_vfs_blockdev_t *self, size_t block_num, size_t num_
     }
 
     if (self->flags & MP_BLOCKDEV_FLAG_NATIVE) {
-        mp_uint_t (*f)(const uint8_t *, uint32_t, uint32_t) = (void *)(uintptr_t)self->writeblocks[2];
-        return f(buf, block_num, num_blocks);
+        mp_uint_t (*f)(mp_obj_t self, const uint8_t *, uint32_t, uint32_t) = (void *)(uintptr_t)self->writeblocks[2];
+        return f(self->writeblocks[1], buf, block_num, num_blocks);
     } else {
         mp_obj_array_t ar = {{&mp_type_bytearray}, BYTEARRAY_TYPECODE, 0, num_blocks *self->block_size, (void *)buf};
         self->writeblocks[2] = MP_OBJ_NEW_SMALL_INT(block_num);
@@ -112,6 +146,15 @@ int mp_vfs_blockdev_write_ext(mp_vfs_blockdev_t *self, size_t block_num, size_t
 
 mp_obj_t mp_vfs_blockdev_ioctl(mp_vfs_blockdev_t *self, uintptr_t cmd, uintptr_t arg) {
     if (self->flags & MP_BLOCKDEV_FLAG_HAVE_IOCTL) {
+        if (self->flags & MP_BLOCKDEV_FLAG_NATIVE) {
+            size_t out_value;
+            bool (*f)(mp_obj_t self, uint32_t, uint32_t, size_t *) = (void *)(uintptr_t)self->u.ioctl[2];
+            bool b = f(self->u.ioctl[1], cmd, arg, &out_value);
+            if (!b) {
+                return mp_const_none;
+            }
+            return MP_OBJ_NEW_SMALL_INT(out_value);
+        }
         // New protocol with ioctl
         self->u.ioctl[2] = MP_OBJ_NEW_SMALL_INT(cmd);
         self->u.ioctl[3] = MP_OBJ_NEW_SMALL_INT(arg);

locale/circuitpython.pot

@@ -1364,6 +1364,10 @@ msgstr ""
 msgid "Missing jmp_pin. %q[%u] jumps on pin"
 msgstr ""
 
+#: shared-module/storage/__init__.c
+msgid "Mount point missing. Create first (maybe via USB)"
+msgstr ""
+
 #: shared-bindings/busio/UART.c shared-bindings/displayio/Group.c
 msgid "Must be a %q subclass."
 msgstr ""

ports/espressif/supervisor/internal_flash.c

@@ -60,7 +60,8 @@ STATIC uint32_t _cache_lba = 0xffffffff;
 #define SECSIZE(fs) ((fs)->ssize)
 #endif // FF_MAX_SS == FF_MIN_SS
 STATIC DWORD fatfs_bytes(void) {
-    FATFS *fatfs = filesystem_circuitpy();
+    fs_user_mount_t *fs_mount = filesystem_circuitpy();
+    FATFS *fatfs = &fs_mount->fatfs;
     return (fatfs->csize * SECSIZE(fatfs)) * (fatfs->n_fatent - 2);
 }
 STATIC bool storage_extended = true;

ports/espressif/supervisor/port.c

@@ -284,21 +284,21 @@ safe_mode_t port_init(void) {
     #define ENABLE_JTAG (defined(DEBUG) && DEBUG)
     #endif
 
-    #if ENABLE_JTAG
-    ESP_LOGI(TAG, "Marking JTAG pins never_reset");
-    // JTAG
-    #ifdef CONFIG_IDF_TARGET_ESP32C3
-    common_hal_never_reset_pin(&pin_GPIO4);
-    common_hal_never_reset_pin(&pin_GPIO5);
-    common_hal_never_reset_pin(&pin_GPIO6);
-    common_hal_never_reset_pin(&pin_GPIO7);
-    #elif defined(CONFIG_IDF_TARGET_ESP32S2) || defined(CONFIG_IDF_TARGET_ESP32S3)
-    common_hal_never_reset_pin(&pin_GPIO39);
-    common_hal_never_reset_pin(&pin_GPIO40);
-    common_hal_never_reset_pin(&pin_GPIO41);
-    common_hal_never_reset_pin(&pin_GPIO42);
-    #endif
-    #endif
+    // #if ENABLE_JTAG
+    // ESP_LOGI(TAG, "Marking JTAG pins never_reset");
+    // // JTAG
+    // #ifdef CONFIG_IDF_TARGET_ESP32C3
+    // common_hal_never_reset_pin(&pin_GPIO4);
+    // common_hal_never_reset_pin(&pin_GPIO5);
+    // common_hal_never_reset_pin(&pin_GPIO6);
+    // common_hal_never_reset_pin(&pin_GPIO7);
+    // #elif defined(CONFIG_IDF_TARGET_ESP32S2) || defined(CONFIG_IDF_TARGET_ESP32S3)
+    // common_hal_never_reset_pin(&pin_GPIO39);
+    // common_hal_never_reset_pin(&pin_GPIO40);
+    // common_hal_never_reset_pin(&pin_GPIO41);
+    // common_hal_never_reset_pin(&pin_GPIO42);
+    // #endif
+    // #endif
 
     _never_reset_spi_ram_flash();
 

shared-bindings/sdcardio/SDCard.c

@@ -131,7 +131,7 @@ MP_DEFINE_CONST_FUN_OBJ_1(sdcardio_sdcard_deinit_obj, sdcardio_sdcard_deinit);
 //|
 //|         :return: None"""
 
-STATIC mp_obj_t sdcardio_sdcard_readblocks(mp_obj_t self_in, mp_obj_t start_block_in, mp_obj_t buf_in) {
+STATIC mp_obj_t _sdcardio_sdcard_readblocks(mp_obj_t self_in, mp_obj_t start_block_in, mp_obj_t buf_in) {
     uint32_t start_block = mp_obj_get_int(start_block_in);
     mp_buffer_info_t bufinfo;
     mp_get_buffer_raise(buf_in, &bufinfo, MP_BUFFER_WRITE);
@@ -143,7 +143,7 @@ STATIC mp_obj_t sdcardio_sdcard_readblocks(mp_obj_t self_in, mp_obj_t start_bloc
     return mp_const_none;
 }
 
-MP_DEFINE_CONST_FUN_OBJ_3(sdcardio_sdcard_readblocks_obj, sdcardio_sdcard_readblocks);
+MP_DEFINE_CONST_FUN_OBJ_3(sdcardio_sdcard_readblocks_obj, _sdcardio_sdcard_readblocks);
 
 //|     def sync(self) -> None:
 //|         """Ensure all blocks written are actually committed to the SD card
@@ -171,7 +171,7 @@ MP_DEFINE_CONST_FUN_OBJ_1(sdcardio_sdcard_sync_obj, sdcardio_sdcard_sync);
 //|         :return: None"""
 //|
 
-STATIC mp_obj_t sdcardio_sdcard_writeblocks(mp_obj_t self_in, mp_obj_t start_block_in, mp_obj_t buf_in) {
+STATIC mp_obj_t _sdcardio_sdcard_writeblocks(mp_obj_t self_in, mp_obj_t start_block_in, mp_obj_t buf_in) {
     uint32_t start_block = mp_obj_get_int(start_block_in);
     mp_buffer_info_t bufinfo;
     mp_get_buffer_raise(buf_in, &bufinfo, MP_BUFFER_READ);
@@ -182,7 +182,7 @@ STATIC mp_obj_t sdcardio_sdcard_writeblocks(mp_obj_t self_in, mp_obj_t start_blo
     }
     return mp_const_none;
 }
-MP_DEFINE_CONST_FUN_OBJ_3(sdcardio_sdcard_writeblocks_obj, sdcardio_sdcard_writeblocks);
+MP_DEFINE_CONST_FUN_OBJ_3(sdcardio_sdcard_writeblocks_obj, _sdcardio_sdcard_writeblocks);
 
 STATIC const mp_rom_map_elem_t sdcardio_sdcard_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_count), MP_ROM_PTR(&sdcardio_sdcard_count_obj) },

shared-bindings/sdcardio/SDCard.h

@@ -27,4 +27,19 @@
 
 #pragma once
 
+#include "shared-module/sdcardio/SDCard.h"
+
 extern const mp_obj_type_t sdcardio_SDCard_type;
+
+void common_hal_sdcardio_sdcard_construct(sdcardio_sdcard_obj_t *self, busio_spi_obj_t *spi, const mcu_pin_obj_t *cs, int baudrate);
+void common_hal_sdcardio_sdcard_deinit(sdcardio_sdcard_obj_t *self);
+void common_hal_sdcardio_sdcard_check_for_deinit(sdcardio_sdcard_obj_t *self);
+int common_hal_sdcardio_sdcard_get_blockcount(sdcardio_sdcard_obj_t *self);
+int common_hal_sdcardio_sdcard_readblocks(sdcardio_sdcard_obj_t *self, uint32_t start_block, mp_buffer_info_t *buf);
+int common_hal_sdcardio_sdcard_sync(sdcardio_sdcard_obj_t *self);
+int common_hal_sdcardio_sdcard_writeblocks(sdcardio_sdcard_obj_t *self, uint32_t start_block, mp_buffer_info_t *buf);
+
+// Used by native vfs blockdev.
+mp_uint_t sdcardio_sdcard_readblocks(mp_obj_t self_in, uint8_t *buf, uint32_t start_block, uint32_t buflen);
+mp_uint_t sdcardio_sdcard_writeblocks(mp_obj_t self_in, uint8_t *buf, uint32_t start_block, uint32_t buflen);
+bool sdcardio_sdcard_ioctl(mp_obj_t self_in, size_t cmd, size_t arg, mp_int_t *out_value);

shared-bindings/sdioio/SDCard.c

@@ -164,7 +164,7 @@ MP_DEFINE_CONST_FUN_OBJ_1(sdioio_sdcard_count_obj, sdioio_sdcard_count);
 //|         :param ~circuitpython_typing.WriteableBuffer buf: The buffer to write into.  Length must be multiple of 512.
 //|
 //|         :return: None"""
-STATIC mp_obj_t sdioio_sdcard_readblocks(mp_obj_t self_in, mp_obj_t start_block_in, mp_obj_t buf_in) {
+STATIC mp_obj_t _sdioio_sdcard_readblocks(mp_obj_t self_in, mp_obj_t start_block_in, mp_obj_t buf_in) {
     uint32_t start_block = mp_obj_get_int(start_block_in);
     mp_buffer_info_t bufinfo;
     mp_get_buffer_raise(buf_in, &bufinfo, MP_BUFFER_WRITE);
@@ -176,7 +176,7 @@ STATIC mp_obj_t sdioio_sdcard_readblocks(mp_obj_t self_in, mp_obj_t start_block_
     return mp_const_none;
 }
 
-MP_DEFINE_CONST_FUN_OBJ_3(sdioio_sdcard_readblocks_obj, sdioio_sdcard_readblocks);
+MP_DEFINE_CONST_FUN_OBJ_3(sdioio_sdcard_readblocks_obj, _sdioio_sdcard_readblocks);
 
 //|     def writeblocks(self, start_block: int, buf: ReadableBuffer) -> None:
 //|         """Write one or more blocks to the card
@@ -185,7 +185,7 @@ MP_DEFINE_CONST_FUN_OBJ_3(sdioio_sdcard_readblocks_obj, sdioio_sdcard_readblocks
 //|         :param ~circuitpython_typing.ReadableBuffer buf: The buffer to read from.  Length must be multiple of 512.
 //|
 //|         :return: None"""
-STATIC mp_obj_t sdioio_sdcard_writeblocks(mp_obj_t self_in, mp_obj_t start_block_in, mp_obj_t buf_in) {
+STATIC mp_obj_t _sdioio_sdcard_writeblocks(mp_obj_t self_in, mp_obj_t start_block_in, mp_obj_t buf_in) {
     uint32_t start_block = mp_obj_get_int(start_block_in);
     mp_buffer_info_t bufinfo;
     mp_get_buffer_raise(buf_in, &bufinfo, MP_BUFFER_READ);
@@ -197,7 +197,7 @@ STATIC mp_obj_t sdioio_sdcard_writeblocks(mp_obj_t self_in, mp_obj_t start_block
     return mp_const_none;
 }
 
-MP_DEFINE_CONST_FUN_OBJ_3(sdioio_sdcard_writeblocks_obj, sdioio_sdcard_writeblocks);
+MP_DEFINE_CONST_FUN_OBJ_3(sdioio_sdcard_writeblocks_obj, _sdioio_sdcard_writeblocks);
 
 //|     frequency: int
 //|     """The actual SDIO bus frequency. This may not match the frequency

shared-bindings/sdioio/SDCard.h

@@ -60,6 +60,11 @@ uint32_t common_hal_sdioio_sdcard_get_count(sdioio_sdcard_obj_t *self);
 int common_hal_sdioio_sdcard_readblocks(sdioio_sdcard_obj_t *self, uint32_t start_block, mp_buffer_info_t *bufinfo);
 int common_hal_sdioio_sdcard_writeblocks(sdioio_sdcard_obj_t *self, uint32_t start_block, mp_buffer_info_t *bufinfo);
 
+// Used by native vfs blockdev.
+mp_uint_t sdioio_sdcard_readblocks(mp_obj_t self_in, uint8_t *buf, uint32_t start_block, uint32_t buflen);
+mp_uint_t sdioio_sdcard_writeblocks(mp_obj_t self_in, uint8_t *buf, uint32_t start_block, uint32_t buflen);
+bool sdioio_sdcard_ioctl(mp_obj_t self_in, size_t cmd, size_t arg, mp_int_t *out_value);
+
 // This is used by the supervisor to claim SDIO devices indefinitely.
 extern void common_hal_sdioio_sdcard_never_reset(sdioio_sdcard_obj_t *self);
 

shared-module/os/getenv.c

@@ -62,8 +62,12 @@ STATIC bool open_file(const char *name, file_arg *active_file) {
         return false;
     }
     #else
-    FATFS *fs = filesystem_circuitpy();
-    FRESULT result = f_open(fs, active_file, name, FA_READ);
+    fs_user_mount_t *fs_mount = filesystem_circuitpy();
+    if (fs_mount == NULL) {
+        return false;
+    }
+    FATFS *fatfs = &fs_mount->fatfs;
+    FRESULT result = f_open(fatfs, active_file, name, FA_READ);
     return result == FR_OK;
     #endif
 }