with-context-engine · ajay-bhargava · Jul 25, 2024 · Aug 5, 2024 · Aug 6, 2024 · Aug 7, 2024
diff --git a/.gitignore b/.gitignore
@@ -16,6 +16,9 @@ abc.history
 test_scratch_pad*.py
 
 # Temp focusing image and production logs
-production/temp_focus_image.jpg
+production/focus_image.*
 production/log.txt
-production/frame-firmware-*.hex
+production/frame-firmware-*.hex
+
+# Python virtual env
+.venv
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -56,17 +56,25 @@
             "preLaunchTask": "Build",
             "interface": "swd",
             "device": "nRF52840_XXAA",
-            "BMPGDBSerialPort": "/dev/cu.usbmodem72AE45F31",
             "showDevDebugOutput": "none",
             "svdFile": "libraries/nrfx/mdk/nrf52840.svd",
             "internalConsoleOptions": "neverOpen",
             "loadFiles": [
                 "libraries/softdevice/s140_nrf52_7.3.0_softdevice.hex",
             ],
             "preLaunchCommands": [
+                "monitor rtt enable",
                 "monitor erase_mass",
                 "load",
             ],
+            "linux": {
+                "BMPGDBSerialPort": "/dev/ttyACM0",
+            },
+            "osx": {
+                // TODO find a way to make this automatic
+                "BMPGDBSerialPort": "/dev/cu.usbmodem72AE45F31",
+                // "BMPGDBSerialPort": "/dev/cu.usbmodem72AE30F31",
+            }
         },
         {
             "name": "Bootloader (Black Magic Probe)",
@@ -78,7 +86,6 @@
             "preLaunchTask": "Build Bootloader",
             "interface": "swd",
             "device": "nRF52840_XXAA",
-            "BMPGDBSerialPort": "/dev/cu.usbmodem72AE45F31",
             "showDevDebugOutput": "none",
             "svdFile": "libraries/nrfx/mdk/nrf52840.svd",
             "internalConsoleOptions": "neverOpen",
@@ -88,9 +95,16 @@
                 "build/settings.hex"
             ],
             "preLaunchCommands": [
+                "monitor rtt enable",
                 "monitor erase_mass",
                 "load",
             ],
+            "linux": {
+                "BMPGDBSerialPort": "/dev/ttyACM0",
+            },
+            "osx": {
+                "BMPGDBSerialPort": "/dev/cu.usbmodem72AE45F31",
+            }
         }
     ]
 }
diff --git a/.vscode/tasks.json b/.vscode/tasks.json
@@ -130,7 +130,7 @@
             }
         },
         {
-            "label": "RTT Debug Console",
+            "label": "RTT Debug Console (J-Link)",
             "type": "shell",
             "command": "JLinkRTTClient",
             "problemMatcher": [],
@@ -142,6 +142,25 @@
                 "showReuseMessage": false,
                 "clear": true
             }
+        },
+        {
+            "label": "RTT Debug Console (Black-Magic)",
+            "type": "shell",
+            "osx": {
+                "command": "pkill SCREEN; screen /dev/tty.usbmodem*3",
+            },
+            "linux": {
+                "command": "pkill screen; screen /dev/ttyACM1",
+            },
+            "problemMatcher": [],
+            "presentation": {
+                "echo": false,
+                "reveal": "always",
+                "focus": false,
+                "panel": "dedicated",
+                "showReuseMessage": false,
+                "clear": true
+            }
         }
     ]
 }
diff --git a/Makefile b/Makefile
@@ -22,7 +22,8 @@
 # PERFORMANCE OF THIS SOFTWARE.
 #
 
-BUILD_VERSION := $(shell TZ= date +v%y.%j.%H%M)
+BUILD_VERSION ?= $(shell TZ= date +v%y.%j.%H%M)
+BUILD_VERSION := $(BUILD_VERSION)
 GIT_COMMIT := $(shell git rev-parse --short HEAD)
 
 BUILD := build
@@ -31,16 +32,16 @@ BUILD := build
 ifeq ($(shell uname), Darwin)
 	PORT = $(shell ls /dev/cu.usbmodem*1 2> /dev/null | grep "cu.")
 else
-	PORT = $(shell uname)
+	PORT = /dev/ttyACM0
 endif
 
 application: 
-	@make -C source/application
+	@make -C source/application BUILD_VERSION=$(BUILD_VERSION)
 
 bootloader:
-	@make -C source/application
-	@make -C source/bootloader
-	@make settings-hex-zip
+	@make -C source/application BUILD_VERSION=$(BUILD_VERSION)
+	@make -C source/bootloader BUILD_VERSION=$(BUILD_VERSION)
+	@make settings-hex-zip BUILD_VERSION=$(BUILD_VERSION)
 
 settings-hex-zip:
 	@echo Building settings file...
@@ -75,8 +76,8 @@ settings-hex-zip:
 release:
 	@echo Releasing...
 	@make clean
-	@make application
-	@make settings-hex-zip
+	@make application BUILD_VERSION=$(BUILD_VERSION)
+	@make settings-hex-zip BUILD_VERSION=$(BUILD_VERSION)
 	@echo Released
 
 clean:

diff --git a/README.md b/README.md
@@ -6,13 +6,13 @@ Welcome to the complete codebase of the Frame hardware. For regular usage, check
 
 The codebase is split into three sections. The **nRF52 Application**, the **nRF52 Bootloader**, and the **FPGA RTL**. 
 
-The nRF52 is designed to handle the overall system operation. It runs Lua, as well as handle Bluetooth networking, AI tasks and power management. The FPGA meanwhile, simply handles acceleration of the graphics and camera.
+The nRF52 is designed to handle the overall system operation. It runs Lua, as well as handles Bluetooth networking, AI tasks and power management. The FPGA meanwhile, simply handles acceleration of the graphics and camera.
 
 ![Frame system architecture diagram](docs/diagrams/frame-system-architecture.drawio.png)
 
 ## Getting started with nRF52 firmware development
 
-1. Ensure you have the [ARM GCC Toolchain](https://developer.arm.com/downloads/-/gnu-rm) installed.
+1. Ensure you have the [ARM GCC Toolchain](https://developer.arm.com/downloads/-/arm-gnu-toolchain-downloads) installed.
 
 1. Ensure you have the [nRF Command Line Tools](https://www.nordicsemi.com/Products/Development-tools/nRF-Command-Line-Tools) installed.
 
@@ -49,36 +49,20 @@ The nRF52 is designed to handle the overall system operation. It runs Lua, as we
 
     Try running the `Build` task. The project should build normally.
 
-    You many need to unlock the device by using the `Erase` task before programming or debugging.
+    You may need to unlock the device by using the `Erase` task before programming or debugging.
 
 1. To enable IntelliSense, be sure to select the correct compiler from within VSCode. `Ctrl-Shift-P` (`Cmd-Shift-P` on MacOS) → `C/C++: Select IntelliSense Configuration` → `Use arm-none-eabi-gcc`.
 
 1. Install the [Cortex-Debug](https://marketplace.visualstudio.com/items?itemName=marus25.cortex-debug) extension for VSCode in order to enable debugging.
 
 1. A debugging launch is already configured within `.vscode/launch.json`. Run the `Application (J-Link)` launch configuration from the `Run and Debug` panel, or press `F5`. The project will automatically build and flash before launching.
 
-1. To monitor the logs, run the task `RTT Console` and ensure the `Application (J-Link)` launch configuration is running.
+1. To monitor the logs, run the task `RTT Console (J-Link)` and ensure the `Application (J-Link)` launch configuration is running.
 
-## Getting started with FPGA development
-
-For quickly getting up and running, the accelerators which run on the FPGA are already pre-built and bundled within this repo. If you wish to modify the FPGA RTL, you will need to rebuild the `fpga_application.h` file which contains the entire FPGA application.
-
-1. Ensure you have the [Yosys](https://github.com/YosysHQ/yosys) installed.
-
-1. Ensure you have the [Project Oxide](https://github.com/gatecat/prjoxide) installed.
-
-1. Ensure you have the [nextpnr](https://github.com/YosysHQ/nextpnr) installed.
-
-1. **MacOS users** can do the above three steps in one using [Homebrew](https://brew.sh).
+1. To debug using [Black Magic Probes](https://black-magic.org/index.html), follow the instructions [here](/production/blackmagic/README.md).
 
-    ```sh
-    brew install --HEAD siliconwitchery/oss-fpga/nextpnr-nexus
-    ```
-
-1. You should now be able to rebuild the project by calling `make`:
+## Getting started with FPGA development
 
-    ```sh
-    make fpga/fpga_application.h
-    ```
+The complete FPGA architecture is described in the documentation [here](docs/fpga-architecture.md).
 
-To understand more around how the FPGA RTL works. Check the documentation [here](docs/fpga-architecture.md).
+The FPGA RTL is prebuilt and included in `fpga_application.h` for convenience. If you wish to modify the FPGA RTL, follow the instructions [here](docs/fpga-toolchain-setup.md).
diff --git a/docs/diagrams/camera-jpeg-encoder-architecture-2.drawio.png b/docs/diagrams/camera-jpeg-encoder-architecture-2.drawio.png
diff --git a/docs/diagrams/camera-jpeg-encoder-architecture.drawio.png b/docs/diagrams/camera-jpeg-encoder-architecture.drawio.png
diff --git a/docs/diagrams/camera-pipeline-architecture.drawio.png b/docs/diagrams/camera-pipeline-architecture.drawio.png
diff --git a/docs/fpga-architecture.md b/docs/fpga-architecture.md
@@ -14,21 +14,26 @@ The SPI driver interfaces the FPGA with the nRF52. The FPGA is fully driven over
 
 Each function is accessed through a register. Registers are always addressed by one byte, followed by a various number of read or write bytes based on the operation.
 
-| Address | Function                    | Description | 
-|:-------:|-----------------------------|-------------|
-| 0x10    | `GRAPHICS_CLEAR`            | Clears the background frame buffer.
-| 0x11    | `GRAPHICS_ASSIGN_COLOR`     | Assigns a color to one of the 16 color palette slots. Color should be provided in YCbCr format.<br>**Write: `palette_index[7:0]`**<br>**Write: `y[7:0]`**<br>**Write: `cb[7:0]`**<br>**Write: `cr[7:0]`**
-| 0x12    | `GRAPHICS_DRAW_SPRITE`      | Draws a sprite on the screen. The first two arguments specify an absolute x and y position to print the sprite. The sprite will be printed from its top left corner. The third argument determines the width of the sprite in pixels. The fourth argument determines the number of colors contained in the sprite. This value may be 2, 4, or 16. The final argument specifies the color palette offset for assigning the color values held in the sprite against the stored colors in the palette. Following bytes will then be printed on the background frame buffer.<br>**Write: `x_position[15:0]`**<br>**Write: `y_position[15:0]`**<br>**Write: `width[15:0]`**<br>**Write: `total_colors[7:0]`**<br>**Write: `palette_offset[7:0]`**<br>**Write: `pixel_data[7:0]`**<br>**...**<br>**Write: `pixel_data[7:0]`**<br>
-| 0x13    | `GRAPHICS_DRAW_VECTOR`      | Draws a cubic Bézier curve from the start position to the end position. Control points 1 and 2 are relative to the start and end positions respectively, and are used to determine the shape of the curve. The final argument determines the color used from the current palette, and can be between 0 and 15.<br>**Write: `x_start_position[15:0]`**<br>**Write: `y_start_position[15:0]`**<br>**Write: `x_end_position[15:0]`**<br>**Write: `y_end_position[15:0]`**<br>**Write: `ctrl_1_x_position[15:0]`**<br>**Write: `ctrl_1_y_position[15:0]`**<br>**Write: `ctrl_2_x_position[15:0]`**<br>**Write: `ctrl_2_y_position[15:0]`**<br>**Write: `color[7:0]`**
-| 0x14    | `GRAPHICS_BUFFER_SHOW`      | The foreground and background buffers are switched. The new foreground buffer is continuously rendered to the display, and the background buffer can be used to load new draw commands.
-| 0x20    | `CAMERA_CAPTURE`            | Starts a new image capture.
-| 0x21    | `CAMERA_BYTES_AVAILABLE`    | Returns how many bytes are available to read within the capture memory.<br>**Read: `bytes_available[23:0]`**
-| 0x22    | `CAMERA_READ_BYTES`         | Reads a number of bytes from the capture memory.<br>**Read: `data[7:0]`**<br>**...**<br>**Read: `data[7:0]`**
-| 0x23    | `CAMERA_ZOOM`               | Sets the zoom factor. A setting of `1` captures a 720x720 image, `2` captures 360x360, `3` captures 240x240, and `4` captures 180x180.<br>**Write: `zoom_factor[7:0]`**
-| 0x24    | `CAMERA_PAN`                | Pans the capture window up or down in discrete steps. A setting of `10` captures the top-most part of the image, `0` is the middle, and `-10` is the bottom-most<br>**Write: `pan_position[7:0]`**
-| 0x25    | `CAMERA_READ_BRIGHTNESS`    | Returns current brightness levels for the red, green and blue channels of the camera.<br>**Read: `red_level[7:0]`**<br>**Read: `green_level[7:0]`**<br>**Read: `blue_level[7:0]`**
-| 0x26    | `CAMERA_COMPRESSION_FACTOR` | Sets the compression factor of the saved image between `0` and `100`.<br>**Write: `compression_factor[7:0]`**
-| 0xDB    | `GET_CHIP_ID`               | Returns the chip ID value.<br>**Read: `0x81`**
+| Address | Function                | Description | 
+|:-------:|-------------------------|-------------|
+| 0x11    | `GRAPHICS_ASSIGN_COLOR` | Assigns a color to one of the 16 color palette slots. Color should be provided in YCbCr format.<br>**Write: `palette_index[7:0]`**<br>**Write: `y[7:0]`**<br>**Write: `cb[7:0]`**<br>**Write: `cr[7:0]`**
+| 0x12    | `GRAPHICS_DRAW_SPRITE`  | Draws a sprite on the screen. The first two arguments specify an absolute x and y position to print the sprite. The sprite will be printed from its top left corner. The third argument determines the width of the sprite in pixels. The fourth argument determines the number of colors contained in the sprite. This value may be 2, 4, or 16. The final argument specifies the color palette offset for assigning the color values held in the sprite against the stored colors in the palette. Following bytes will then be printed on the background frame buffer.<br>**Write: `x_position[15:0]`**<br>**Write: `y_position[15:0]`**<br>**Write: `width[15:0]`**<br>**Write: `total_colors[7:0]`**<br>**Write: `palette_offset[7:0]`**<br>**Write: `pixel_data[7:0]`**<br>**...**<br>**Write: `pixel_data[7:0]`**<br>
+| 0x13    | `GRAPHICS_DRAW_VECTOR`  | Draws a cubic Bézier curve from the start position to the end position. Control points 1 and 2 are relative to the start and end positions respectively, and are used to determine the shape of the curve. The final argument determines the color used from the current palette, and can be between 0 and 15.<br>**Write: `x_start_position[15:0]`**<br>**Write: `y_start_position[15:0]`**<br>**Write: `x_end_position[15:0]`**<br>**Write: `y_end_position[15:0]`**<br>**Write: `ctrl_1_x_position[15:0]`**<br>**Write: `ctrl_1_y_position[15:0]`**<br>**Write: `ctrl_2_x_position[15:0]`**<br>**Write: `ctrl_2_y_position[15:0]`**<br>**Write: `color[7:0]`**
+| 0x14    | `GRAPHICS_BUFFER_SHOW`  | The foreground and background buffers are switched. The new foreground buffer is continuously rendered to the display, and the background buffer can be used to load new draw commands.<br><br>**Note**: It is recommended to verify that the value of bit [1] of `GRAPHICS_BUFFER_STATUS` (`0x18`) is zero, in order to ensure that the previous `GRAPHICS_BUFFER_SHOW` command has been accepted before issuing a new `GRAPHICS_BUFFER_SHOW` command.
+| 0x18    | `GRAPHICS_BUFFER_STATUS`| Status of graphics buffer.<br>**Read: `buffer_status[1:0]`**<br>`buffer_status[0]`: Current buffer<br>`buffer_status[1]`: Switch buffer pending
+| 0x20    | `CAMERA_CAPTURE`        | Starts a new image capture.
+| 0x30    | `CAMERA_IMAGE_READY`    | Flag indicating that the JPEG compression has been completed. It is recommended to read a 1 twice before reading the image data.<br>**Read: `image_complete[0]`**
+| 0x21    | `CAMERA_BYTES_AVAILABLE`| Returns how many bytes are available to read within the capture memory.<br>**Read: `bytes_available[23:0]`**
+| 0x31    | `CAMERA_BYTES_TOTAL`    | Returns the size of the entropy coded segment of the JPEG data.<br>**Read: `compressed_bytes[15:0]`**
+| 0x22    | `CAMERA_READ_BYTES`     | Reads a number of bytes from the capture memory.<br>**Read: `data[7:0]`**<br>**...**<br>**Read: `data[7:0]`**
+| 0x23    | `CAMERA_RESOLUTION`     | Sets the resolution of the image capture in pixels. Captured images are always square, so only one value is required.<br>**Write: `resolution[10:0]`**
+| 0x25    | `CAMERA_READ_METERING`  | Returns the current brightness levels for the red, green and blue channels of the camera. Two sets of values are returned representing spot and average metering.<br>**Read: `center_red_level[7:0]`**<br>**Read: `center_green_level[7:0]`**<br>**Read: `center_blue_level[7:0]`**<br>**Read: `average_red_level[7:0]`**<br>**Read: `average_green_level[7:0]`**<br>**Read: `average_blue_level[7:0]`**
+| 0x26    | `CAMERA_QUALITY_FACTOR` | Sets the Quality Factor (QF) of the saved JPEG image. High values are higher quality but bigger size.<br>**Write: `quality_factor[2:0]`**<br>`0x0` = Lowest quality<br>`0x1`<br>`0x2`<br>`0x3`<br>`0x4`<br>`0x5`<br>`0x6`<br>`0x7` = Highest quality<br>
+| 0x28    | `CAMERA_DPHY_POWER_DOWN`| Enables or disables the MIPI D-PHY for power saving when the camera is not needed.<br>**Write: `dphy_power_down[0]`**<br>`0x0` = Normal operation (default)<br>`0x1` = Power down
+| 0x32    | `CAMERA_GAMMA_BYPASS`   | Bypasses the gamma correction block when enabled. This is needed for factory camera focusing and calibration.<br>**Write: `gamma_bypass[0]`**<br>`0x0` = Normal operation (default)<br>`0x1` = Gamma correction bypassed
+| 0x40    | `PLL_CONTROL`           | PLL Control Register. Controls PLL power and clock mux of image buffer.<br>**Write: `pll_powerdown_n[0]`**<br>`0x00` = Power down PLL<br>`0x01` = Power on PLL (default)<br>**Write: `image_buffer_clock_sel[1]`**<br>`0x00` = Image buffer clocked from PLL generated clock (default)<br>`0x01` = Image buffer clocked from SPI clock<br>
+| 0x41    | `PLL_STATUS`            | Status of PLL clock outputs.<br>**Read: `pll_status[0]`**<br>`0x00` = PLL powered down or not stable yet<br>`0x01` = PLL powered on and stable
+| 0xDB    | `GET_CHIP_ID`           | Returns the chip ID value.<br>**Read: `id_value[7:0]`**<br>`0x81` = FPGA running correctly (default)
 
 ## Graphics
 
@@ -68,10 +73,14 @@ Vectors can be drawn with the `GRAPHICS_DRAW_VECTOR` command. By setting the con
 
 ## Camera
 
-The complete pipeline for the camera subsection is as follows:
+The complete pipeline for the camera subsystem is as follows:
 
 ![Camera pipeline for Frame](diagrams/camera-pipeline-architecture.drawio.png)
 
+And the JPEG encoding subsystem is further broken down as follows:
+
+![JPEG encoding pipeline for Frame](diagrams/camera-jpeg-encoder-architecture-2.drawio.png)
+
 ### Capturing Images
 
 TODO