Bring up Linux kernel

.ci/boot-linux.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+function cleanup {
+    sleep 1
+    pkill -9 rv32emu
+}
+
+function ASSERT {
+    $*
+    local RES=$?
+    if [ $RES -ne 0 ]; then
+        echo 'Assert failed: "' $* '"'
+        exit $RES
+    fi
+}
+
+cleanup
+
+TIMEOUT=50
+OPTS=" -k build/linux-image/Image "
+OPTS+=" -i build/linux-image/rootfs.cpio "
+OPTS+=" -b build/minimal.dtb "
+RUN_LINUX="build/rv32emu ${OPTS}"
+
+ASSERT expect <<DONE
+set timeout ${TIMEOUT}
+spawn ${RUN_LINUX}
+expect "buildroot login:" { send "root\n" } timeout { exit 1 }
+expect "# " { send "uname -a\n" } timeout { exit 2 }
+expect "riscv32 GNU/Linux" { send "\x01"; send "x" } timeout { exit 3 }
+DONE
+
+ret=$?
+cleanup
+
+MESSAGES=("OK!" \
+     "Fail to boot" \
+     "Fail to login" \
+     "Fail to run commands" \
+)
+
+COLOR_G='\e[32;01m' # Green
+COLOR_N='\e[0m' # No color
+printf "\n[ ${COLOR_G}${MESSAGES[$ret]}${COLOR_N} ]\n"
+
+exit ${ret}

.github/workflows/build-artifact.yml

@@ -27,6 +27,12 @@ jobs:
             tests/quake/**
             tests/scimark2/**
             tests/*.c
+      - name: Test file change of Linux image
+        id: test-linux-image-version-change
+        uses: tj-actions/changed-files@v45
+        with:
+          files: |
+            mk/external.mk
       - name: Set alias
         id: has_changed_files
         run: |
@@ -35,8 +41,60 @@ jobs:
           else
             echo "has_changed_files=false" >> $GITHUB_OUTPUT
           fi
+          if [[ ${{ steps.test-linux-image-version-change.outputs.any_modified }} == true ]]; then
+            # Determine if the changes are from Buildroot or the Linux version (The Linux might have several patches, so also need to check the SHA value)
+            echo -n $(git --no-pager diff HEAD^ HEAD | grep -e "+BUILDROOT_VERSION" -e "+LINUX_VERSION" -e "+LINUX_DATA_SHA1") >> linux-image-version-change
+            if [[ -s linux-image-version-change ]]; then
+                echo "has_changed_linux_image_version=true" >> $GITHUB_OUTPUT
+            else
+                echo "has_changed_linux_image_version=false" >> $GITHUB_OUTPUT
+            fi
+          else
+            echo "has_changed_linux_image_version=false" >> $GITHUB_OUTPUT
+          fi
     outputs:
       has_changed_files: ${{ steps.has_changed_files.outputs.has_changed_files }}
+      has_changed_linux_image_version: ${{ steps.has_changed_files.outputs.has_changed_linux_image_version }}
+
+  build-linux-image-artifact:
+    needs: [detect-file-change]
+    if: ${{ needs.detect-file-change.outputs.has_changed_linux_image_version == 'true' || github.event_name == 'workflow_dispatch' }}
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          submodules: 'true'
+      - name: Install dependencies
+        run: |
+          sudo apt-get update -q -y
+          sudo apt-get upgrade -q -y
+          sudo apt-get install -q -y build-essential git
+      - name: Build Linux image
+        run: |
+          make build-linux-image
+          make artifact ENABLE_PREBUILT=0 ENABLE_SYSTEM=1
+          mkdir -p /tmp/rv32emu-linux-image-prebuilt/linux-image
+          mv build/linux-image/Image /tmp/rv32emu-linux-image-prebuilt/linux-image
+          mv build/linux-image/rootfs.cpio /tmp/rv32emu-linux-image-prebuilt/linux-image
+          mv build/sha1sum-linux-image /tmp
+      - name: Create tarball
+        run: |
+          cd /tmp
+          tar -zcvf rv32emu-linux-image-prebuilt.tar.gz rv32emu-linux-image-prebuilt
+      - name: Create GitHub Release
+        env:
+          GH_TOKEN: ${{ secrets.RV32EMU_PREBUILT_TOKEN }}
+        run: |
+          RELEASE_TAG=$(date +'%Y.%m.%d'-Linux-Image)
+          cd /tmp
+          gh release create $RELEASE_TAG \
+            --repo sysprog21/rv32emu-prebuilt \
+            --title "$RELEASE_TAG""-nightly"
+          gh release upload $RELEASE_TAG \
+            rv32emu-linux-image-prebuilt.tar.gz \
+            sha1sum-linux-image \
+            --repo sysprog21/rv32emu-prebuilt
 
   build-artifact:
     needs: [detect-file-change]
@@ -83,7 +141,7 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.RV32EMU_PREBUILT_TOKEN }}
         run: |
-          RELEASE_TAG=$(date +'%Y.%m.%d')
+          RELEASE_TAG=$(date +'%Y.%m.%d'-ELF)
           cd /tmp
           gh release create $RELEASE_TAG \
             --repo sysprog21/rv32emu-prebuilt \

.github/workflows/main.yml

@@ -42,7 +42,7 @@ jobs:
     - name: install-dependencies
       run: |
             sudo apt-get update -q -y
-            sudo apt-get install -q -y libsdl2-dev libsdl2-mixer-dev
+            sudo apt-get install -q -y libsdl2-dev libsdl2-mixer-dev device-tree-compiler expect
             .ci/riscv-toolchain-install.sh
             echo "${{ github.workspace }}/toolchain/bin" >> $GITHUB_PATH
             wget https://apt.llvm.org/llvm.sh
@@ -67,11 +67,11 @@ jobs:
     - name: misalignment test in block emulation
       run: |
             make -C tests/system/alignment/
-            make distclean && make ENABLE_EXT_C=0 ENABLE_SYSTEM=1 misalign-in-blk-emu -j$(nproc)
+            make distclean && make ENABLE_ELF_LOADER=1 ENABLE_EXT_C=0 ENABLE_SYSTEM=1 misalign-in-blk-emu -j$(nproc)
     - name: MMU test
       run: |
             make -C tests/system/mmu/
-            make distclean && make ENABLE_SYSTEM=1 mmu-test -j$(nproc)
+            make distclean && make ENABLE_ELF_LOADER=1 ENABLE_SYSTEM=1 mmu-test -j$(nproc)
     - name: gdbstub test
       run: |
             make distclean && make ENABLE_GDBSTUB=1 gdbstub-test -j$(nproc)
@@ -85,6 +85,11 @@ jobs:
       run: |
             make clean && make ENABLE_UBSAN=1 check -j$(nproc)
             make ENABLE_JIT=1 clean && make ENABLE_JIT=1 ENABLE_UBSAN=1 check -j$(nproc)
+    - name: boot Linux kernel test
+      run: |
+            make clean && make ENABLE_SYSTEM=1 && make ENABLE_SYSTEM=1 artifact -j$(nproc)
+            .ci/boot-linux.sh
+            make ENABLE_SYSTEM=1 clean
 
   host-arm64:
     needs: [detect-code-related-file-changes]

Makefile

@@ -9,8 +9,24 @@ CONFIG_FILE := $(OUT)/.config
 
 CFLAGS = -std=gnu99 -O2 -Wall -Wextra
 CFLAGS += -Wno-unused-label
-CFLAGS += -include src/common.h
+CFLAGS += -include src/common.h -Isrc/
 
+# In the system test suite, the executable is an ELF file (e.g., MMU).
+# However, the Linux kernel emulation includes the Image, DT, and
+# root filesystem (rootfs). Therefore, the test suite needs this
+# flag to load the ELF and differentiate it from the kernel emulation.
+ENABLE_ELF_LOADER ?= 0
+$(call set-feature, ELF_LOADER)
+
+# Enable MOP fusion, easier for ablation study
+ENABLE_MOP_FUSION ?= 1
+$(call set-feature, MOP_FUSION)
+
+# Enable block chaining, easier for ablation study
+ENABLE_BLOCK_CHAINING ?= 1
+$(call set-feature, BLOCK_CHAINING)
+
+# Enable system emulation
 ENABLE_SYSTEM ?= 0
 $(call set-feature, SYSTEM)
 
@@ -45,10 +61,6 @@ CFLAGS += $(CFLAGS_NO_CET)
 
 OBJS_EXT :=
 
-ifeq ($(call has, SYSTEM), 1)
-OBJS_EXT += system.o
-endif
-
 # Integer Multiplication and Division instructions
 ENABLE_EXT_M ?= 1
 $(call set-feature, EXT_M)
@@ -71,7 +83,14 @@ src/softfloat/build/Linux-RISCV-GCC/Makefile:
 SOFTFLOAT_LIB := $(SOFTFLOAT_OUT)/softfloat.a
 $(SOFTFLOAT_LIB): src/softfloat/build/Linux-RISCV-GCC/Makefile
 	$(MAKE) -C $(dir $<) BUILD_DIR=$(SOFTFLOAT_OUT) CC=$(CC) AR=$(AR)
-$(OUT)/decode.o $(OUT)/riscv.o: $(SOFTFLOAT_LIB)
+ifeq ($(call has, SYSTEM), 1)
+DEV_OUT := $(OUT)/devices
+endif
+OBJS_NEED_SOFTFLOAT := $(OUT)/decode.o \
+                       $(OUT)/riscv.o \
+                       $(DEV_OUT)/uart.o \
+                       $(DEV_OUT)/plic.o
+$(OBJS_NEED_SOFTFLOAT): $(SOFTFLOAT_LIB)
 LDFLAGS += $(SOFTFLOAT_LIB)
 LDFLAGS += -lm
 endif
@@ -204,8 +223,9 @@ $(OUT)/emulate.o: CFLAGS += -foptimize-sibling-calls -fomit-frame-pointer -fno-s
 include mk/external.mk
 include mk/artifact.mk
 include mk/wasm.mk
+include mk/system.mk
 
-all: config $(BIN)
+all: config $(BUILD_DTB) $(BIN)
 
 OBJS := \
 	map.o \
@@ -222,7 +242,7 @@ OBJS := \
 	main.o
 
 OBJS := $(addprefix $(OUT)/, $(OBJS))
-deps := $(OBJS:%.o=%.o.d)
+deps += $(OBJS:%.o=%.o.d) # mk/system.mk includes prior this line, so declare deps at there
 
 ifeq ($(call has, EXT_F), 1)
 $(OBJS): $(SOFTFLOAT_LIB)
@@ -236,7 +256,7 @@ $(OUT)/%.o: src/%.c $(deps_emcc)
 	$(VECHO) "  CC\t$@\n"
 	$(Q)$(CC) -o $@ $(CFLAGS) $(CFLAGS_emcc) -c -MMD -MF [email protected] $<
 
-$(BIN): $(OBJS)
+$(BIN): $(OBJS) $(DEV_OBJS)
 	$(VECHO) "  LD\t$@\n"
 	$(Q)$(CC) -o $@ $(CFLAGS_emcc) $^ $(LDFLAGS)
 
@@ -333,9 +353,10 @@ endif
 endif
 
 clean:
-	$(RM) $(BIN) $(OBJS) $(HIST_BIN) $(HIST_OBJS) $(deps) $(WEB_FILES) $(CACHE_OUT) src/rv32_jit.c
+	$(RM) $(BIN) $(OBJS) $(DEV_OBJS) $(BUILD_DTB) $(HIST_BIN) $(HIST_OBJS) $(deps) $(WEB_FILES) $(CACHE_OUT) src/rv32_jit.c
 distclean: clean
-	-$(RM) $(DOOM_DATA) $(QUAKE_DATA)
+	-$(RM) $(DOOM_DATA) $(QUAKE_DATA) $(BUILDROOT_DATA) $(LINUX_DATA)
+	$(RM) -r $(OUT)/linux-image
 	$(RM) -r $(TIMIDITY_DATA)
 	$(RM) -r $(OUT)/id1
 	$(RM) -r $(DEMO_DIR)

README.md

@@ -58,6 +58,34 @@ If you don't want the JIT compilation feature, simply build with the following:
 $ make
 ```
 
+### Experimental system emulation
+Device Tree compiler (dtc) is required. To install it on Debian/Ubuntu Linux, enter the following command:
+```
+$ sudo apt install device-tree-compiler
+```
+For macOS, use the following command:
+```
+$ brew install dtc
+```
+
+#### Build and run system emulation
+Build and run using default images (the default images will be fetched from [rv32emu-prebuilt](https://github.com/sysprog21/rv32emu-prebuilt) before running):
+```shell
+$ make ENABLE_SYSTEM=1 system
+```
+
+Build using run using specified images:
+```shell
+$ make ENABLE_SYSTEM=1
+$ build/rv32emu -k <kernel_img_path> -i <rootfs_img_path> -b <dtb_path>
+```
+
+#### Build Linux image
+An automated build script is provided to compile the RISC-V cross-compiler, Busybox, and Linux kernel from source. Please note that it only supports the Linux host environment. It can be found at tools/build-linux-image.sh.
+```
+$ make build-linux-img
+```
+
 ### Verify with prebuilt RISC-V ELF files
 
 Run sample RV32I[M] programs:
@@ -98,7 +126,9 @@ The image containing all the necessary tools for development and testing can be
 * `ENABLE_FULL4G` : Full access to 4 GiB address space
 * `ENABLE_SDL` : Experimental Display and Event System Calls
 * `ENABLE_JIT` : Experimental JIT compiler
-* `ENABLE_SYSTEM` : Experimental system emulation, allowing booting Linux kernel
+* `ENABLE_SYSTEM`: Experimental system emulation, allowing booting Linux kernel. To enable this feature, additional features must also be enabled. However, by default, when `ENABLE_SYSTEM` is enabled, CSR, fence, integer multiplication/division, and atomic Instructions are automatically enabled
+* `ENABLE_MOP_FUSION` : Macro-operation fusion
+* `ENABLE_BLOCK_CHAINING` : Block chaining of translated blocks
 
 e.g., run `make ENABLE_EXT_F=0` for the build without floating-point support.
 
@@ -145,7 +175,7 @@ For macOS users, installing `sdiff` might be required:
 $ brew install diffutils
 ```
 
-To run the tests for specific extension, set the environmental variable `RISCV_DEVICE` to one of `I`, `M`, `A`, `F`, `C`, `Zifencei`, `privilege`.
+To run the tests for specific extension, set the environmental variable `RISCV_DEVICE` to one of `I`, `M`, `A`, `F`, `C`, `Zifencei`, `privilege`, `SYSTEM`.
 ```shell
 $ make arch-test RISCV_DEVICE=I
 ```

assets/system/configs/buildroot.config

@@ -0,0 +1,47 @@
+BR2_riscv=y
+BR2_ARCH="riscv32"
+BR2_NORMALIZED_ARCH="riscv"
+BR2_ENDIAN="LITTLE"
+BR2_GCC_TARGET_ABI="ilp32"
+BR2_READELF_ARCH_NAME="RISC-V"
+BR2_RISCV_ISA_RVI=y
+BR2_RISCV_ISA_RVM=y
+BR2_RISCV_ISA_RVA=y
+# BR2_riscv_g is not set
+BR2_riscv_custom=y
+BR2_RISCV_ISA_CUSTOM_RVM=y
+BR2_RISCV_ISA_CUSTOM_RVA=y
+# BR2_RISCV_ISA_CUSTOM_RVF is not set
+# BR2_RISCV_ISA_CUSTOM_RVC is not set
+BR2_RISCV_32=y
+# BR2_RISCV_64 is not set
+BR2_RISCV_ABI_ILP32=y
+BR2_BINFMT_ELF=y
+BR2_TOOLCHAIN_BUILDROOT_VENDOR="buildroot"
+BR2_TOOLCHAIN_BUILDROOT_GLIBC=y
+BR2_TOOLCHAIN_BUILDROOT_LIBC="glibc"
+# BR2_TOOLCHAIN_BUILDROOT_CXX is not set
+BR2_KERNEL_HEADERS_6_1=y
+BR2_BINUTILS_VERSION_2_42_X=y
+# BR2_BINUTILS_GPROFNG is not set
+BR2_GCC_VERSION_14_X=y
+BR2_EXTRA_GCC_CONFIG_OPTIONS="--enable-softfloat"
+BR2_TOOLCHAIN_HEADERS_AT_LEAST="6.1"
+BR2_TOOLCHAIN_GCC_AT_LEAST_14=y
+BR2_TOOLCHAIN_GCC_AT_LEAST="14"
+BR2_SSP_NONE=y
+# BR2_SSP_REGULAR is not set
+# BR2_SSP_STRONG is not set
+BR2_RELRO_NONE=y
+# BR2_RELRO_PARTIAL is not set
+# BR2_RELRO_FULL is not set
+BR2_FORTIFY_SOURCE_1=y
+# BR2_PACKAGE_URANDOM_SCRIPTS is not set
+BR2_TARGET_ROOTFS_CPIO=y
+BR2_TARGET_ROOTFS_CPIO_FULL=y
+# BR2_TARGET_ROOTFS_CPIO_DRACUT is not set
+BR2_TARGET_ROOTFS_CPIO_NONE=y
+# BR2_TARGET_ROOTFS_TAR is not set
+BR2_PACKAGE_BUSYBOX_CONFIG="busybox.config"
+BR2_PACKAGE_COREMARK=y
+BR2_PACKAGE_DHRYSTONE=y