fix(fw): fix verkle transition tests (naive).

.github/PULL_REQUEST_TEMPLATE.md

@@ -10,5 +10,6 @@
 - [ ] All: Added an entry to [CHANGELOG.md](/ethereum/execution-spec-tests/blob/main/docs/CHANGELOG.md).
 - [ ] All: Considered updating the online docs in the [./docs/](/ethereum/execution-spec-tests/blob/main/docs/) directory.
 - [ ] Tests: All converted JSON/YML tests from [ethereum/tests](/ethereum/tests) have been added to [converted-ethereum-tests.txt](/ethereum/execution-spec-tests/blob/main/converted-ethereum-tests.txt).
+- [ ] Tests: A PR with removal of converted JSON/YML tests from [ethereum/tests](/ethereum/tests) have been opened.
 - [ ] Tests: Included the type and version of evm t8n tool used to locally execute test cases:  e.g., ref with commit hash or geth 1.13.1-stable-3f40e65.
 - [ ] Tests: Ran `mkdocs serve` locally and verified the auto-generated docs for new tests in the [Test Case Reference](https://ethereum.github.io/execution-spec-tests/main/tests/) are correctly formatted.

.github/actions/build-besu-evm/action.yaml

@@ -0,0 +1,40 @@
+name: 'Build Besu EVM'
+description: 'Builds the Besu EVM binary'
+inputs:
+  repo:
+    description: 'Source repository to use to build the EVM binary'
+    required: true
+    default: 'hyperledger/besu'
+  ref:
+    description: 'Reference to branch, commit, or tag to use to build the EVM binary'
+    required: true
+    default: 'main'
+  java:
+    description: 'Java version to use to build Besu'
+    required: false
+    default: '21'
+  java-distro:
+    description: 'Java distribution to use to build Besu'
+    required: false
+    default: 'temurin'
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout Besu
+      uses: actions/checkout@v4
+      with:
+        repository: ${{ inputs.repo }}
+        ref: ${{ inputs.ref }}
+        path: besu
+    - name: Setup Java
+      uses: actions/setup-java@v4
+      with:
+        java-version: ${{ inputs.java }}
+        distribution: ${{ inputs.java-distro }}
+        cache: 'gradle'
+    - name: Build evm cmd
+      shell: bash
+      run: |
+        cd $GITHUB_WORKSPACE/besu
+        ./gradlew installDist
+        echo $GITHUB_WORKSPACE/besu/build/install/besu/bin/evmtool >> $GITHUB_PATH

.github/actions/build-evm/action.yaml

@@ -45,4 +45,10 @@ runs:
       uses: ./.github/actions/build-evmone-evm
       with:
         repo: ${{ steps.config-evm-reader.outputs.repo }}
-        ref: ${{ steps.config-evm-reader.outputs.ref }}
+        ref: ${{ steps.config-evm-reader.outputs.ref }}
+    - name: Build the EVM using Besu action
+      if: steps.config-evm-reader.outputs.impl == 'besu'
+      uses: ./.github/actions/build-besu-evm
+      with:
+        repo: ${{ steps.config-evm-reader.outputs.repo }}
+        ref: ${{ steps.config-evm-reader.outputs.ref }}

.github/actions/build-fixtures/action.yaml

@@ -31,16 +31,7 @@ runs:
         source env/bin/activate
         pip install -e .
         solc-select use  ${{ steps.properties.outputs.solc }} --always-install
-        fill -n auto --evm-bin=${{ steps.evm-builder.outputs.evm-bin }} ${{ steps.properties.outputs.fill-params }}
-    - name: Create fixtures info file
-      shell: bash
-      run: |
-        echo -e "ref: $GITHUB_REF \ncommit: $GITHUB_SHA\nbuild: $(date +"%Y-%m-%dT%H:%M:%SZ")" \
-        > fixtures/info.txt
-    - name: Tar fixtures output
-      shell: bash
-      run: |
-        tar -czvf fixtures_${{ inputs.name }}.tar.gz ./fixtures
+        fill -n auto --evm-bin=${{ steps.evm-builder.outputs.evm-bin }} ${{ steps.properties.outputs.fill-params }} --output=fixtures_${{ inputs.name }}.tar.gz --build-name ${{ inputs.name }}
     - uses: actions/upload-artifact@v4
       with:
         name: fixtures_${{ inputs.name }}

.github/workflows/coverage.yaml

@@ -8,32 +8,35 @@ on:
 jobs:
   evmone-coverage-diff:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        driver: [retesteth, native]
+
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
 
       - name: Fetch target branch
         run: git fetch origin ${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}
 
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
+        with:
+          username: winsvega
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
       - name: Install deps
         run: |
           echo $(pwd)
           echo ${{ github.workspace }}
 
-          #install solc for `pip install -e .` command
-          curl -L --output solc "https://github.com/ethereum/solidity/releases/download/v0.8.25/solc-static-linux"
-          sudo mv solc /usr/local/bin
-          sudo chmod +x /usr/local/bin/solc
-          solc --version
-
           #install pyspec deps from root repo
           python3 --version
+          pip install --upgrade pip
           python3 -m venv ./venv/
           source ./venv/bin/activate
           pip install -e .
-
-          # fix pyspec dependecy
-          pip3 install solc-select
           solc-select use 0.8.25 --always-install
 
       # Required to fill .py tests
@@ -82,15 +85,37 @@ jobs:
           for file in $files; do
               echo $file
           done
+          echo "----------------"
+          echo "Discovered existing json tests that will be BASE files:"
+
 
           BASE_TESTS_PATH=${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS
           mkdir -p $BASE_TESTS_PATH
           for file in $files; do
-              cp ${{ github.workspace }}/testpath/$file $BASE_TESTS_PATH
+              # Make sure each file exist at least in develop or legacy tests
+              file_found=0
+              file_path=${{ github.workspace }}/testpath/$file
+              if [ -e "$file_path" ]; then
+                file_found=1
+                cp $file_path $BASE_TESTS_PATH
+                echo $file_path
+              fi
+
+              # Do not search EOF files in legacy tests (assuming blockchain files we do not cover yet)
               if [[ "$file" == *"GeneralStateTests"* ]]; then
+                  file_path=${{ github.workspace }}/legacytestpath/Cancun/$file
                   base_name=$(basename "$file")
                   legacy_file_name="legacy_$base_name"
-                  cp ${{ github.workspace }}/legacytestpath/Cancun/$file $BASE_TESTS_PATH/$legacy_file_name
+                  if [ -e "$file_path" ]; then
+                    file_found=1
+                    cp $file_path $BASE_TESTS_PATH/$legacy_file_name
+                    echo $file_path
+                  fi
+              fi
+
+              if [ $file_found -eq 0 ]; then
+               echo "Error: Failed to find the test file $file in test repo"
+                exit 1
               fi
           done
           
@@ -101,12 +126,20 @@ jobs:
           python3 -m venv ./venv/
           source ./venv/bin/activate
 
-          # Fetch the base branch and the head branch
-          git fetch origin ${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}
-          git fetch origin ${{ github.head_ref }}:refs/remotes/origin/${{ github.head_ref }}
+          if [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then
+              # Fetch changes when PR comes from remote repo
+              git fetch origin +refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}
+              git fetch origin +refs/pull/${{ github.event.pull_request.number }}/head:refs/remotes/origin/PR-${{ github.event.pull_request.number }}
+              files=$(git diff --name-status origin/${{ github.base_ref }}...origin/PR-${{ github.event.pull_request.number }} -- tests/ | grep -E '^[AM]' | grep '\.py$')
+          else
+              # Fetch the base branch and the head branch
+              git fetch origin ${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}
+              git fetch origin ${{ github.head_ref }}:refs/remotes/origin/${{ github.head_ref }}
+
+              # Perform the diff
+              files=$(git diff --name-status origin/${{ github.base_ref }}...origin/${{ github.head_ref }} -- tests/ | grep -E '^[AM]' | grep '\.py$')
+          fi
 
-          # Perform the diff
-          files=$(git diff --name-status origin/${{ github.base_ref }}...origin/${{ github.head_ref }} -- tests/ | grep -E '^[AM]' | grep '\.py$')
 
           echo "Modified or new .py files in tests folder:"
           echo "$files" | while read line; do
@@ -115,26 +148,42 @@ jobs:
           done
 
           # fill new tests
+          # using `|| true` here because if no tests found, pyspec fill returns error code
           mkdir -p fixtures/state_tests
           mkdir -p fixtures/eof_tests
           echo "$files" | while read line; do
             file=$(echo "$line" | cut -c 3-)
-            fill $file --until=Cancun --evm-bin evmone-t8n || true
-            fill $file --fork=CancunEIP7692 --evm-bin evmone-t8n || true
+            fill $file --until=Cancun --evm-bin evmone-t8n || true >> filloutput.log 2>&1
+            (fill $file --fork=CancunEIP7692 --evm-bin evmone-t8n -k eof_test || true) > >(tee -a filloutput.log filloutputEOF.log) 2>&1
           done
 
+          if grep -q "FAILURES" filloutput.log; then
+             echo "Error: failed to generate .py tests."
+              exit 1
+          fi
+          if [ "${{ matrix.driver }}" = "retesteth" ] && grep -q "passed" filloutputEOF.log; then
+              echo "Disabling retesteth coverage check as EOF tests detected!"
+              echo "retesteth_skip=true" >> $GITHUB_ENV
+              exit 0
+          else
+              echo "retesteth_skip=false" >> $GITHUB_ENV
+          fi
+
+
           filesState=$(find fixtures/state_tests -type f -name "*.json")
           filesEOF=$(find fixtures/eof_tests -type f -name "*.json")
-          if [ -z "$filesState" || -z "$filesEOF" ]; then
+          if [ -z "$filesState" ] && [ -z "$filesEOF" ]; then
               echo "Error: No filled JSON fixtures found in fixtures."
                exit 1
           fi
 
-          mkdir -p ${{ github.workspace }}/evmtest_coverage/coverage/PATCH_TESTS
-          find fixtures/state_tests -type f -name "*.json" -exec cp {} ${{ github.workspace }}/evmtest_coverage/coverage/PATCH_TESTS \;
-          find fixtures/eof_tests -type f -name "*.json" -exec cp {} ${{ github.workspace }}/evmtest_coverage/coverage/PATCH_TESTS \;
+          PATCH_TEST_PATH=${{ github.workspace }}/evmtest_coverage/coverage/PATCH_TESTS
+          mkdir -p $PATCH_TEST_PATH
+          find fixtures/state_tests -type f -name "*.json" -exec cp {} $PATCH_TEST_PATH \;
+          find fixtures/eof_tests -type f -name "*.json" -exec cp {} $PATCH_TEST_PATH \;
 
       - name: Print tests that will be covered
+        if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
         run: |
           echo "Original BASE tests:"
           ls ${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS
@@ -144,38 +193,44 @@ jobs:
 
       - name: Run coverage of the BASE tests
         uses: addnab/docker-run-action@v3
+        if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
         with:
           image: winsvega/evmone-coverage-script:latest
           options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
-          run: /entrypoint.sh --mode=cover --testpath=/tests/BASE_TESTS --outputname=BASE
+          run: /entrypoint.sh --mode=cover --driver=${{ matrix.driver }} --testpath=/tests/BASE_TESTS --outputname=BASE
 
       - name: Run coverage of the PATCH tests
         uses: addnab/docker-run-action@v3
+        if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
         with:
           image: winsvega/evmone-coverage-script:latest
           options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
-          run: /entrypoint.sh --mode=cover --testpath=/tests/PATCH_TESTS --outputname=PATCH
+          run: /entrypoint.sh --mode=cover --driver=${{ matrix.driver }} --testpath=/tests/PATCH_TESTS --outputname=PATCH
 
       - name: Run coverage DIFF of the PATCH tests compared to BASE tests
         uses: addnab/docker-run-action@v3
+        if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
         with:
           image: winsvega/evmone-coverage-script:latest
           options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
           run: /entrypoint.sh --mode=diff --basefile=coverage_BASE.lcov --patchfile=coverage_PATCH.lcov
 
       - name: Chmod coverage results
+        if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
         run: |
           user=$(whoami)
           sudo chown -R $user:$user ${{ github.workspace }}/evmtest_coverage/coverage
 
       - name: Upload coverage results
         uses: actions/upload-artifact@v3
+        if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
         with:
           name: coverage-diff
           path: ${{ github.workspace }}/evmtest_coverage/coverage
 
       - name: Verify coverage results
         uses: addnab/docker-run-action@v3
+        if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
         with:
           image: winsvega/evmone-coverage-script:latest
           options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests

.github/workflows/tox_verify.yaml

@@ -16,15 +16,21 @@ jobs:
             python: '3.12'
             evm-type: 'stable'
             tox-cmd: 'tox run-parallel --parallel-no-spinner'
-          - os: ubuntu-latest
-            python: '3.11'
-            evm-type: 'develop'
-            tox-cmd: 'tox -e tests-develop'
+          # Disabled due to unavailable evm implementation for devnet-1
+          # - os: ubuntu-latest
+          #   python: '3.11'
+          #   evm-type: 'develop'
+          #   tox-cmd: 'tox -e tests-develop'
           # Disabled to not be gated by evmone implementation
           # - os: ubuntu-latest
           #   python: '3.11'
           #   evm-type: 'eip7692'
           #   tox-cmd: 'tox -e tests-eip7692'
+          # Disabled for now until verkle testing framework is refined
+          # - os: ubuntu-latest
+          #   python: '3.11'
+          #   evm-type: 'eip6800'
+          #   tox-cmd: 'tox -e tests-eip6800'
           - os: macos-latest
             python: '3.11'
             evm-type: 'stable'

README.md

@@ -106,7 +106,7 @@ This guide installs stable versions of the required external (go-ethereum) `evm`
    cd execution-spec-tests
    python3 -m venv ./venv/
    source ./venv/bin/activate
-   pip install -e '.[docs,lint,test]'
+   pip install -e .[docs,lint,test]
    ```
 
 3. Verify the installation:

configs/evm.yaml

@@ -6,10 +6,15 @@ stable:
 develop:
   impl: geth
   repo: lightclient/go-ethereum
-  ref: prague-devnet-0
+  ref: prague-devnet-1
   evm-bin: evm
 eip7692:
   impl: evmone
   repo: ethereum/evmone
   ref: master
-  evm-bin: evmone-t8n
+  evm-bin: evmone-t8n
+eip6800:
+  impl: geth
+  repo: gballet/go-ethereum
+  ref: base-root-last-merkle-root-merge 
+  evm-bin: evm

configs/feature.yaml

@@ -9,4 +9,8 @@ develop:
 eip7692:
   evm-type: eip7692
   fill-params: --fork=CancunEIP7692 ./tests/prague
+  solc: 0.8.21
+eip6800:
+  evm-type: eip6800 
+  fill-params: --fork=EIP6800Transition -m blockchain_test
   solc: 0.8.21

converted-ethereum-tests.txt

@@ -0,0 +1,27 @@
+([#497](https://github.com/ethereum/execution-spec-tests/pull/497))
+GeneralStateTests/stCreate2/call_outsize_then_create2_successful_then_returndatasize.json
+GeneralStateTests/stCreate2/call_then_create2_successful_then_returndatasize.json
+
+([#647](https://github.com/ethereum/execution-spec-tests/pull/647))
+EOFTests/efValidation/EOF1_returncontract_invalid_.json
+EOFTests/efValidation/EOF1_returncontract_valid_.json
+
+([#440](https://github.com/ethereum/execution-spec-tests/pull/440))
+GeneralStateTests/Cancun/stEIP1153-transientStorage/01_tloadBeginningTxn.json
+GeneralStateTests/Cancun/stEIP1153-transientStorage/02_tloadAfterTstore.json
+GeneralStateTests/Cancun/stEIP1153-transientStorage/03_tloadAfterStoreIs0.json
+GeneralStateTests/Cancun/stEIP1153-transientStorage/04_tloadAfterCall.json
+GeneralStateTests/Cancun/stEIP1153-transientStorage/05_tloadReentrancy.json
+GeneralStateTests/Cancun/stEIP1153-transientStorage/06_tstoreInReentrancyCall.json
+GeneralStateTests/Cancun/stEIP1153-transientStorage/07_tloadAfterReentrancyStore.json
+GeneralStateTests/Cancun/stEIP1153-transientStorage/08_revertUndoesTransientStore.json
+GeneralStateTests/Cancun/stEIP1153-transientStorage/09_revertUndoesAll.json
+
+GeneralStateTests/Cancun/stEIP1153-transientStorage/11_tstoreDelegateCall.json
+GeneralStateTests/Cancun/stEIP1153-transientStorage/12_tloadDelegateCall.json
+GeneralStateTests/Cancun/stEIP1153-transientStorage/13_tloadStaticCall.json
+
+GeneralStateTests/Cancun/stEIP1153-transientStorage/16_tloadGas.json
+GeneralStateTests/Cancun/stEIP1153-transientStorage/18_tloadAfterStore.json
+
+GeneralStateTests/Cancun/stEIP1153-transientStorage/20_oogUndoesTransientStoreInCall.json