Fix workflows (#50)

* revert devcontainer python 3.11 until python3.12 is stable, use vscode user * switch to vscode user for runner * switch back to root user, try virtual env * added coverage using a broad array of ghidra-python containers (varies python, java, and Ghidra verions
clearbluejar · Oct 31, 2023 · b9b17db · b9b17db
1 parent d05cb1e
commit b9b17db
Show file tree

Hide file tree

Showing 7 changed files with 210 additions and 30 deletions.
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -3,7 +3,8 @@
 {
 	"name": "ghidriff",
 	// image from https://github.com/clearbluejar/ghidra-python
-	"image": "ghcr.io/clearbluejar/ghidra-python:latest",
+	//"image": "ghcr.io/clearbluejar/ghidra-python:latest",
+	"image": "ghcr.io/clearbluejar/ghidra-python:10.4ghidra3.11python-bookworm",
 	// Configure tool-specific properties.
 	"customizations": {
 		// Configure properties specific to VS Code.

diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh
@@ -8,9 +8,6 @@ pip install --upgrade pip
 # Download latest pyi typings for Ghidra Version
 pip install ghidra-stubs
 
-# Install pyhdira
-pip install pyhidra
-
 # If arm64 os, need to build native binaries for Ghidra
 if uname -a | grep -q 'aarch64'; then
     $GHIDRA_INSTALL_DIR/support/buildNatives
@@ -19,6 +16,9 @@ fi
 # install local workspace and test requirements
 pip install -e ".[testing]"
 
+# initialize pyhidra
+python -m pyhidra.install_plugins
+
 # git clone test data if dir doesn't exist
 TEST_DATA_PATH="tests/data"
 

diff --git a/.github/workflows/devcontainer.json b/.github/workflows/devcontainer.json
@@ -0,0 +1,55 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.245.2/containers/python-3
+{
+    "name": "ghidriff",
+    // image from https://github.com/clearbluejar/ghidra-python
+    //"image": "ghcr.io/clearbluejar/ghidra-python:latest",
+    "image": "ghcr.io/clearbluejar/ghidra-python:${localEnv:DC_IMAGE_TAG}",
+    // Configure tool-specific properties.
+    // "customizations": {
+    //     // Configure properties specific to VS Code.
+    //     "vscode": {
+    //         // Set *default* container specific settings.json values on container create.
+    //         "settings": {
+    //             "python.defaultInterpreterPath": "/usr/local/bin/python",
+    //             "python.linting.enabled": true,
+    //             "python.linting.pylintEnabled": true,
+    //             "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
+    //             "python.formatting.blackPath": "/usr/local/py-utils/bin/black",
+    //             "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
+    //             "python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
+    //             "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8",
+    //             "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
+    //             "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
+    //             "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
+    //             "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint",
+    //             // VS code settings for ghidra-stubs autocomplete
+    //             "python.analysis.stubPath": "${workspaceFolder}/.env/lib/python3.11/site-packages/ghidra-stubs/",
+    //             "python.autoComplete.extraPaths": [
+    //                 "${workspaceFolder}/.env/lib/python3.11/site-packages/ghidra-stubs/"
+    //             ],
+    //             "python.analysis.extraPaths": [
+    //                 "${workspaceFolder}/.env/lib/python3.11/site-packages/ghidra-stubs/"
+    //             ],
+    //             "[python]": {
+    //                 "editor.formatOnSave": true,
+    //             },
+    //         },
+    //         // Add the IDs of extensions you want installed when the container is created.
+    //         "extensions": [
+    //             "ms-python.python",
+    //             "ms-python.vscode-pylance",
+    //             "yzhang.markdown-all-in-one"
+    //         ]
+    //     }
+    // },
+    // Use 'forwardPorts' to make a list of ports inside the container available locally.
+    // "forwardPorts": [],
+    // Use 'postCreateCommand' to run commands after the container is created.
+    // "postCreateCommand": [
+    //     "bash",
+    //     ".devcontainer/post-create.sh"
+    // ],
+    // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+    "remoteUser": "vscode",
+}
diff --git a/.github/workflows/pytest-devcontainer-pypi.yml b/.github/workflows/pytest-devcontainer-pypi.yml
@@ -0,0 +1,54 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Pytest Python PyPi Package Repo In Devcontainer
+
+on:
+  push:
+    branches: [ "main" ]
+    paths:
+      - '.github/workflows/pytest-devcontainer-pypi.yml'
+  schedule:
+    - cron: '30 5,15 * * *'
+  workflow_dispatch:
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # cover the latest and all versions of all subreleases   
+        # add "latest" when jpype is update for python 3.12     
+        image: ["10.4ghidra3.11python-bookworm","10.4ghidra3.9python-bookworm", "10.3.3ghidra3.10python-bookworm", "10.2.3ghidra3.11python-bookworm", "10.1.5ghidra3.11python-bookworm" ]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Overwrite original devcontainer with workflow devcontainer
+      run: |
+        cp .github/workflows/devcontainer.json .devcontainer/devcontainer.json 
+    - name: Test with pytest on devcontainer  
+      env:
+        # this will set the imageName in the base docker container .github/workflows/.devcontainer/devcontainer.json
+        # (which is needed because imageName for devcontainers/[email protected] doesn't affect which image you use!
+        DC_IMAGE_TAG : ${{matrix.image}}
+      uses: devcontainers/[email protected]
+      with: 
+        # this is not the image name used for building the dev container.
+        imageName: ghcr.io/clearbluejar/ghidra-python
+        cacheFrom: ghcr.io/clearbluejar/ghidra-python
+        imageTag: ${{matrix.image}}
+        push: never
+        #platform: linux/amd64,linux/arm64
+        runCmd: |
+          env
+          ls /usr/local/include/
+          pip install --upgrade pip
+          # install package and testing
+          # install from pypi
+          pip install "ghidriff[testing]"
+          pip list
+          # download data to shared test data
+          if [ ! -d "tests/data" ]; then git clone https://github.com/clearbluejar/ghidriff-test-data.git tests/data; fi
+          pytest -rA
diff --git a/.github/workflows/pytest-devcontainer-repo-all.yml b/.github/workflows/pytest-devcontainer-repo-all.yml
@@ -0,0 +1,54 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Pytest Python Package Repo Devcontainer Across Versions
+
+on:
+  push:
+    branches: [ "main", "fix-workflows" ]
+    paths:
+      - 'ghidriff/**'
+      - 'tests/**'
+      - '.github/workflows/pytest-devcontainer-repo-all.yml'
+  pull_request:
+    branches: [ "main" ]
+  workflow_dispatch:
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # cover the latest and all versions of all subreleases   
+        # add "latest" when jpype is update for python 3.12     
+        image: ["10.4ghidra3.11python-bookworm","10.4ghidra3.9python-bookworm", "10.3.3ghidra3.10python-bookworm", "10.2.3ghidra3.11python-bookworm", "10.1.5ghidra3.11python-bookworm" ]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Overwrite original devcontainer with workflow devcontainer
+      run: |
+        cp .github/workflows/devcontainer.json .devcontainer/devcontainer.json 
+    - name: Test with pytest on devcontainer  
+      env:
+        # this will set the imageName in the base docker container .github/workflows/.devcontainer/devcontainer.json
+        # (which is needed because imageName for devcontainers/[email protected] doesn't affect which image you use!
+        DC_IMAGE_TAG : ${{matrix.image}}
+      uses: devcontainers/[email protected]
+      with: 
+        # this is not the image name used for building the dev container.
+        imageName: ghcr.io/clearbluejar/ghidra-python
+        cacheFrom: ghcr.io/clearbluejar/ghidra-python
+        imageTag: ${{matrix.image}}
+        push: never
+        #platform: linux/amd64,linux/arm64
+        runCmd: |
+          env
+          ls /usr/local/include/
+          pip install --upgrade pip
+          # install package and testing
+          pip install -e ".[testing]"
+          # download data to shared test data
+          if [ ! -d "tests/data" ]; then git clone https://github.com/clearbluejar/ghidriff-test-data.git tests/data; fi
+          pytest -rA
diff --git a/.github/workflows/pytest-devcontainer.yml b/.github/workflows/pytest-devcontainer.yml
@@ -1,14 +1,15 @@
 # This workflow will install Python dependencies, run tests and lint with a variety of Python versions
 # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
 
-name: Pytest Python Package In Devcontainer
+name: Pytest Python Package Devcontainer
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "main", "fix-workflows" ]
     paths:
       - 'ghidriff/**'
       - 'tests/**'
+      - '.github/workflows/pytest-devcontainer.yml'
   pull_request:
     branches: [ "main" ]
   workflow_dispatch:
@@ -17,21 +18,15 @@ jobs:
   test:
 
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        # cover the latest and all versions of all subreleases        
-        image: ["latest", "10.4ghidra3.9python-bookworm", "10.3.3ghidra3.10python-bookworm", "10.2.3ghidra3.11python-bookworm", "10.1.5ghidra3.11python-bookworm" ]
-
+
     steps:
     - uses: actions/checkout@v3
-    - name: Test with pytest on devcontainer    
+    - name: Test with pytest on devcontainer  
       uses: devcontainers/[email protected]
-      with:
-        imageName: ghcr.io/clearbluejar/ghidra-python
-        # cacheFrom: ghcr.io/clearbluejar/ghidra-python
-        imageTag: ${{matrix.image}}
+      with: 
+        cacheFrom: ghcr.io/clearbluejar/ghidra-python
         push: never
+        #platform: linux/amd64,linux/arm64
         runCmd: |
           env
           ls /usr/local/include/

diff --git a/.github/workflows/pytest-docker.yml b/.github/workflows/pytest-docker.yml
@@ -5,12 +5,13 @@ name: Pytest Python Package In Docker Image
 
 on:
   push:
-    branches: [ "main" ]
+    # branches: [ "main" ]
     paths:
-      - 'ghidriff/**'
-      - 'tests/**'
-  pull_request:
-    branches: [ "main" ]
+      # - 'ghidriff/**'
+      # - 'tests/**'
+      - '.github/workflows/pytest-docker.yml'
+  # pull_request:
+  #   branches: [ "main" ]
   workflow_dispatch:
 
 jobs:
@@ -21,7 +22,7 @@ jobs:
       fail-fast: false
       matrix:
         # cover the latest and all versions of all subreleases        
-        image: ["latest", "10.4ghidra3.9python-bookworm", "10.3.3ghidra3.10python-bookworm", "10.2.3ghidra3.11python-bookworm", "10.1.5ghidra3.11python-bookworm" ]
+        image: ["10.4ghidra3.11python-bookworm", "10.4ghidra3.9python-bookworm", "10.3.3ghidra3.10python-bookworm", "10.2.3ghidra3.11python-bookworm", "10.1.5ghidra3.11python-bookworm" ]
 
     container:
         image: ghcr.io/clearbluejar/ghidra-python:${{ matrix.image }}
@@ -30,14 +31,34 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - run: | 
-        pwd
-        ls -a        
+      with:
+        path: ghidriff
+    - run: |
+        mkdir -p /github
+        ln -s /root $HOME # Create Symlink /github/home
+        #chown -R 777 /__w
+        # python -m venv .env
+        # source .env/bin/activate        
+        # ls -a        
         env
-        ls /usr/local/include/
+        # ls /usr/local/include/
+        ls -R ~/.
         pip install --upgrade pip
-        # install package and testing
-        pip install -e ".[testing]"
+        cd ghidriff
+        # install ghidriff package and testing reqs
+        pip install ".[testing]"
+        ls -R /root
+        #pyhidra &
+        #sleep 10
+        #killall python
+        # source .env/bin/activate
+        # #init pyhidra
+        python -m pyhidra.install_plugins        
+        # pushd /tmp
+        # popd
         # download data to shared test data
         if [ ! -d "tests/data" ]; then git clone https://github.com/clearbluejar/ghidriff-test-data.git tests/data; fi
-        pytest -rA
+
+        pytest -rAv
+      shell: bash
+