fix: Decouple go feature server from gopy bindings (#118)

* fix: Decouple go feature server from gopy bindings --------- Co-authored-by: Bhargav Dodla <[email protected]>
ExpediaGroup · Jul 23, 2024 · b67ade0 · b67ade0
1 parent 99f332a
commit b67ade0
Show file tree

Hide file tree

Showing 32 changed files with 565 additions and 898 deletions.
diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
@@ -116,7 +116,7 @@ jobs:
     needs: get-version
     strategy:
       matrix:
-        component: [feature-server, feature-server-java, feature-transformation-server]
+        component: [feature-server, feature-server-java, feature-transformation-server, feature-server-go]
     env:
       REGISTRY: feastdev
     steps:

diff --git a/.github/workflows/go_pr.yml b/.github/workflows/go_pr.yml
@@ -0,0 +1,85 @@
+name: go_pr
+
+on:
+  pull_request_target:
+    types:
+      - opened
+      - synchronize
+      - labeled
+
+jobs:
+  lint-go:
+    # when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes.
+    if:
+      ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) ||
+      (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) &&
+      github.repository == 'ExpediaGroup/feast'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # pull_request_target runs the workflow in the context of the base repo
+          # as such actions/checkout needs to be explicit configured to retrieve
+          # code from the PR.
+          ref: refs/pull/${{ github.event.pull_request.number }}/merge
+          submodules: recursive
+      - name: Setup Go
+        id: setup-go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.22.5
+      - name: Lint go
+        run: make lint-go
+
+  unit-test-go:
+    # when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes.
+    if:
+      ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) ||
+      (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) &&
+      github.repository == 'ExpediaGroup/feast'
+    runs-on: ubuntu-latest
+    needs: lint-go
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # pull_request_target runs the workflow in the context of the base repo
+          # as such actions/checkout needs to be explicit configured to retrieve
+          # code from the PR.
+          ref: refs/pull/${{ github.event.pull_request.number }}/merge
+          submodules: recursive
+      - name: Setup Go
+        id: setup-go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.22.5
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        id: setup-python
+        with:
+          python-version: "3.11"
+          architecture: x64
+      - name: Test Go
+        run: make test-go
+      - uses: actions/upload-artifact@v3
+        with:
+          name: go-coverage-report
+          path: ${{ github.workspace }}/coverage.html
+
+  build-docker-image-go:
+    # when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes.
+    if:
+      ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) ||
+      (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) &&
+      github.repository == 'ExpediaGroup/feast'
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        component: [ feature-server-go ]
+    env:
+      REGISTRY: gcr.io/kf-feast
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'true'
+      - name: Build image
+        run: make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA}
diff --git a/.github/workflows/lint_pr.yml b/.github/workflows/lint_pr.yml
@@ -14,7 +14,7 @@ permissions:
 jobs:
   validate-title:
     if:
-      github.repository == 'feast-dev/feast'
+      github.repository != 'feast-dev/feast'
     name: Validate PR title
     runs-on: ubuntu-latest
     steps:

diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml
@@ -15,21 +15,8 @@ jobs:
         with:
           python-version: "3.11"
           architecture: x64
-      - name: Setup Go
-        id: setup-go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.19.7
       - name: Install uv
         run: curl -LsSf https://astral.sh/uv/install.sh | sh
-      - name: Install apache-arrow on ubuntu
-        run: |
-          sudo apt update
-          sudo apt install -y -V ca-certificates lsb-release wget
-          wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
-          sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
-          sudo apt update
-          sudo apt install -y -V "libarrow-dev=11.0.0-1"
       - name: Install dependencies
         run: |
           make install-python-ci-dependencies-uv
@@ -39,27 +26,11 @@ jobs:
   lint-go:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Setup Go
         id: setup-go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.19.7
-      - name: Setup Python
-        id: setup-python
-        uses: actions/setup-python@v2
-        with:
-          python-version: "3.8"
-      - name: Upgrade pip version
-        run: |
-          pip install --upgrade pip
-      - name: Install apache-arrow on ubuntu
-        run: |
-            sudo apt update
-            sudo apt install -y -V ca-certificates lsb-release wget
-            wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
-            sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
-            sudo apt update
-            sudo apt install -y -V "libarrow-dev=11.0.0-1"
+          go-version: 1.22.5
       - name: Lint go
-        run: make lint-go
+        run: make lint-go
diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml
@@ -52,11 +52,6 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           architecture: x64
-      - name: Setup Go
-        id: setup-go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.19.7
       - name: Authenticate to Google Cloud
         uses: 'google-github-actions/auth@v1'
         with:
@@ -75,20 +70,6 @@ jobs:
           aws-region: us-west-2
       - name: Use AWS CLI
         run: aws sts get-caller-identity
-      - name: Install apache-arrow on ubuntu
-        if: matrix.os == 'ubuntu-latest'
-        run: |
-            sudo apt update
-            sudo apt install -y -V ca-certificates lsb-release wget
-            wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
-            sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
-            sudo apt update
-            sudo apt install -y -V "libarrow-dev=11.0.0-1"
-      - name: Install apache-arrow on macos
-        if: matrix.os == 'macOS-latest'
-        run: |
-            brew install apache-arrow
-            brew install pkg-config
       - name: Install uv
         run: curl -LsSf https://astral.sh/uv/install.sh | sh
       - name: Get uv cache dir

diff --git a/.github/workflows/pr_local_integration_tests.yml b/.github/workflows/pr_local_integration_tests.yml
@@ -38,12 +38,6 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           architecture: x64
-
-      - name: Setup Go
-        id: setup-go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.19.7
       - name: Install uv
         run: curl -LsSf https://astral.sh/uv/install.sh | sh
       - name: Get uv cache dir
@@ -55,24 +49,9 @@ jobs:
         with:
           path: ${{ steps.uv-cache.outputs.dir }}
           key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
-      - name: Install apache-arrow on ubuntu
-        if: matrix.os == 'ubuntu-latest'
-        run: |
-            sudo apt update
-            sudo apt install -y -V ca-certificates lsb-release wget
-            wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
-            sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
-            sudo apt update
-            sudo apt install -y -V "libarrow-dev=11.0.0-1"
 
       - name: Install dependencies
         run: make install-python-ci-dependencies-uv
-      - name: Compile Go along with Extensions
-        run: |
-          make install-go-proto-dependencies 
-          make install-go-ci-dependencies
-          COMPILE_GO=true python setup.py develop
-          CGO_LDFLAGS_ALLOW=".*" COMPILE_GO=True python setup.py build_ext --inplace
       - name: Test local integration tests
         if: ${{ always() }}  # this will guarantee that step won't be canceled and resources won't leak
-        run: make test-python-integration-local
+        run: make test-python-integration-local
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -49,7 +49,7 @@ jobs:
     needs: [get-version, publish-python-sdk]
     strategy:
       matrix:
-        component: [feature-server, feature-server-java, feature-transformation-server, feast-operator]
+        component: [feature-server, feature-server-java, feature-transformation-server, feast-operator, feature-server-go]
     env:
       MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar
       REGISTRY: feastdev

diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
@@ -23,21 +23,6 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           architecture: x64
-      - name: Setup Go
-        id: setup-go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.19.7
-      - name: Install apache-arrow on ubuntu
-        if: matrix.os == 'ubuntu-latest'
-        run: |
-            sudo apt update
-            sudo apt install -y -V ca-certificates lsb-release wget
-            wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
-            sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
-            sudo apt update
-            sudo apt install -y -V "libarrow-dev=11.0.0-1"
-            sudo apt install -y -V pkg-config
       - name: Install uv
         run: |
           curl -LsSf https://astral.sh/uv/install.sh | sh
@@ -67,7 +52,7 @@ jobs:
         run: make test-python-unit
       - name: Test Python for Mac
         if: matrix.os != 'ubuntu-latest'
-        run: python -m pytest -n 8 --color=yes sdk/python/tests --ignore=sdk/python/tests/expediagroup --ignore=sdk/python/tests/integration/e2e/test_go_feature_server.py
+        run: python -m pytest -n 8 --color=yes sdk/python/tests --ignore=sdk/python/tests/expediagroup
 
   unit-test-go:
     runs-on: ${{ matrix.os }}
@@ -81,6 +66,11 @@ jobs:
       PYTHON: ${{ matrix.python-version }}
     steps:
       - uses: actions/checkout@v4
+      - name: Setup Go
+        id: setup-go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.22.5
       - name: Setup Python
         id: setup-python
         uses: actions/setup-python@v5
@@ -90,20 +80,6 @@ jobs:
       - name: Upgrade pip version
         run: |
           pip install --upgrade "pip>=22.1,<23"
-      - name: Setup Go
-        id: setup-go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.19.7
-      - name: Install apache-arrow on ubuntu
-        run: |
-            sudo apt update
-            sudo apt install -y -V ca-certificates lsb-release wget
-            wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
-            sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
-            sudo apt update
-            sudo apt install -y -V "libarrow-dev=11.0.0-1"
-            sudo apt install -y -V pkg-config
       - name: Test
         run: make test-go
 
@@ -125,4 +101,4 @@ jobs:
         run: yarn build:lib
       - name: Run yarn tests
         working-directory: ./ui
-        run: yarn test --watchAll=false
+        run: yarn test --watchAll=false