Add flight sql client (#30)

* Added flight_sql_client, and renamed flight_sql executable to: flight_sql_server * Updated Python Docker image to 3.11.8 * Added gflags library to CI/Dockerfile * Fix sign/notarize * Remove signing of client app * Fixing pgrep command in test_flight_sql.sh * Fixing pgrep command in test_flight_sql.sh, again * Attempting to sign client again (with modified action) * Added mTLS support in flight_sql_client. Updated README.md with instructions on how to use the client.
voltrondata · Feb 29, 2024 · 6280bfb · 6280bfb
1 parent fff3407
commit 6280bfb
Show file tree

Hide file tree

Showing 12 changed files with 379 additions and 50 deletions.
diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml
@@ -28,29 +28,39 @@ jobs:
 
       - name: Install build requirements
         run: |
-          brew install boost
+          brew install boost gflags
 
       - name: Configure Project
         uses: threeal/[email protected]
         with:
           generator: Ninja
           run-build: true
 
-      - name: Sign and notarize the release build
-        uses: toitlang/action-macos-sign-notarize@v1.1.1
+      - name: Sign and notarize the server release build
+        uses: prmoore77/action-macos-sign-notarize@b0f525e0d98a47b0884558b786f21453889a04d7
         with:
           certificate: ${{ secrets.APPLE_CERTIFICATE }}
           certificate-password: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
           username: ${{ secrets.APPLE_ID_USERNAME }}
           password: ${{ secrets.APPLE_ID_PASSWORD }}
           apple-team-id: ${{ secrets.APPLE_TEAM_ID }}
-          app-path: build/flight_sql
+          app-path: build/flight_sql_server
           entitlements-path: macos/entitlements.plist
 
+      - name: Sign and notarize the server release build
+        uses: prmoore77/action-macos-sign-notarize@b0f525e0d98a47b0884558b786f21453889a04d7
+        with:
+          certificate: ${{ secrets.APPLE_CERTIFICATE }}
+          certificate-password: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
+          username: ${{ secrets.APPLE_ID_USERNAME }}
+          password: ${{ secrets.APPLE_ID_PASSWORD }}
+          apple-team-id: ${{ secrets.APPLE_TEAM_ID }}
+          app-path: build/flight_sql_client
+
       - name: Zip artifacts
         run: |
-          mv build/flight_sql .
-          zip -j ${{ env.zip_file_name }} flight_sql
+          mv build/flight_sql_server build/flight_sql_client .
+          zip -j ${{ env.zip_file_name }} flight_sql_server flight_sql_client
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
@@ -86,7 +96,8 @@ jobs:
             cmake \
             gcc \
             git \
-            libboost-all-dev
+            libboost-all-dev \
+            libgflags-dev
           sudo apt-get clean
           sudo rm -rf /var/lib/apt/lists/*
 
@@ -98,8 +109,8 @@ jobs:
 
       - name: Zip artifacts
         run: |
-          mv build/flight_sql .
-          zip -j ${{ env.zip_file_name }} flight_sql
+          mv build/flight_sql_server build/flight_sql_client .
+          zip -j ${{ env.zip_file_name }} flight_sql_server flight_sql_client
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v4

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -134,8 +134,6 @@ target_include_directories(flightsqlserver PRIVATE
 target_link_libraries(flightsqlserver
         PRIVATE
         Threads::Threads
-        Arrow::arrow_static
-        ArrowFlight::arrow_flight_static
         ArrowFlightSql::arrow_flight_sql_static
         sqlite
         duckdb
@@ -156,18 +154,43 @@ install(TARGETS flightsqlserver
         PUBLIC_HEADER DESTINATION include
 )
 
-# ------------ Executable section ------------
-add_executable(flight_sql
-        src/flight_sql.cpp
+# ------------ Server Executable section ------------
+add_executable(flight_sql_server
+        src/flight_sql_server.cpp
 )
 
-target_link_libraries(flight_sql PRIVATE
+target_link_libraries(flight_sql_server PRIVATE
         flightsqlserver
         ${Boost_LIBRARIES}
 )
 
-target_compile_options(flight_sql PRIVATE "-static")
+target_compile_options(flight_sql_server PRIVATE "-static")
 
-install(TARGETS flight_sql
+install(TARGETS flight_sql_server
+        DESTINATION bin
+)
+
+# ------------ Client Executable section ------------
+add_executable(flight_sql_client
+        src/flight_sql_client.cpp
+)
+
+target_link_libraries(flight_sql_client PRIVATE
+        Threads::Threads
+        ArrowFlightSql::arrow_flight_sql_static
+        ${Boost_LIBRARIES}
+        "-lresolv"
+)
+
+if (APPLE)
+    # macOS-specific libraries and options
+    target_link_libraries(flight_sql_client PRIVATE "-framework CoreFoundation")
+elseif (UNIX AND NOT APPLE)
+    target_link_libraries(flight_sql_client PRIVATE "-lssl -lcrypto")
+endif ()
+
+target_compile_options(flight_sql_client PRIVATE "-static")
+
+install(TARGETS flight_sql_client
         DESTINATION bin
 )
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.11.7
+FROM python:3.11.8
 
 ARG TARGETPLATFORM
 ARG TARGETARCH
@@ -19,6 +19,7 @@ RUN apt-get update && \
     git \
     ninja-build \
     libboost-all-dev \
+    libgflags-dev \
     sqlite3 \
     vim && \
     apt-get clean && \

diff --git a/Dockerfile.ci b/Dockerfile.ci
@@ -1,4 +1,4 @@
-FROM python:3.11.7
+FROM python:3.11.8
 
 ARG TARGETPLATFORM
 ARG TARGETARCH
@@ -72,9 +72,11 @@ RUN python "scripts/create_duckdb_database_file.py" \
            --overwrite-file=true \
            --scale-factor=0.01
 
-COPY --chown=app_user:app_user flight_sql /usr/local/bin/flight_sql
+COPY --chown=app_user:app_user flight_sql_server /usr/local/bin/flight_sql_server
+RUN chmod +x /usr/local/bin/flight_sql_server
 
-RUN chmod +x /usr/local/bin/flight_sql
+COPY --chown=app_user:app_user flight_sql_client /usr/local/bin/flight_sql_client
+RUN chmod +x /usr/local/bin/flight_sql_client
 
 COPY --chown=app_user:app_user tls tls
 

diff --git a/README.md b/README.md
@@ -171,6 +171,36 @@ n_nationkey: [[24]]
 n_name: [["UNITED STATES"]]
 ```
 
+### Connecting via the new `flight_sql_client` CLI tool
+You can also use the new `flight_sql_client` CLI tool to connect to the Flight SQL server, and then run a single command.  This tool is built into the Docker image, and is also available as a standalone executable for Linux and MacOS.   
+
+Example (run from the host computer's terminal):
+```bash
+flight_sql_client \
+  --command Execute \
+  --host "localhost" \
+  --port 31337 \
+  --username "flight_username" \
+  --password "flight_password" \
+  --query "SELECT version()" \
+  --use-tls \
+  --tls-skip-verify
+```
+
+That should return:
+```text
+Results from endpoint 1 of 1
+Schema:
+version(): string
+
+Results:
+version():   [
+    "v0.10.0"
+  ]
+
+Total: 1
+```
+
 ### Tear-down
 Stop the docker image with:
 ```bash
@@ -179,20 +209,20 @@ docker stop flight-sql
 
 ## Option 2 - Download and run the flight_sql CLI executable
 
-Download (and unzip) the latest release of the **flight_sql** CLI executable from these currently supported platforms:   
+Download (and unzip) the latest release of the **flight_sql_server** CLI executable from these currently supported platforms:   
 [Linux x86-64](https://github.com/voltrondata/flight-sql-server-example/releases/latest/download/flight_sql_cli_linux_amd64.zip)   
 [Linux arm64](https://github.com/voltrondata/flight-sql-server-example/releases/latest/download/flight_sql_cli_linux_arm64.zip)   
 [MacOS x86-64](https://github.com/voltrondata/flight-sql-server-example/releases/latest/download/flight_sql_cli_macos_amd64.zip)   
 [MacOS arm64](https://github.com/voltrondata/flight-sql-server-example/releases/latest/download/flight_sql_cli_macos_arm64.zip)   
 
 Then from a terminal - you can run:
 ```bash
-FLIGHT_PASSWORD="flight_password" flight_sql --database-filename data/some_db.duckdb --print-queries
+FLIGHT_PASSWORD="flight_password" flight_sql_server --database-filename data/some_db.duckdb --print-queries
 ```
 
 To see all program options - run:
 ```bash
-flight_sql --help
+flight_sql_server --help
 ```
 
 ## Option 3 - Steps to build the solution manually
@@ -243,14 +273,14 @@ popd
 
 6. Start the Flight SQL server (and print client SQL commands as they run using the --print-queries option)
 ```bash
-FLIGHT_PASSWORD="flight_password" flight_sql --database-filename data/TPC-H-small.duckdb --print-queries
+FLIGHT_PASSWORD="flight_password" flight_sql_server --database-filename data/TPC-H-small.duckdb --print-queries
 ```
 
 ## Selecting different backends
 This option allows choosing from two backends: SQLite and DuckDB. It defaults to DuckDB.
 
 ```bash
-$ FLIGHT_PASSWORD="flight_password" flight_sql --database-filename data/TPC-H-small.duckdb
+$ FLIGHT_PASSWORD="flight_password" flight_sql_server --database-filename data/TPC-H-small.duckdb
 Apache Arrow version: 15.0.0
 WARNING - TLS is disabled for the Flight SQL server - this is insecure.
 DuckDB version: v0.10.0
@@ -264,14 +294,14 @@ Apache Arrow Flight SQL server - with engine: DuckDB - will listen on grpc+tcp:/
 Flight SQL server - started
 ```
 
-The above call is equivalent to running `flight_sql -B duckdb` or `flight_sql --backend duckdb`. To select SQLite run
+The above call is equivalent to running `flight_sql_server -B duckdb` or `flight_sql --backend duckdb`. To select SQLite run
 
 ```bash
-FLIGHT_PASSWORD="flight_password" flight_sql -B sqlite -D data/TPC-H-small.sqlite 
+FLIGHT_PASSWORD="flight_password" flight_sql_server -B sqlite -D data/TPC-H-small.sqlite 
 ```
 or 
 ```bash
-FLIGHT_PASSWORD="flight_password" flight_sql --backend sqlite --database-filename data/TPC-H-small.sqlite
+FLIGHT_PASSWORD="flight_password" flight_sql_server --backend sqlite --database-filename data/TPC-H-small.sqlite
 ```
 The above will produce the following:
 
@@ -286,10 +316,10 @@ Flight SQL server - started
 ```
 
 ## Print help
-To see all the available options run `flight_sql --help`.
+To see all the available options run `flight_sql_server --help`.
 
 ```bash
-flight_sql --help
+flight_sql_server --help
 Allowed options:
   --help                              produce this help message
   --version                           Print the version and exit

diff --git a/requirements.txt b/requirements.txt
@@ -2,5 +2,5 @@ pandas==2.1.*
 duckdb==0.10.0
 click==8.1.*
 pyarrow==15.0.0
-adbc-driver-flightsql==0.9.*
-adbc-driver-manager==0.9.*
+adbc-driver-flightsql==0.10.*
+adbc-driver-manager==0.10.*
diff --git a/scripts/start_flight_sql.sh b/scripts/start_flight_sql.sh
@@ -30,4 +30,4 @@ then
   PRINT_QUERIES_FLAG="--print-queries"
 fi
 
-flight_sql --backend="${L_DATABASE_BACKEND}" --database-filename="${L_DATABASE_FILENAME}" ${TLS_ARG} ${PRINT_QUERIES_FLAG}
+flight_sql_server --backend="${L_DATABASE_BACKEND}" --database-filename="${L_DATABASE_FILENAME}" ${TLS_ARG} ${PRINT_QUERIES_FLAG}
diff --git a/scripts/test_flight_sql.py b/scripts/test_flight_sql.py
@@ -1,17 +1,39 @@
 import os
+from time import sleep
+import pyarrow
 from adbc_driver_flightsql import dbapi as flight_sql, DatabaseOptions
 
-flight_password = os.getenv("FLIGHT_PASSWORD")
 
-with flight_sql.connect(uri="grpc+tls://localhost:31337",
-                        db_kwargs={"username": "flight_username",
-                                   "password": flight_password,
-                                   DatabaseOptions.TLS_SKIP_VERIFY.value: "true"  # Not needed if you use a trusted CA-signed TLS cert
-                                   }
-                        ) as conn:
-    with conn.cursor() as cur:
-        cur.execute("SELECT n_nationkey, n_name FROM nation WHERE n_nationkey = ?",
-                    parameters=[24]
-                    )
-        x = cur.fetch_arrow_table()
-        print(x)
+# Setup variables
+max_attempts: int = 10
+sleep_interval: int = 10
+flight_password = os.environ["FLIGHT_PASSWORD"]
+
+def main():
+    for attempt in range(max_attempts):
+        try:
+            with flight_sql.connect(uri="grpc+tls://localhost:31337",
+                                    db_kwargs={"username": "flight_username",
+                                               "password": flight_password,
+                                               DatabaseOptions.TLS_SKIP_VERIFY.value: "true"  # Not needed if you use a trusted CA-signed TLS cert
+                                               }
+                                    ) as conn:
+                with conn.cursor() as cur:
+                    cur.execute("SELECT n_nationkey, n_name FROM nation WHERE n_nationkey = ?",
+                                parameters=[24]
+                                )
+                    x = cur.fetch_arrow_table()
+                    print(x)
+        except Exception as e:
+            if attempt == max_attempts - 1:
+                raise e
+            else:
+                print(f"Attempt {attempt + 1} failed: {e}, sleeping for {sleep_interval} seconds")
+                sleep(sleep_interval)
+        else:
+            print("Success!")
+            break
+
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/test_flight_sql.sh b/scripts/test_flight_sql.sh
@@ -17,8 +17,8 @@ started="0"
 
 # Check if the process is running
 while [ $elapsed_time -lt $timeout_limit ]; do
-    # Check if the process is running using --exact to match the whole process name
-    if pgrep --exact "flight_sql" > /dev/null; then
+    # Check if the process is running
+    if pgrep "flight_sql" > /dev/null; then
         echo "Flight SQL Server process started successfully!"
         started="1"
         # Sleep for a few more seconds...