diff --git a/.cirrus.yml b/.cirrus.yml
index 1b680ba4..56ec5ced 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -20,6 +20,7 @@ task:
           gcc \
           g++ \
           libboost-dev \
+          libcgal-dev \
           libmpfr-dev \
           libgmp-dev \
           swig \
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e36aef5e..31986aea 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -48,12 +48,14 @@ jobs:
       - name: Install compiler tools on macOS
         if: runner.os == 'macOS'
         run: |
-          brew install make automake swig gmp mpfr boost
+          brew install make automake swig gmp mpfr boost cgal
           export PATH="/usr/local/opt/make/libexec/gnubin:$PATH"
 
       - name: Install extra deps on Linux
         if: runner.os == 'Linux'
-        run: sudo apt-get update && sudo apt-get install -y libboost-dev libmpfr-dev swig autoconf libtool
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libboost-dev libcgal-dev libmpfr-dev swig autoconf libtool
 
       - name: Install package
         run: |
@@ -89,7 +91,7 @@ jobs:
       - name: Install compiler tools on macOS
         if: runner.os == 'macOS'
         run: |
-          brew install make automake swig mpfr boost
+          brew install make automake swig mpfr boost cgal
           export PATH="/usr/local/opt/make/libexec/gnubin:$PATH"
 
       - name: Clone gmp
diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
index 703eb050..8d80a2b6 100644
--- a/.github/workflows/wheels.yml
+++ b/.github/workflows/wheels.yml
@@ -62,7 +62,7 @@ jobs:
       - name: Install compiler tools on macOS
         if: runner.os == 'macOS'
         run: |
-          brew install make automake swig mpfr boost
+          brew install make automake swig mpfr boost cgal
           export PATH="/usr/local/opt/make/libexec/gnubin:$PATH"
 
       - name: Clone gmp
@@ -112,7 +112,9 @@ jobs:
 
       - name: Install extra deps on Linux
         if: runner.os == 'Linux'
-        run: sudo apt-get update && sudo apt-get install -y libboost-dev swig autoconf libtool
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libboost-dev libcgal-dev swig autoconf libtool
 
       - name: test sdist
         run: python -m pip install dist/*.tar.gz
diff --git a/.gitignore b/.gitignore
index fa91a4c6..0adb767f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -172,6 +172,3 @@ cython_debug/
 *.exe
 *.out
 *.app
-
-# Build dependencies
-CGAL*
diff --git a/README.md b/README.md
index 78f5e8e0..2fa5e6cc 100644
--- a/README.md
+++ b/README.md
@@ -67,7 +67,7 @@ To install the build-time dependencies run the following installation commands f
 ### Debian/Ubuntu
 
 ``` bash
-sudo apt-get update && sudo apt-get install -y libboost-dev libmpfr-dev libgmp-dev swig autoconf libtool
+sudo apt-get update && sudo apt-get install -y libboost-dev libcgal-dev libmpfr-dev libgmp-dev swig autoconf libtool
 ```
 
 ## Build and install
diff --git a/pyproject.toml b/pyproject.toml
index fad0ad45..4d1cfdf8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -17,9 +17,12 @@ manylinux-i686-image = "manylinux2014"
 before-all = [
     "yum update -y",
     "yum install -y mpfr-devel",
-    "curl -L https://boostorg.jfrog.io/artifactory/main/release/1.84.0/source/boost_1_84_0.tar.bz2 -o boost_1_84_0.tar.bz2",
+    "curl -LO https://boostorg.jfrog.io/artifactory/main/release/1.84.0/source/boost_1_84_0.tar.bz2",
     "tar --bzip2 -xf boost_1_84_0.tar.bz2",
     "mv boost_1_84_0/boost /usr/include/boost",
+    "curl -LO https://github.com/CGAL/cgal/releases/download/v5.6.1/CGAL-5.6.1-library.zip",
+    "unzip -q CGAL-5.6.1-library.zip",
+    "mv CGAL-5.6.1/include/CGAL /usr/include/",
 ]
 # Skip musllinux builds for the moment
 skip = "*-musllinux_*"
diff --git a/setup.py b/setup.py
index 56971382..674dd72e 100644
--- a/setup.py
+++ b/setup.py
@@ -15,14 +15,10 @@
 import subprocess
 import sys
 import sysconfig
-import urllib.request
-import zipfile
 
 import setuptools.command.build_ext
 import setuptools.command.install
 
-CGAL_ZIP = "https://github.com/CGAL/cgal/releases/download/v5.6/CGAL-5.6-library.zip"
-
 DIR = pathlib.Path(__file__).parent.resolve()
 FASTJET = DIR / "extern" / "fastjet-core"
 FASTJET_CONTRIB = DIR / "extern" / "fastjet-contrib"
@@ -56,16 +52,6 @@ def get_version() -> str:
 class FastJetBuild(setuptools.command.build_ext.build_ext):
     def build_extensions(self):
         if not OUTPUT.exists():
-            zip_filename = DIR / pathlib.Path(CGAL_ZIP).parts[-1]
-
-            with urllib.request.urlopen(CGAL_ZIP) as http_obj:
-                with open(zip_filename, "wb") as file_obj:
-                    shutil.copyfileobj(http_obj, file_obj)
-
-            with zipfile.ZipFile(zip_filename) as zip_obj:
-                cgal_dir = DIR / zip_obj.namelist()[0]
-                zip_obj.extractall(DIR)
-
             # Patch for segfault of LimitedWarning
             # For more info see https://github.com/scikit-hep/fastjet/pull/131
             subprocess.run(
@@ -92,7 +78,6 @@ def build_extensions(self):
                 "--enable-allcxxplugins",
                 "--enable-cgal-header-only",
                 "--enable-cgal",
-                f"--with-cgaldir={cgal_dir}",
                 "--enable-swig",
                 "--enable-pyext",
                 f'LDFLAGS={env["LDFLAGS"]}',