27 - Middleware for distributed policies (#139)

* Endpoint for listing managed resources (i.e. resources for which there is
  an `acl:Control` policy)
* Middleware for policy distribution
* Basic CI test for policy distribution

Co-authored-by: Cerfoglg <[email protected]>
Co-authored-by: Federico M. Facca <[email protected]>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

.env

@@ -6,7 +6,7 @@ REACT_APP_URI=http://localhost:3000/
 REACT_APP_ANUBIS_API_URL=http://localhost:8085/
 REACT_APP_CONFIGURATION_API_URL=http://localhost:4000/configuration
 REACT_APP_OIDC_ISSUER=http://localhost:8080/realms/default
-REACT_APP_OIDC_CLIENT=ngsi
+REACT_APP_OIDC_CLIENT=configure
 REACT_APP_OIDC_SCOPE="openid profile email"
 MONGO_DB=mongodb://mongo:27017/graphql
 CONFIGURATION_API_PORT=4000

.github/workflows/docker.yml

@@ -58,3 +58,49 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
+  anubis-middleware:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          # list of Docker images to use as base name for tags
+          images: |
+            orchestracities/anubis-middleware
+          # generate Docker tags based on the following events/attributes
+          tags: |
+            type=edge,branch=master
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Login to DockerHub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v1 
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: policy-governance-middleware
+          platforms: linux/amd64,linux/arm64
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/openapi-spec.yml

@@ -14,6 +14,11 @@ jobs:
         run: |
           docker build -t orchestracities/anubis:latest .
           docker run -p 127.0.0.1:8085:8000/tcp --name anubis -d orchestracities/anubis:latest
+      - name: Start middleware
+        working-directory: ./policy-governance-middleware
+        run: |
+          docker build -t orchestracities/anubis-middleware:latest .
+          docker run -p 127.0.0.1:8098:8098/tcp --name middleware -d orchestracities/anubis-middleware
       - name: Update Open API spec
         working-directory: ./scripts
         run: |
@@ -22,9 +27,16 @@ jobs:
         working-directory: .
         run: |
           npm install -g openapi-markdown
-          openapi-markdown -i open-api-spec/openapi.json -o docs/user/walkthrough.md
+          openapi-markdown -i open-api-spec/api-manager/openapi.json -o docs/user/walkthrough.md
           sed -i '1 i\
           <!-- markdownlint-disable -->' docs/user/walkthrough.md
+      - name: Generate walktrhough
+        working-directory: .
+        run: |
+          npm install -g openapi-markdown
+          openapi-markdown -i open-api-spec/middleware/openapi.json -o docs/user/walkthrough-middleware.md
+          sed -i '1 i\
+          <!-- markdownlint-disable -->' docs/user/walkthrough-middleware.md
       - name: Commit changes
         uses: stefanzweifel/git-auto-commit-action@v4
         with:

.github/workflows/policy-api-tests.yml

@@ -65,10 +65,18 @@ jobs:
         working-directory: ./scripts
         run: |
           ./test_audit_logs.sh
-      - name: Clean up
+  e2e-middleware:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Setup Middleware Demo
+        working-directory: ./scripts
+        run: |
+          ./run_demo_with_middleware.sh
+      - name: Test Middleware
         working-directory: ./scripts
         run: |
-          ./clean.sh
+          ./test_middleware.sh
   # opa:
   #   runs-on: ubuntu-latest
   #   steps:

.gitignore

@@ -9,3 +9,8 @@ anubis-management-api/.idea/*
 .idea/*
 *.json
 *.bin
+policy-governance-middleware/node_modules/
+policy-governance-middleware/data.json
+.idea
+*/*.pyc
+*.pyc

README.md

@@ -123,6 +123,56 @@ specifically for the [NGSIv2 Context Broker](https://fiware-orion.readthedocs.io
 Anubis management, and JWT-based authentication. You can see Orion rules in this
 [rego file](config/opa-service/rego/context_broker_policy.rego).
 
+## Policy Distribution
+
+The policy distribution architecture relies on [libp2p](https://libp2p.io/)
+middleware to distribute policies across differed Policies Administration
+Points. The architecture decouples the PAP from the distribution middleware.
+This allows:
+
+- different PAP to share the same distribution node.
+- deployment without the distribution functionalities (and hence
+with a smaller footprint), when this is not required.
+
+The distribution middleware is called Policy Distribution Point.
+
+```ascii
+    ┌──────────────┐        ┌──────────────┐
+    │   Policy     │        │    Policy    │
+    │ Distribution │◄──────►│Administration│
+    │   Point 1    │        │    Point 1   │
+    └──────────────┘        └──────────────┘
+           ▲
+         2 │
+           ▼
+    ┌──────────────┐        ┌──────────────┐
+    │   Policy     │        │    Policy    │
+    │ Distribution │◄──────►│Administration│
+    │   Point 2    │        │    Point 2   │
+    └──────────────┘        └──────────────┘
+```
+
+There are two distribution modalities:
+
+- *public*, i.e. when the different middleware belong to different
+  organisations in the public internet. In this case:
+
+  - resources are considered to be univocally identifiable (if they have
+    the same id they are the same resource);
+
+  - only user specific policies are distributed;
+
+  - only resource specific policies are distributed.
+
+- *private*, i.e. when the different middleware belong to the same
+  organisation. In this case:
+
+  - resources are considered to be univocally identifiable only within the same
+    service and service path;
+
+  - all policies are distributed (including the ones for roles and groups and
+    `*` and `default` resource policies).
+
 ## Policies
 
 The formal policy specification is defined by the [oc-acl vocabulary](https://github.com/orchestracities/anubis-vocabulary/blob/master/oc-acl.ttl)
@@ -212,8 +262,31 @@ $ cd scripts
 $ ./clean.sh
 ```
 
-In case you are using an ARM64 based architecture, before running the scripts,
-use the proper image (see comment in [docker-compose](docker-compose.yaml)).
+#### Demo for distributed policy management
+
+To deploy the demo that includes two instances of the Auth API,
+two instances of the distribution middleware (plus as well OPA, Keycloak,
+and a Context Broker), run the following script:
+
+```bash
+$ cd scripts
+$ ./run_demo_with_middleware.sh
+```
+
+You can run a script to make a few test API calls. You can run the test
+script with:
+
+```bash
+$ cd scripts
+$ ./test_middleware.sh
+```
+
+To clean up the deployment after you're done, run:
+
+```bash
+$ cd scripts
+$ ./clean.sh
+```
 
 ## Installation
 
@@ -268,6 +341,8 @@ For the policy API, the following env variables are also available:
 - `KEYCLOACK_ADMIN_ENDPOINT`: The endpoint of the admin api of Keycloak.
 - `DB_TYPE`: The database type to be used by the API. Valid options for
   now are `postgres` and `sqlite`.
+- `MIDDLEWARE_ENDPOINT`: The endpoint of the policy distribution middleware
+  (if `None` the policy distribution is disabled).
 
 If postgres is the database being used, the following variables are available
 as well:
@@ -277,6 +352,14 @@ as well:
 - `DB_PASSWORD`: The password for the database user.
 - `DB_NAME`: The name of the database.
 
+The policy distribution middleware is an add-on the basic Anubis deployment.
+The following environment variables can be configured:
+
+- `SERVER_PORT`: The port where the middleware API is exposed.
+- `ANUBIS_API_URI`: The anubis management api instance linked to the middleware.
+- `LISTEN_ADDRESS`: The multiaddress format address the middleware listens on.
+- `IS_PRIVATE_ORG`: The middleware modality to public or private.
+
 For customizing the default policies that are created alongside a tenant, see
 [the configuration file](config/opa-service/default_policies.yml) that's mounted
 as a volume in the policy-api service from the

RELEASE_NOTES.md

@@ -4,6 +4,10 @@
 
 ### New features
 
+- Endpoint for listing managed resources (i.e. resources for which there is
+  an `acl:Control` policy)
+- Middleware for policy distribution
+
 ### Bug fixes
 
 ### Documentation

anubis-management-api/anubis/main.py

@@ -5,6 +5,7 @@
 from anubis.policies import models as p_models
 from anubis.audit import routers as a
 from anubis.audit import models as a_models
+from anubis.middleware import routers as m
 from anubis.version import ANUBIS_VERSION
 from fastapi.openapi.utils import get_openapi
 from anubis.database import engine
@@ -49,6 +50,7 @@
 app.include_router(t.router)
 app.include_router(p.router)
 app.include_router(a.router)
+app.include_router(m.router)
 
 
 @app.get("/", summary="Return Anubis API endpoints")
@@ -64,6 +66,7 @@ async def v1_root():
     return {
         "tenants_url": "/v1/tenants",
         "policies_url": "/v1/policies",
+        "middleware_url": "/v1/middleware",
         "audit_url": "/v1/audit"}
 
 

anubis-management-api/anubis/middleware/operations.py

@@ -0,0 +1,68 @@
+from sqlalchemy.orm import Session
+
+import anubis.default as default
+from ..policies import models as pm
+from ..tenants import models as tm
+
+
+# TODO it would be good to have also the list of owners, but query needs
+# to be defined
+def get_resources(
+        db: Session,
+        tenant: str = None,
+        service_path: str = None,
+        resource: str = None,
+        resource_type: str = None,
+        owner: str = None,
+        skip: int = 0,
+        limit: int = 100):
+    db_policies = db.query(
+        pm.Policy.access_to,
+        pm.Policy.resource_type,
+        tm.ServicePath.path,
+        tm.Tenant.name).distinct().join(
+        pm.Policy.mode).filter(
+            pm.Mode.iri == default.CONTROL_MODE_IRI).join(
+                pm.Policy.service_path).join(
+        tm.ServicePath.tenant)
+    if resource:
+        db_policies = db_policies.filter(
+            pm.Policy.access_to == resource)
+    if resource_type:
+        db_policies = db_policies.filter(
+            pm.Policy.resource_type == resource_type)
+    if tenant:
+        db_policies = db_policies.filter(
+            tm.Tenant.name == tenant)
+    if service_path:
+        db_policies = db_policies.filter(
+            tm.ServicePath.path == service_path)
+    if owner:
+        db_policies = db_policies.join(
+            pm.Policy.agent).filter(pm.Agent.iri == owner)
+    return db_policies.offset(skip).limit(limit).all()
+
+
+def get_policies(
+        db: Session,
+        tenant: str = None,
+        service_path: str = None,
+        resource: str = None,
+        resource_type: str = None,
+        skip: int = 0,
+        limit: int = 100):
+    db_policies = db.query(pm.Policy)
+    if resource:
+        db_policies = db_policies.filter(
+            pm.Policy.access_to == resource)
+    if resource_type:
+        db_policies = db_policies.filter(
+            pm.Policy.resource_type == resource_type)
+    if tenant:
+        db_policies = db_policies.join(pm.Policy.service_path).join(
+            tm.ServicePath.tenant).filter(
+            tm.Tenant.name == tenant)
+    if service_path:
+        db_policies = db_policies.join(pm.Policy.service_path).filter(
+            tm.ServicePath.path == service_path)
+    return db_policies.offset(skip).limit(limit).all()