Skip to content

Commit

Permalink
Remove spec_hash and allow the use of IDs instead
Browse files Browse the repository at this point in the history 
Rather than provide spec_hash, the POST endpoint now accepts a parent
policy ID and/or policy ID. This saves on network bandwidth and resource
utilization when the ID is known.

This also changes spec to be JSONB instead of a JSON string. This allows
a unique constraint without serializing it in a special way with map
keys sorted.

Signed-off-by: mprahl <[email protected]>
(cherry picked from commit 7f05e96)
  • Loading branch information
@mprahl
mprahl authored and Magic Mirror committed Jan 5, 2024
1 parent edbd7be commit 4fa3772
Show file tree
Hide file tree
Showing 5 changed files with 291 additions and 270 deletions.
28 changes: 13 additions & 15 deletions controllers/complianceeventsapi/migrations/000001_compliance_history_initial_tables.up.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,40 +21,38 @@ CREATE TABLE IF NOT EXISTS parent_policies(

-- This is required until we only support Postgres 15+ to utilize NULLS NOT DISTINCT.
-- Partial indexes with 1 nullable unique field provided (e.g. A, B, C)
CREATE UNIQUE INDEX parent_policies_null1 ON parent_policies (name, namespace, controls, standards) WHERE categories IS NULL;
CREATE UNIQUE INDEX parent_policies_null2 ON parent_policies (name, namespace, categories, standards) WHERE controls IS NULL;
CREATE UNIQUE INDEX parent_policies_null3 ON parent_policies (name, namespace, categories, controls) WHERE standards IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS parent_policies_null1 ON parent_policies (name, namespace, controls, standards) WHERE categories IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS parent_policies_null2 ON parent_policies (name, namespace, categories, standards) WHERE controls IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS parent_policies_null3 ON parent_policies (name, namespace, categories, controls) WHERE standards IS NULL;

-- Partial indexes with 2 nullable unique field provided (e.g. AB AC BC)
CREATE UNIQUE INDEX parent_policies_null4 ON parent_policies (name, namespace, standards) WHERE categories IS NULL AND controls IS NULL;
CREATE UNIQUE INDEX parent_policies_null5 ON parent_policies (name, namespace, controls) WHERE categories IS NULL AND standards IS NULL;
CREATE UNIQUE INDEX parent_policies_null6 ON parent_policies (name, namespace, categories) WHERE controls IS NULL AND standards IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS parent_policies_null4 ON parent_policies (name, namespace, standards) WHERE categories IS NULL AND controls IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS parent_policies_null5 ON parent_policies (name, namespace, controls) WHERE categories IS NULL AND standards IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS parent_policies_null6 ON parent_policies (name, namespace, categories) WHERE controls IS NULL AND standards IS NULL;

-- Partial index with no nullable unique fields provided (e.g. ABC)
CREATE UNIQUE INDEX parent_policies_null7 ON parent_policies (name, namespace) WHERE categories IS NULL AND controls IS NULL AND standards IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS parent_policies_null7 ON parent_policies (name, namespace) WHERE categories IS NULL AND controls IS NULL AND standards IS NULL;

CREATE TABLE IF NOT EXISTS policies(
id serial PRIMARY KEY,
kind TEXT NOT NULL,
api_group TEXT NOT NULL,
name TEXT NOT NULL,
namespace TEXT,
spec TEXT NOT NULL,
-- SHA1 hash
spec_hash CHAR(40) NOT NULL,
spec JSONB NOT NULL,
severity TEXT,
UNIQUE (kind, api_group, name, namespace, spec_hash, severity)
UNIQUE (kind, api_group, name, namespace, spec, severity)
);

-- This is required until we only support Postgres 15+ to utilize NULLS NOT DISTINCT.
-- Partial indexes with 1 nullable unique field provided (e.g. A, B)
CREATE UNIQUE INDEX policies_null1 ON policies (kind, api_group, name, spec_hash, severity) WHERE namespace IS NULL;
CREATE UNIQUE INDEX policies_null2 ON policies (kind, api_group, name, namespace, spec_hash) WHERE severity IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS policies_null1 ON policies (kind, api_group, name, spec, severity) WHERE namespace IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS policies_null2 ON policies (kind, api_group, name, namespace, spec) WHERE severity IS NULL;

-- Partial index with no nullable unique fields provided (e.g. AB)
CREATE UNIQUE INDEX policies_null3 ON policies (kind, api_group, name, spec_hash) WHERE namespace IS NULL AND severity IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS policies_null3 ON policies (kind, api_group, name, spec) WHERE namespace IS NULL AND severity IS NULL;

CREATE INDEX IF NOT EXISTS idx_policies_spec_hash ON policies (spec_hash);
CREATE INDEX IF NOT EXISTS idx_policies_spec ON policies (spec);

CREATE TABLE IF NOT EXISTS compliance_events(
id serial PRIMARY KEY,
Expand Down
48 changes: 9 additions & 39 deletions controllers/complianceeventsapi/server.go
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,10 @@
package complianceeventsapi

import (
"bytes"
"context"
"crypto/sha1" // #nosec G505 -- for convenience, not cryptography
"database/sql"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"sync"
Expand Down Expand Up @@ -113,7 +109,7 @@ func postComplianceEvent(db *sql.DB, w http.ResponseWriter, r *http.Request) {
return
}

if err := reqEvent.Validate(); err != nil {
if err := reqEvent.Validate(r.Context(), db); err != nil {
writeErrMsgJSON(w, err.Error(), http.StatusBadRequest)

return
Expand Down Expand Up @@ -165,8 +161,8 @@ func postComplianceEvent(db *sql.DB, w http.ResponseWriter, r *http.Request) {
return
}

// remove the spec to only respond with the specHash
reqEvent.Policy.Spec = ""
// remove the spec so it's not returned in the JSON.
reqEvent.Policy.Spec = nil

resp, err := json.Marshal(reqEvent)
if err != nil {
Expand Down Expand Up @@ -201,6 +197,10 @@ func getClusterForeignKey(ctx context.Context, db *sql.DB, cluster Cluster) (int
}

func getParentPolicyForeignKey(ctx context.Context, db *sql.DB, parent ParentPolicy) (int32, error) {
if parent.KeyID != 0 {
return parent.KeyID, nil
}

// Check cache
parKey := parent.key()

Expand All @@ -220,15 +220,8 @@ func getParentPolicyForeignKey(ctx context.Context, db *sql.DB, parent ParentPol
}

func getPolicyForeignKey(ctx context.Context, db *sql.DB, pol Policy) (int32, error) {
// Fill in missing fields that can be inferred from other fields
if pol.SpecHash == "" {
var buf bytes.Buffer
if err := json.Compact(&buf, []byte(pol.Spec)); err != nil {
return 0, err // This kind of error would have been found during validation
}

sum := sha1.Sum(buf.Bytes()) // #nosec G401 -- for convenience, not cryptography
pol.SpecHash = hex.EncodeToString(sum[:])
if pol.KeyID != 0 {
return pol.KeyID, nil
}

// Check cache
Expand All @@ -239,29 +232,6 @@ func getPolicyForeignKey(ctx context.Context, db *sql.DB, pol Policy) (int32, er
return key.(int32), nil
}

if pol.Spec == "" {
row := db.QueryRowContext(
ctx, "SELECT spec FROM policies WHERE spec_hash=$1 LIMIT 1", pol.SpecHash,
)
if row.Err() != nil {
return 0, fmt.Errorf("could not determine the spec from the provided spec hash: %w", row.Err())
}

err := row.Scan(&pol.Spec)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return 0, fmt.Errorf(
"%w: could not determine the spec from the provided spec hash; the spec is required in the request",
errRequiredFieldNotProvided,
)
}

return 0, fmt.Errorf(
"the database returned an unexpected spec value for the provided spec hash: %w", err,
)
}
}

err := pol.GetOrCreate(ctx, db)
if err != nil {
return 0, err
Expand Down
Loading

0 comments on commit 4fa3772

Please sign in to comment.