Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🤖 Sync from open-cluster-management-io/governance-policy-propagator: #160 #534

Merged
merged 2 commits into from
Jan 5, 2024
Merged

🤖 Sync from open-cluster-management-io/governance-policy-propagator: #160 #534

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
edbd7be
Set the primary key types to int32
mprahl Jan 3, 2024
4fa3772
Remove spec_hash and allow the use of IDs instead
mprahl Jan 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 13 additions & 15 deletions controllers/complianceeventsapi/migrations/000001_compliance_history_initial_tables.up.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,40 +21,38 @@ CREATE TABLE IF NOT EXISTS parent_policies(

-- This is required until we only support Postgres 15+ to utilize NULLS NOT DISTINCT.
-- Partial indexes with 1 nullable unique field provided (e.g. A, B, C)
CREATE UNIQUE INDEX parent_policies_null1 ON parent_policies (name, namespace, controls, standards) WHERE categories IS NULL;
CREATE UNIQUE INDEX parent_policies_null2 ON parent_policies (name, namespace, categories, standards) WHERE controls IS NULL;
CREATE UNIQUE INDEX parent_policies_null3 ON parent_policies (name, namespace, categories, controls) WHERE standards IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS parent_policies_null1 ON parent_policies (name, namespace, controls, standards) WHERE categories IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS parent_policies_null2 ON parent_policies (name, namespace, categories, standards) WHERE controls IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS parent_policies_null3 ON parent_policies (name, namespace, categories, controls) WHERE standards IS NULL;

-- Partial indexes with 2 nullable unique field provided (e.g. AB AC BC)
CREATE UNIQUE INDEX parent_policies_null4 ON parent_policies (name, namespace, standards) WHERE categories IS NULL AND controls IS NULL;
CREATE UNIQUE INDEX parent_policies_null5 ON parent_policies (name, namespace, controls) WHERE categories IS NULL AND standards IS NULL;
CREATE UNIQUE INDEX parent_policies_null6 ON parent_policies (name, namespace, categories) WHERE controls IS NULL AND standards IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS parent_policies_null4 ON parent_policies (name, namespace, standards) WHERE categories IS NULL AND controls IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS parent_policies_null5 ON parent_policies (name, namespace, controls) WHERE categories IS NULL AND standards IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS parent_policies_null6 ON parent_policies (name, namespace, categories) WHERE controls IS NULL AND standards IS NULL;

-- Partial index with no nullable unique fields provided (e.g. ABC)
CREATE UNIQUE INDEX parent_policies_null7 ON parent_policies (name, namespace) WHERE categories IS NULL AND controls IS NULL AND standards IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS parent_policies_null7 ON parent_policies (name, namespace) WHERE categories IS NULL AND controls IS NULL AND standards IS NULL;

CREATE TABLE IF NOT EXISTS policies(
id serial PRIMARY KEY,
kind TEXT NOT NULL,
api_group TEXT NOT NULL,
name TEXT NOT NULL,
namespace TEXT,
spec TEXT NOT NULL,
-- SHA1 hash
spec_hash CHAR(40) NOT NULL,
spec JSONB NOT NULL,
severity TEXT,
UNIQUE (kind, api_group, name, namespace, spec_hash, severity)
UNIQUE (kind, api_group, name, namespace, spec, severity)
);

-- This is required until we only support Postgres 15+ to utilize NULLS NOT DISTINCT.
-- Partial indexes with 1 nullable unique field provided (e.g. A, B)
CREATE UNIQUE INDEX policies_null1 ON policies (kind, api_group, name, spec_hash, severity) WHERE namespace IS NULL;
CREATE UNIQUE INDEX policies_null2 ON policies (kind, api_group, name, namespace, spec_hash) WHERE severity IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS policies_null1 ON policies (kind, api_group, name, spec, severity) WHERE namespace IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS policies_null2 ON policies (kind, api_group, name, namespace, spec) WHERE severity IS NULL;

-- Partial index with no nullable unique fields provided (e.g. AB)
CREATE UNIQUE INDEX policies_null3 ON policies (kind, api_group, name, spec_hash) WHERE namespace IS NULL AND severity IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS policies_null3 ON policies (kind, api_group, name, spec) WHERE namespace IS NULL AND severity IS NULL;

CREATE INDEX IF NOT EXISTS idx_policies_spec_hash ON policies (spec_hash);
CREATE INDEX IF NOT EXISTS idx_policies_spec ON policies (spec);

CREATE TABLE IF NOT EXISTS compliance_events(
id serial PRIMARY KEY,
Expand Down
60 changes: 15 additions & 45 deletions controllers/complianceeventsapi/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,10 @@
package complianceeventsapi

import (
"bytes"
"context"
"crypto/sha1" // #nosec G505 -- for convenience, not cryptography
"database/sql"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"sync"
Expand Down Expand Up @@ -113,7 +109,7 @@ func postComplianceEvent(db *sql.DB, w http.ResponseWriter, r *http.Request) {
return
}

if err := reqEvent.Validate(); err != nil {
if err := reqEvent.Validate(r.Context(), db); err != nil {
writeErrMsgJSON(w, err.Error(), http.StatusBadRequest)

return
Expand Down Expand Up @@ -165,8 +161,8 @@ func postComplianceEvent(db *sql.DB, w http.ResponseWriter, r *http.Request) {
return
}

// remove the spec to only respond with the specHash
reqEvent.Policy.Spec = ""
// remove the spec so it's not returned in the JSON.
reqEvent.Policy.Spec = nil

resp, err := json.Marshal(reqEvent)
if err != nil {
Expand All @@ -183,11 +179,11 @@ func postComplianceEvent(db *sql.DB, w http.ResponseWriter, r *http.Request) {
}
}

func getClusterForeignKey(ctx context.Context, db *sql.DB, cluster Cluster) (int, error) {
func getClusterForeignKey(ctx context.Context, db *sql.DB, cluster Cluster) (int32, error) {
// Check cache
key, ok := clusterKeyCache.Load(cluster.ClusterID)
if ok {
return key.(int), nil
return key.(int32), nil
}

err := cluster.GetOrCreate(ctx, db)
Expand All @@ -200,13 +196,17 @@ func getClusterForeignKey(ctx context.Context, db *sql.DB, cluster Cluster) (int
return cluster.KeyID, nil
}

func getParentPolicyForeignKey(ctx context.Context, db *sql.DB, parent ParentPolicy) (int, error) {
func getParentPolicyForeignKey(ctx context.Context, db *sql.DB, parent ParentPolicy) (int32, error) {
if parent.KeyID != 0 {
return parent.KeyID, nil
}

// Check cache
parKey := parent.key()

key, ok := parentPolicyKeyCache.Load(parKey)
if ok {
return key.(int), nil
return key.(int32), nil
}

err := parent.GetOrCreate(ctx, db)
Expand All @@ -219,47 +219,17 @@ func getParentPolicyForeignKey(ctx context.Context, db *sql.DB, parent ParentPol
return parent.KeyID, nil
}

func getPolicyForeignKey(ctx context.Context, db *sql.DB, pol Policy) (int, error) {
// Fill in missing fields that can be inferred from other fields
if pol.SpecHash == "" {
var buf bytes.Buffer
if err := json.Compact(&buf, []byte(pol.Spec)); err != nil {
return 0, err // This kind of error would have been found during validation
}

sum := sha1.Sum(buf.Bytes()) // #nosec G401 -- for convenience, not cryptography
pol.SpecHash = hex.EncodeToString(sum[:])
func getPolicyForeignKey(ctx context.Context, db *sql.DB, pol Policy) (int32, error) {
if pol.KeyID != 0 {
return pol.KeyID, nil
}

// Check cache
polKey := pol.key()

key, ok := policyKeyCache.Load(polKey)
if ok {
return key.(int), nil
}

if pol.Spec == "" {
row := db.QueryRowContext(
ctx, "SELECT spec FROM policies WHERE spec_hash=$1 LIMIT 1", pol.SpecHash,
)
if row.Err() != nil {
return 0, fmt.Errorf("could not determine the spec from the provided spec hash: %w", row.Err())
}

err := row.Scan(&pol.Spec)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return 0, fmt.Errorf(
"%w: could not determine the spec from the provided spec hash; the spec is required in the request",
errRequiredFieldNotProvided,
)
}

return 0, fmt.Errorf(
"the database returned an unexpected spec value for the provided spec hash: %w", err,
)
}
return key.(int32), nil
}

err := pol.GetOrCreate(ctx, db)
Expand Down
Loading