From a8f7c05eeaf3550d49e36157c1e04e3128433772 Mon Sep 17 00:00:00 2001 From: Yi Rae Kim Date: Thu, 1 Feb 2024 09:31:28 -0500 Subject: [PATCH] Add an API endpoint to generate a CSV compliance report Ref: https://issues.redhat.com/browse/ACM-6884 Signed-off-by: Yi Rae Kim (cherry picked from commit 6efa4c03b45157c96024ed6acaaa6f96d830e02d) --- controllers/complianceeventsapi/server.go | 277 ++++++++++++++++-- .../complianceeventsapi/server_test.go | 142 +++++++++ test/e2e/case18_compliance_api_test.go | 206 +++++++++++++ 3 files changed, 594 insertions(+), 31 deletions(-) diff --git a/controllers/complianceeventsapi/server.go b/controllers/complianceeventsapi/server.go index d31a11e6..fa592f78 100644 --- a/controllers/complianceeventsapi/server.go +++ b/controllers/complianceeventsapi/server.go @@ -5,6 +5,7 @@ import ( "crypto/tls" "crypto/x509" "database/sql" + "encoding/csv" "encoding/json" "errors" "fmt" @@ -238,6 +239,19 @@ func (s *ComplianceAPIServer) Start(ctx context.Context, serverContext *Complian getSingleComplianceEvent(serverContext.DB, w, r) }) + mux.HandleFunc("/api/v1/reports/compliance-events", func(w http.ResponseWriter, r *http.Request) { + // This header is for error writings + w.Header().Set("Content-Type", "application/json") + + if r.Method != http.MethodGet { + writeErrMsgJSON(w, "Method not allowed", http.StatusMethodNotAllowed) + + return + } + + getComplianceEventsCSV(serverContext.DB, w, r) + }) + serveErr := make(chan error) go func() { @@ -309,7 +323,7 @@ func splitQueryValue(value string) []string { // parseQueryArgs will parse the HTTP request's query arguments and convert them to a usable format for constructing // the SQL query. All defaults are set and any invalid query arguments result in an error being returned. -func parseQueryArgs(queryArgs url.Values) (*queryOptions, error) { +func parseQueryArgs(queryArgs url.Values, isCSV bool) (*queryOptions, error) { parsed := &queryOptions{ Direction: "desc", Page: 1, @@ -320,6 +334,11 @@ func parseQueryArgs(queryArgs url.Values) (*queryOptions, error) { NullFilters: []string{}, } + // Case return CSV file, default PerPage is 0. Unlimited + if isCSV { + parsed.PerPage = 0 + } + for arg := range queryArgs { valid := false @@ -433,6 +452,17 @@ func parseQueryArgs(queryArgs url.Values) (*queryOptions, error) { // generateGetComplianceEventsQuery will return a SELECT query with results ready to be parsed by // scanIntoComplianceEvent. The caller is responsible for adding filters to the query. func generateGetComplianceEventsQuery(includeSpec bool) string { + return fmt.Sprintf(`SELECT %s +FROM + compliance_events + LEFT JOIN clusters ON compliance_events.cluster_id = clusters.id + LEFT JOIN parent_policies ON compliance_events.parent_policy_id = parent_policies.id + LEFT JOIN policies ON compliance_events.policy_id = policies.id`, + strings.Join(generateSelectedArgs(includeSpec), ", "), + ) +} + +func generateSelectedArgs(includeSpec bool) []string { selectArgs := []string{ "compliance_events.id", "compliance_events.compliance", @@ -460,14 +490,19 @@ func generateGetComplianceEventsQuery(includeSpec bool) string { selectArgs = append(selectArgs, "policies.spec") } - return fmt.Sprintf(`SELECT %s -FROM - compliance_events - LEFT JOIN clusters ON compliance_events.cluster_id = clusters.id - LEFT JOIN parent_policies ON compliance_events.parent_policy_id = parent_policies.id - LEFT JOIN policies ON compliance_events.policy_id = policies.id`, - strings.Join(selectArgs, ", "), - ) + return selectArgs +} + +// generate Headers for CSV. "." replace by "_" +// Example: parent_policies.namespace -> parent_policies_namespace +func getCsvHeader(includeSpec bool) []string { + localSelectArgs := generateSelectedArgs(includeSpec) + + for i, arg := range localSelectArgs { + localSelectArgs[i] = strings.ReplaceAll(arg, ".", "_") + } + + return localSelectArgs } type Scannable interface { @@ -672,7 +707,7 @@ func getWhereClause(options *queryOptions) (string, []any) { // getComplianceEvents handles the list API endpoint for compliance events. func getComplianceEvents(db *sql.DB, w http.ResponseWriter, r *http.Request) { - queryArgs, err := parseQueryArgs(r.URL.Query()) + queryArgs, err := parseQueryArgs(r.URL.Query(), false) if err != nil { writeErrMsgJSON(w, err.Error(), http.StatusBadRequest) @@ -682,27 +717,7 @@ func getComplianceEvents(db *sql.DB, w http.ResponseWriter, r *http.Request) { // Note that the where clause could be an empty string if not filters were passed in the query arguments. whereClause, filterValues := getWhereClause(queryArgs) - // Example query: - // SELECT compliance_events.id, compliance_events.compliance, ... - // FROM compliance_events - // LEFT JOIN clusters ON compliance_events.cluster_id = clusters.id - // LEFT JOIN parent_policies ON compliance_events.parent_policy_id = parent_policies.id - // LEFT JOIN policies ON compliance_events.policy_id = policies.id - // WHERE (policies.name=$1 OR policies.name=$2) AND (policies.kind=$3) - // ORDER BY compliance_events.timestamp desc - // LIMIT 20 - // OFFSET 0 ROWS; - query := fmt.Sprintf(`%s%s -ORDER BY %s %s -LIMIT %d -OFFSET %d ROWS;`, - generateGetComplianceEventsQuery(queryArgs.IncludeSpec), - whereClause, - strings.Join(queryArgs.Sort, ", "), - queryArgs.Direction, - queryArgs.PerPage, - (queryArgs.Page-1)*queryArgs.PerPage, - ) + query := getComplianceEventsQuery(whereClause, queryArgs) rows, err := db.QueryContext(r.Context(), query, filterValues...) if err == nil { @@ -887,6 +902,206 @@ func postComplianceEvent(db *sql.DB, } } +func getComplianceEventsQuery(whereClause string, queryArgs *queryOptions) string { + // Getting CSV without the page argument + // Query should fetch all rows (unlimited) + if queryArgs.PerPage == 0 { + return fmt.Sprintf(`%s%s + ORDER BY %s %s;`, + generateGetComplianceEventsQuery(queryArgs.IncludeSpec), + whereClause, + strings.Join(queryArgs.Sort, ", "), + queryArgs.Direction, + ) + } + // Example query + // SELECT compliance_events.id, compliance_events.compliance, ... + // FROM compliance_events + // LEFT JOIN clusters ON compliance_events.cluster_id = clusters.id + // LEFT JOIN parent_policies ON compliance_events.parent_policy_id = parent_policies.id + // LEFT JOIN policies ON compliance_events.policy_id = policies.id + // WHERE (policies.name=$1 OR policies.name=$2) AND (policies.kind=$3) + // ORDER BY compliance_events.timestamp desc + // LIMIT 20 + // OFFSET 0 ROWS; + return fmt.Sprintf(`%s%s + ORDER BY %s %s + LIMIT %d + OFFSET %d ROWS;`, + generateGetComplianceEventsQuery(queryArgs.IncludeSpec), + whereClause, + strings.Join(queryArgs.Sort, ", "), + queryArgs.Direction, + queryArgs.PerPage, + (queryArgs.Page-1)*queryArgs.PerPage, + ) +} + +func getComplianceEventsCSV(db *sql.DB, w http.ResponseWriter, r *http.Request) { + queryArgs, err := parseQueryArgs(r.URL.Query(), true) + if err != nil { + writeErrMsgJSON(w, err.Error(), http.StatusBadRequest) + + return + } + + // Note that the where clause could be an empty string if no filters were passed in the query arguments. + whereClause, filterValues := getWhereClause(queryArgs) + + query := getComplianceEventsQuery(whereClause, queryArgs) + + rows, err := db.QueryContext(r.Context(), query, filterValues...) + if err == nil { + err = rows.Err() + } + + if err != nil { + log.Error(err, "Failed to query for compliance events") + writeErrMsgJSON(w, "Internal Error", http.StatusInternalServerError) + + return + } + + defer rows.Close() + + headers := getCsvHeader(queryArgs.IncludeSpec) + + writer := csv.NewWriter(w) + + err = writer.Write(headers) + if err != nil { + log.Error(err, "Failed to write csv header") + writeErrMsgJSON(w, "Internal Error", http.StatusInternalServerError) + + return + } + + for rows.Next() { + ce, err := scanIntoComplianceEvent(rows, queryArgs.IncludeSpec) + if err != nil { + log.Error(err, "Failed to unmarshal the database results") + writeErrMsgJSON(w, "Internal Error", http.StatusInternalServerError) + + return + } + + stringValues := convertToCsvLine(ce, queryArgs.IncludeSpec) + + err = writer.Write(stringValues) + if err != nil { + log.Error(err, "Failed to write csv list") + writeErrMsgJSON(w, "Internal Error", http.StatusInternalServerError) + + return + } + } + + w.Header().Set("Content-Disposition", "attachment; filename=reports.csv") + w.Header().Set("Content-Type", "text/csv") + // It's going to be divided into chunks. if the user don't get it all at once, + // the user can receive one by one in the meantime + w.Header().Set("Transfer-Encoding", "chunked") + + writer.Flush() +} + +func convertToCsvLine(ce *ComplianceEvent, includeSpec bool) []string { + nilString := "" + + if ce.ParentPolicy == nil { + ce.ParentPolicy = &ParentPolicy{ + KeyID: 0, + Name: "", + Namespace: "", + Categories: nil, + Controls: nil, + Standards: nil, + } + } + + if ce.Event.ReportedBy == nil { + ce.Event.ReportedBy = &nilString + } + + if ce.Policy.Severity == nil { + ce.Policy.Severity = &nilString + } + + if ce.Policy.Namespace == nil { + ce.Policy.Namespace = &nilString + } + + values := []string{ + convertToString(ce.EventID), + convertToString(ce.Event.Compliance), + convertToString(ce.Event.Message), + convertToString(ce.Event.Metadata), + convertToString(*ce.Event.ReportedBy), + convertToString(ce.Event.Timestamp), + convertToString(ce.Cluster.ClusterID), + convertToString(ce.Cluster.Name), + convertToString(ce.ParentPolicy.KeyID), + convertToString(ce.ParentPolicy.Name), + convertToString(ce.ParentPolicy.Namespace), + convertToString(ce.ParentPolicy.Categories), + convertToString(ce.ParentPolicy.Controls), + convertToString(ce.ParentPolicy.Standards), + convertToString(ce.Policy.KeyID), + convertToString(ce.Policy.APIGroup), + convertToString(ce.Policy.Kind), + convertToString(ce.Policy.Name), + convertToString(*ce.Policy.Namespace), + convertToString(*ce.Policy.Severity), + } + + if includeSpec { + values = append(values, convertToString(ce.Policy.Spec)) + } + + return values +} + +func convertToString(v interface{}) string { + switch vv := v.(type) { + case *string: + if vv == nil { + return "" + } + + return *vv + case string: + return vv + case int32: + // All int32 related id + if int(vv) == 0 { + return "" + } + + return strconv.Itoa(int(vv)) + case time.Time: + return vv.String() + case pq.StringArray: + // nil will be [] + return strings.Join(vv, ", ") + case bool: + return strconv.FormatBool(vv) + case JSONMap: + if vv == nil { + return "" + } + + jsonByte, err := json.MarshalIndent(vv, "", " ") + if err != nil { + return "" + } + + return string(jsonByte) + default: + // case nil: + return fmt.Sprintf("%v", vv) + } +} + func getClusterForeignKey(ctx context.Context, db *sql.DB, cluster Cluster) (int32, error) { // Check cache key, ok := clusterKeyCache.Load(cluster.ClusterID) diff --git a/controllers/complianceeventsapi/server_test.go b/controllers/complianceeventsapi/server_test.go index 26de6001..b57a1b69 100644 --- a/controllers/complianceeventsapi/server_test.go +++ b/controllers/complianceeventsapi/server_test.go @@ -3,6 +3,7 @@ package complianceeventsapi import ( "fmt" "testing" + "time" . "github.com/onsi/gomega" ) @@ -35,3 +36,144 @@ func TestSplitQueryValue(t *testing.T) { ) } } + +func TestConvertToCsvLine(t *testing.T) { + t.Parallel() + + theTime := time.Date(2021, 8, 15, 14, 30, 45, 100, time.UTC) + + reportBy := "cat1" + + ce := ComplianceEvent{ + EventID: 1, + Event: EventDetails{ + Compliance: "cp1", + Message: "event1 message", + Metadata: nil, + ReportedBy: &reportBy, + Timestamp: theTime, + }, + Cluster: Cluster{ + ClusterID: "1111", + Name: "cluster1", + }, + Policy: Policy{ + KeyID: 0, + Kind: "", + APIGroup: "v1", + Name: "", + Spec: map[string]interface{}{ + "name": "hi", + "namespace": "cat-1", + }, + }, + } + + values := convertToCsvLine(&ce, true) + + g := NewWithT(t) + g.Expect(values).Should(HaveLen(21)) + // Should follow this order + // "compliance_events_id", + // "compliance_events_compliance", + // "compliance_events_message", + // "compliance_events_metadata", + // "compliance_events_reported_by", + // "compliance_events_timestamp", + // "clusters_cluster_id", + // "clusters_name", + // "parent_policies_id", + // "parent_policies_name", + // "parent_policies_namespace", + // "parent_policies_categories", + // "parent_policies_controls", + // "parent_policies_standards", + // "policies_id", + // "policies_api_group", + // "policies_kind", + // "policies_name", + // "policies_namespace", + // "policies_severity", + // "policies_spec", + g.Expect(values).Should(Equal([]string{ + "1", "cp1", "event1 message", + "", "cat1", "2021-08-15 14:30:45.0000001 +0000 UTC", + "1111", "cluster1", "", "", "", "", "", "", "", "v1", "", "", "", "", + "{\n \"name\": \"hi\",\n \"namespace\": \"cat-1\"\n}", + })) + + // Test includeSpec = false + values = convertToCsvLine(&ce, false) + g.Expect(values).Should(HaveLen(20), "Test Some fields set") + + parentPolicy := &ParentPolicy{ + KeyID: 11, + Name: "parent-my-name", + Namespace: "ns-pp", + Categories: []string{"cate-1", "cate-2"}, + Controls: []string{"control-1", "control-2"}, + Standards: []string{"stand-1", "stand-2"}, + } + + // Test All fields set + ce = ComplianceEvent{ + EventID: 1, + ParentPolicy: parentPolicy, + Event: EventDetails{ + Compliance: "cp1", + Message: "event1 message", + Metadata: JSONMap{ + "pet": "cat1", + "flower": []string{"rose", "sunflower"}, + "number": 1, + }, + ReportedBy: &reportBy, + Timestamp: theTime, + }, + Cluster: Cluster{ + ClusterID: "22", + Name: "cluster1", + }, + Policy: Policy{ + KeyID: 0, + Kind: "configuration", + APIGroup: "v1", + Name: "policy-name", + Spec: JSONMap{ + "name": "hi", + "namespace": "cat-1", + }, + }, + } + + values = convertToCsvLine(&ce, true) + g.Expect(values).Should(Equal([]string{ + "1", "cp1", "event1 message", + "{\n \"flower\": [\n \"rose\",\n \"sunflower\"\n ],\n \"number\": 1,\n \"pet\": \"cat1\"\n}", + "cat1", "2021-08-15 14:30:45.0000001 +0000 UTC", "22", "cluster1", + "11", "parent-my-name", "ns-pp", "cate-1, cate-2", + "control-1, control-2", "stand-1, stand-2", "", + "v1", "configuration", "policy-name", "", "", + "{\n \"name\": \"hi\",\n \"namespace\": \"cat-1\"\n}", + }), "Test All fields set") +} + +func TestGetCsvHeader(t *testing.T) { + g := NewWithT(t) + + result := getCsvHeader(true) + g.Expect(result).Should(HaveLen(21)) + g.Expect(result).Should(Equal([]string{ + "compliance_events_id", + "compliance_events_compliance", + "compliance_events_message", "compliance_events_metadata", + "compliance_events_reported_by", "compliance_events_timestamp", "clusters_cluster_id", + "clusters_name", "parent_policies_id", "parent_policies_name", + "parent_policies_namespace", "parent_policies_categories", "parent_policies_controls", + "parent_policies_standards", "policies_id", "policies_api_group", "policies_kind", "policies_name", + "policies_namespace", "policies_severity", "policies_spec", + })) + + result = getCsvHeader(false) + g.Expect(result).Should(HaveLen(20)) +} diff --git a/test/e2e/case18_compliance_api_test.go b/test/e2e/case18_compliance_api_test.go index 9e981dca..86d28d33 100644 --- a/test/e2e/case18_compliance_api_test.go +++ b/test/e2e/case18_compliance_api_test.go @@ -7,6 +7,7 @@ import ( "context" "crypto/tls" "database/sql" + "encoding/csv" "encoding/json" "errors" "fmt" @@ -1725,6 +1726,211 @@ var _ = Describe("Test the compliance events API", Label("compliance-events-api" "invalid query argument: event.timestamp_after must be in the format of RFC 3339", ), ) + + Describe("Test the /api/v1/reports/compliance-events endpoint", func() { + It("should send CSV file in http response", func(ctx context.Context) { + endpoints := "http://localhost:8385/api/v1/reports/compliance-events" + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoints, nil) + Expect(err).ShouldNot(HaveOccurred()) + + resp, err := httpClient.Do(req) + Expect(err).ShouldNot(HaveOccurred()) + + defer resp.Body.Close() + + By("Content-type should be CSV") + Expect(resp.Header.Get("Content-Type")).Should(Equal("text/csv")) + + csvReader := csv.NewReader(resp.Body) + + records, err := csvReader.ReadAll() + Expect(err).ShouldNot(HaveOccurred()) + + Expect(len(records)).Should(BeNumerically(">", 10)) + + By("First line should be the titles") + Expect(records[0]).Should(ContainElements([]string{ + "compliance_events_id", + "compliance_events_compliance", + "compliance_events_message", + "compliance_events_metadata", + "compliance_events_reported_by", + "compliance_events_timestamp", + "clusters_cluster_id", + "clusters_name", + "parent_policies_id", + "parent_policies_name", + "parent_policies_namespace", + "parent_policies_categories", + "parent_policies_controls", + "parent_policies_standards", + "policies_id", + "policies_api_group", + "policies_kind", + "policies_name", + "policies_namespace", + "policies_severity", + })) + + By("All line should have 20 columns") + for _, r := range records { + Expect(r).Should(HaveLen(20)) + } + }) + + DescribeTable("Should filter CSV file", + func(ctx context.Context, queryArgs []string, expectedLine int) { + endpoints := "http://localhost:8385/api/v1/reports/compliance-events" + + endpoints += "?" + strings.Join(queryArgs, "&") + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoints, nil) + Expect(err).ShouldNot(HaveOccurred()) + + resp, err := httpClient.Do(req) + Expect(err).ShouldNot(HaveOccurred()) + + defer resp.Body.Close() + + csvReader := csv.NewReader(resp.Body) + records, err := csvReader.ReadAll() + Expect(err).ShouldNot(HaveOccurred()) + + // The first element is title + Expect(records).Should(HaveLen(expectedLine)) + }, + Entry( + "Filter by cluster.cluster_id", + []string{"cluster.cluster_id=test1-cluster1-fake-uuid-1,test6-cluster6-fake-uuid-6"}, + // titles + actual data + 4, + ), + Entry( + "Filter by cluster.name", + []string{"cluster.name=cluster1,cluster6"}, + 4, + ), + Entry( + "Filter by event.compliance", + []string{"event.compliance=Compliant"}, + 7, + ), + Entry( + "Filter by event.message", + []string{"event.message=configmaps%20%5Bcommon%5D%20not%20found%20in%20namespace%20default"}, + 4, + ), + Entry( + "Filter by event.message_includes", + []string{"event.message_includes=etcd"}, + 4, + ), + Entry( + "Filter by event.message_like", + []string{"event.message_like=configmaps%20%5B%25common%25%5D%25"}, + 9, + ), + Entry( + "Filter by event.timestamp", + []string{"event.timestamp=2023-01-01T01:01:01.111Z"}, + 2, + ), + Entry( + "Filter by event.timestamp_after", + []string{"event.timestamp_after=2023-04-01T01:01:01.111Z"}, + 6, + ), + Entry( + "Filter by event.timestamp_before", + []string{"event.timestamp_before=2023-04-01T01:01:01.111Z"}, + 7, + ), + Entry( + "Filter by event.timestamp_after and event.timestamp_before", + []string{ + "event.timestamp_after=2023-01-01T01:01:01.111Z", + "event.timestamp_before=2023-04-01T01:01:01.111Z", + }, + 6, + ), + Entry( + "Filter by parent_policy.categories", + []string{"parent_policy.categories=cat-1,cat-3"}, + 5, + ), + Entry( + "Filter by parent_policy.controls", + []string{"parent_policy.controls=ctrl-2"}, + 4, + ), + Entry( + "Filter by parent_policy.id", + []string{"parent_policy.id=2"}, + 4, + ), + Entry( + "Filter by parent_policy.name", + []string{"parent_policy.name=etcd-encryption1"}, + 2, + ), + Entry( + "Filter by parent_policy.namespace", + []string{"parent_policy.namespace=policies"}, + 10, + ), + Entry( + "Filter by parent_policy.standards", + []string{"parent_policy.standards=stand-2"}, + 4, + ), + Entry( + "Filter by policy.apiGroup", + []string{"policy.apiGroup=policy.open-cluster-management.io"}, + 12, + ), + Entry( + "Filter by policy.apiGroup no results", + []string{"policy.apiGroup=does-not-exist"}, + 1, + ), + Entry( + "Filter by policy.id", + []string{"policy.id=4"}, + 4, + ), + Entry( + "Filter by policy.kind", + []string{"policy.kind=ConfigurationPolicy"}, + 12, + ), + Entry( + "Filter by policy.kind no results", + []string{"policy.kind=something-else"}, + 1, + ), + Entry( + "Filter by policy.name", + []string{"policy.name=common-b"}, + 3, + ), + Entry( + "Filter by policy.namespace", + []string{"policy.namespace=default"}, + 2, + ), + Entry( + "Filter by policy.severity", + []string{"policy.severity=low"}, + 10, + ), + Entry( + "Filter by policy.severity is null", + []string{"policy.severity"}, + 3, + ), + ) + }) }) Describe("Duplicate compliance event", func() {