diff --git a/pkg/checks/checks.go b/pkg/checks/checks.go index d495ab12..b8cbd6b0 100644 --- a/pkg/checks/checks.go +++ b/pkg/checks/checks.go @@ -74,6 +74,10 @@ type Result struct { Err string `json:"error"` } +type GlobalTarget struct { + Url string `json:"url"` + LastSeen time.Time `json:"lastSeen"` +} type ResultDTO struct { Name string Result *Result diff --git a/pkg/sparrow/gitlab/gitlab.go b/pkg/sparrow/gitlab/gitlab.go new file mode 100644 index 00000000..7984693a --- /dev/null +++ b/pkg/sparrow/gitlab/gitlab.go @@ -0,0 +1,250 @@ +package gitlab + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + + "github.com/caas-team/sparrow/internal/logger" + "github.com/caas-team/sparrow/pkg/checks" +) + +// Gitlab handles interaction with a gitlab repository containing +// the global targets for the Sparrow instance +type Gitlab interface { + FetchFiles(ctx context.Context) ([]checks.GlobalTarget, error) + PutFile(ctx context.Context, file File) error + PostFile(ctx context.Context, file File) error +} + +// Client implements Gitlab +type Client struct { + // the base URL of the gitlab instance + baseUrl string + // the ID of the project containing the global targets + projectID int + // the token used to authenticate with the gitlab instance + token string + client *http.Client +} + +func New(url, token string, pid int) Gitlab { + return &Client{ + baseUrl: url, + token: token, + projectID: pid, + client: &http.Client{}, + } +} + +// FetchFiles fetches the files from the global targets repository from the configured gitlab repository +func (g *Client) FetchFiles(ctx context.Context) ([]checks.GlobalTarget, error) { + log := logger.FromContext(ctx).With("name", "FetchFiles") + fl, err := g.fetchFileList(ctx) + if err != nil { + log.Error("Failed to fetch files", "error", err) + return nil, err + } + + result, err := g.fetchFiles(ctx, fl) + if err != nil { + log.Error("Failed to fetch files", "error", err) + return nil, err + } + log.Info("Successfully fetched all target files", "files", len(result)) + return result, nil +} + +// fetchFiles fetches the files from the global targets repository from the configured gitlab repository +func (g *Client) fetchFiles(ctx context.Context, fl []string) ([]checks.GlobalTarget, error) { + var result []checks.GlobalTarget + log := logger.FromContext(ctx).With("name", "fetchFiles") + log.Debug("Fetching global files") + for _, f := range fl { + // URL encode the name + n := url.PathEscape(f) + req, err := http.NewRequestWithContext(ctx, + http.MethodGet, + fmt.Sprintf("%s/api/v4/projects/%d/repository/files/%s/raw?ref=main", g.baseUrl, g.projectID, n), + http.NoBody, + ) + if err != nil { + log.Error("Failed to create request", "error", err) + return nil, err + } + req.Header.Add("PRIVATE-TOKEN", g.token) + req.Header.Add("Content-Type", "application/json") + + res, err := g.client.Do(req) + if err != nil { + log.Error("Failed to fetch file", "file", f, "error", err) + return nil, err + } + if res.StatusCode != http.StatusOK { + log.Error("Failed to fetch file", "status", res.Status) + return nil, fmt.Errorf("request failed, status is %s", res.Status) + } + + defer res.Body.Close() + var gt checks.GlobalTarget + err = json.NewDecoder(res.Body).Decode(>) + if err != nil { + log.Error("Failed to decode file after fetching", "file", f, "error", err) + return nil, err + } + + log.Debug("Successfully fetched file", "file", f) + result = append(result, gt) + } + return result, nil +} + +// fetchFileList fetches the files from the global targets repository from the configured gitlab repository +func (g *Client) fetchFileList(ctx context.Context) ([]string, error) { + log := logger.FromContext(ctx).With("name", "fetchFileList") + log.Debug("Fetching global files") + type file struct { + Name string `json:"name"` + } + + req, err := http.NewRequestWithContext(ctx, + http.MethodGet, + fmt.Sprintf("%s/api/v4/projects/%d/repository/tree?ref=main", g.baseUrl, g.projectID), + http.NoBody, + ) + if err != nil { + log.Error("Failed to create request", "error", err) + return nil, err + } + + req.Header.Add("PRIVATE-TOKEN", g.token) + req.Header.Add("Content-Type", "application/json") + + res, err := g.client.Do(req) + if err != nil { + log.Error("Failed to fetch file list", "error", err) + return nil, err + } + if res.StatusCode != http.StatusOK { + log.Error("Failed to fetch file list", "status", res.Status) + return nil, fmt.Errorf("request failed, status is %s", res.Status) + } + + defer res.Body.Close() + var fl []file + err = json.NewDecoder(res.Body).Decode(&fl) + if err != nil { + log.Error("Failed to decode file list", "error", err) + return nil, err + } + + var result []string + for _, f := range fl { + result = append(result, f.Name) + } + + log.Debug("Successfully fetched file list", "files", len(result)) + return result, nil +} + +// PutFile commits the current instance to the configured gitlab repository +// as a global target for other sparrow instances to discover +func (g *Client) PutFile(ctx context.Context, body File) error { + log := logger.FromContext(ctx).With("name", "AddRegistration") + log.Debug("Registering sparrow instance to gitlab") + + // chose method based on whether the registration has already happened + n := url.PathEscape(body.fileName) + b, err := body.Bytes() + if err != nil { + log.Error("Failed to create request", "error", err) + return err + } + req, err := http.NewRequestWithContext(ctx, + http.MethodPut, + fmt.Sprintf("%s/api/v4/projects/%d/repository/files/%s", g.baseUrl, g.projectID, n), + bytes.NewBuffer(b), + ) + if err != nil { + log.Error("Failed to create request", "error", err) + return err + } + + req.Header.Add("PRIVATE-TOKEN", g.token) + req.Header.Add("Content-Type", "application/json") + + resp, err := g.client.Do(req) + if err != nil { + log.Error("Failed to push registration file", "error", err) + return err + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + log.Error("Failed to push registration file", "status", resp.Status) + return fmt.Errorf("request failed, status is %s", resp.Status) + } + + return nil +} + +// PostFile commits the current instance to the configured gitlab repository +// as a global target for other sparrow instances to discover +func (g *Client) PostFile(ctx context.Context, body File) error { + log := logger.FromContext(ctx).With("name", "AddRegistration") + log.Debug("Registering sparrow instance to gitlab") + + // chose method based on whether the registration has already happened + n := url.PathEscape(body.fileName) + b, err := body.Bytes() + if err != nil { + log.Error("Failed to create request", "error", err) + return err + } + req, err := http.NewRequestWithContext(ctx, + http.MethodPost, + fmt.Sprintf("%s/api/v4/projects/%d/repository/files/%s", g.baseUrl, g.projectID, n), + bytes.NewBuffer(b), + ) + if err != nil { + log.Error("Failed to create request", "error", err) + return err + } + + req.Header.Add("PRIVATE-TOKEN", g.token) + req.Header.Add("Content-Type", "application/json") + + resp, err := g.client.Do(req) + if err != nil { + log.Error("Failed to push registration file", "error", err) + return err + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + log.Error("Failed to push registration file", "status", resp.Status) + return fmt.Errorf("request failed, status is %s", resp.Status) + } + + return nil +} + +type File struct { + Branch string `json:"branch"` + AuthorEmail string `json:"author_email"` + AuthorName string `json:"author_name"` + Content checks.GlobalTarget `json:"content"` + CommitMessage string `json:"commit_message"` + fileName string +} + +// Bytes returns the bytes of the File +func (g File) Bytes() ([]byte, error) { + b, err := json.Marshal(g) + return b, err +} diff --git a/pkg/sparrow/gitlab/gitlab_test.go b/pkg/sparrow/gitlab/gitlab_test.go new file mode 100644 index 00000000..327f404f --- /dev/null +++ b/pkg/sparrow/gitlab/gitlab_test.go @@ -0,0 +1,403 @@ +package gitlab + +import ( + "context" + "fmt" + "net/http" + "reflect" + "testing" + "time" + + "github.com/caas-team/sparrow/pkg/checks" + "github.com/jarcoal/httpmock" +) + +func Test_gitlab_fetchFileList(t *testing.T) { + type file struct { + Name string `json:"name"` + } + tests := []struct { + name string + want []string + wantErr bool + mockBody []file + mockCode int + }{ + { + name: "success - 0 targets", + want: nil, + wantErr: false, + mockCode: http.StatusOK, + mockBody: []file{}, + }, + { + name: "success - 1 target", + want: []string{ + "test", + }, + wantErr: false, + mockCode: http.StatusOK, + mockBody: []file{ + { + Name: "test", + }, + }, + }, + { + name: "success - 2 targets", + want: []string{ + "test", + "test2", + }, + wantErr: false, + mockCode: http.StatusOK, + mockBody: []file{ + { + Name: "test", + }, + { + Name: "test2", + }, + }, + }, + { + name: "failure - API error", + want: nil, + wantErr: true, + mockCode: http.StatusInternalServerError, + }, + } + + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resp, err := httpmock.NewJsonResponder(tt.mockCode, tt.mockBody) + if err != nil { + t.Fatalf("error creating mock response: %v", err) + } + httpmock.RegisterResponder("GET", "http://test/api/v4/projects/1/repository/tree?ref=main", resp) + + g := &Client{ + baseUrl: "http://test", + projectID: 1, + token: "test", + client: http.DefaultClient, + } + got, err := g.fetchFileList(context.Background()) + if (err != nil) != tt.wantErr { + t.Fatalf("FetchFiles() error = %v, wantErr %v", err, tt.wantErr) + } + if !reflect.DeepEqual(got, tt.want) { + t.Fatalf("FetchFiles() got = %v, want %v", got, tt.want) + } + }) + } +} + +// The filelist and url are the same, so we HTTP responders can +// be created without much hassle +func Test_gitlab_fetchFiles(t *testing.T) { + tests := []struct { + name string + want []checks.GlobalTarget + fileList []string + wantErr bool + mockCode int + }{ + { + name: "success - 0 targets", + want: nil, + wantErr: false, + mockCode: http.StatusOK, + }, + { + name: "success - 1 target", + want: []checks.GlobalTarget{ + { + Url: "test", + LastSeen: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), + }, + }, + fileList: []string{ + "test", + }, + wantErr: false, + mockCode: http.StatusOK, + }, + { + name: "success - 2 targets", + want: []checks.GlobalTarget{ + { + Url: "test", + LastSeen: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), + }, + { + Url: "test2", + LastSeen: time.Date(2021, 2, 1, 0, 0, 0, 0, time.UTC), + }, + }, + fileList: []string{ + "test", + "test2", + }, + wantErr: false, + mockCode: http.StatusOK, + }, + } + + httpmock.Activate() + defer httpmock.DeactivateAndReset() + g := &Client{ + baseUrl: "http://test", + projectID: 1, + token: "test", + client: http.DefaultClient, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // setup mock responses + for i, target := range tt.want { + resp, err := httpmock.NewJsonResponder(tt.mockCode, target) + if err != nil { + t.Fatalf("error creating mock response: %v", err) + } + httpmock.RegisterResponder("GET", fmt.Sprintf("http://test/api/v4/projects/1/repository/files/%s/raw?ref=main", tt.fileList[i]), resp) + } + + got, err := g.fetchFiles(context.Background(), tt.fileList) + if (err != nil) != tt.wantErr { + t.Fatalf("FetchFiles() error = %v, wantErr %v", err, tt.wantErr) + } + if !reflect.DeepEqual(got, tt.want) { + t.Fatalf("FetchFiles() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_gitlab_fetchFiles_error_cases(t *testing.T) { + type mockResponses struct { + response checks.GlobalTarget + err bool + } + + tests := []struct { + name string + mockResponses []mockResponses + fileList []string + }{ + { + name: "failure - direct API error", + mockResponses: []mockResponses{ + { + err: true, + }, + }, + fileList: []string{ + "test", + }, + }, + { + name: "failure - API error after one successful request", + mockResponses: []mockResponses{ + { + response: checks.GlobalTarget{ + Url: "test", + LastSeen: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), + }, + err: false, + }, + { + response: checks.GlobalTarget{}, + err: true, + }, + }, + fileList: []string{ + "test", + "test2-will-fail", + }, + }, + } + + httpmock.Activate() + defer httpmock.DeactivateAndReset() + g := &Client{ + baseUrl: "http://test", + projectID: 1, + token: "test", + client: http.DefaultClient, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for i, target := range tt.mockResponses { + if target.err { + errResp := httpmock.NewStringResponder(http.StatusInternalServerError, "") + httpmock.RegisterResponder("GET", fmt.Sprintf("http://test/api/v4/projects/1/repository/files/%s/raw?ref=main", tt.fileList[i]), errResp) + continue + } + resp, err := httpmock.NewJsonResponder(http.StatusOK, target) + if err != nil { + t.Fatalf("error creating mock response: %v", err) + } + httpmock.RegisterResponder("GET", fmt.Sprintf("http://test/api/v4/projects/1/repository/files/%s/raw?ref=main", tt.fileList[i]), resp) + } + + _, err := g.fetchFiles(context.Background(), tt.fileList) + if err == nil { + t.Fatalf("Expected error but got none.") + } + }) + } +} + +func TestClient_PutFile(t *testing.T) { + now := time.Now() + tests := []struct { + name string + file File + mockCode int + wantErr bool + }{ + { + name: "success", + file: File{ + Branch: "main", + AuthorEmail: "test@sparrow", + AuthorName: "sparrpw", + Content: checks.GlobalTarget{ + Url: "https://test.de", + LastSeen: now, + }, + CommitMessage: "test-commit", + fileName: "test.de.json", + }, + mockCode: http.StatusOK, + }, + { + name: "failure - API error", + file: File{ + Branch: "main", + AuthorEmail: "test@sparrow", + AuthorName: "sparrpw", + Content: checks.GlobalTarget{ + Url: "https://test.de", + LastSeen: now, + }, + CommitMessage: "test-commit", + fileName: "test.de.json", + }, + mockCode: http.StatusInternalServerError, + wantErr: true, + }, + { + name: "failure - empty file", + wantErr: true, + }, + } + + httpmock.Activate() + defer httpmock.DeactivateAndReset() + g := &Client{ + baseUrl: "http://test", + projectID: 1, + token: "test", + client: http.DefaultClient, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.wantErr { + resp := httpmock.NewStringResponder(tt.mockCode, "") + httpmock.RegisterResponder("PUT", fmt.Sprintf("http://test/api/v4/projects/1/repository/files/%s", tt.file.fileName), resp) + } else { + resp, err := httpmock.NewJsonResponder(tt.mockCode, tt.file) + if err != nil { + t.Fatalf("error creating mock response: %v", err) + } + httpmock.RegisterResponder("PUT", fmt.Sprintf("http://test/api/v4/projects/1/repository/files/%s", tt.file.fileName), resp) + } + + if err := g.PutFile(context.Background(), tt.file); (err != nil) != tt.wantErr { + t.Fatalf("PutFile() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestClient_PostFile(t *testing.T) { + now := time.Now() + tests := []struct { + name string + file File + mockCode int + wantErr bool + }{ + { + name: "success", + file: File{ + Branch: "main", + AuthorEmail: "test@sparrow", + AuthorName: "sparrpw", + Content: checks.GlobalTarget{ + Url: "https://test.de", + LastSeen: now, + }, + CommitMessage: "test-commit", + fileName: "test.de.json", + }, + mockCode: http.StatusCreated, + }, + { + name: "failure - API error", + file: File{ + Branch: "main", + AuthorEmail: "test@sparrow", + AuthorName: "sparrpw", + Content: checks.GlobalTarget{ + Url: "https://test.de", + LastSeen: now, + }, + CommitMessage: "test-commit", + fileName: "test.de.json", + }, + mockCode: http.StatusInternalServerError, + wantErr: true, + }, + { + name: "failure - empty file", + wantErr: true, + }, + } + + httpmock.Activate() + defer httpmock.DeactivateAndReset() + g := &Client{ + baseUrl: "http://test", + projectID: 1, + token: "test", + client: http.DefaultClient, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.wantErr { + resp := httpmock.NewStringResponder(tt.mockCode, "") + httpmock.RegisterResponder("POST", fmt.Sprintf("http://test/api/v4/projects/1/repository/files/%s", tt.file.fileName), resp) + } else { + resp, err := httpmock.NewJsonResponder(tt.mockCode, tt.file) + if err != nil { + t.Fatalf("error creating mock response: %v", err) + } + httpmock.RegisterResponder("POST", fmt.Sprintf("http://test/api/v4/projects/1/repository/files/%s", tt.file.fileName), resp) + } + + if err := g.PostFile(context.Background(), tt.file); (err != nil) != tt.wantErr { + t.Fatalf("PostFile() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/pkg/sparrow/gitlab/test/mockclient.go b/pkg/sparrow/gitlab/test/mockclient.go new file mode 100644 index 00000000..bd00e45f --- /dev/null +++ b/pkg/sparrow/gitlab/test/mockclient.go @@ -0,0 +1,38 @@ +package gitlabmock + +import ( + "context" + "fmt" + + "github.com/caas-team/sparrow/pkg/checks" + "github.com/caas-team/sparrow/pkg/sparrow/gitlab" +) + +type MockClient struct { + targets []checks.GlobalTarget + err error +} + +func (m MockClient) PutFile(ctx context.Context, file gitlab.File) error { + panic("implement me") +} + +func (m MockClient) PostFile(ctx context.Context, f gitlab.File) error { + panic("implement me") +} + +func (m MockClient) FetchFiles(ctx context.Context) ([]checks.GlobalTarget, error) { + return m.targets, m.err +} + +// New creates a new MockClient to mock Gitlab interaction +func New(targets []checks.GlobalTarget, err bool) gitlab.Gitlab { + var e error + if err { + e = fmt.Errorf("error") + } + return &MockClient{ + targets: targets, + err: e, + } +} diff --git a/pkg/sparrow/run.go b/pkg/sparrow/run.go index 92199f62..c475e57e 100644 --- a/pkg/sparrow/run.go +++ b/pkg/sparrow/run.go @@ -24,12 +24,14 @@ import ( "net/http" "time" + "github.com/caas-team/sparrow/pkg/sparrow/gitlab" + targets "github.com/caas-team/sparrow/pkg/sparrow/targets" + "github.com/caas-team/sparrow/internal/logger" "github.com/caas-team/sparrow/pkg/api" "github.com/caas-team/sparrow/pkg/checks" "github.com/caas-team/sparrow/pkg/config" "github.com/caas-team/sparrow/pkg/db" - targets "github.com/caas-team/sparrow/pkg/sparrow/targets" "github.com/go-chi/chi/v5" ) @@ -63,7 +65,7 @@ func New(cfg *config.Config) *Sparrow { cCfgChecks: make(chan map[string]any, 1), routingTree: api.NewRoutingTree(), router: chi.NewRouter(), - targets: targets.NewGitlabManager(targets.NewGitlab("targetsRepo", "gitlabToken"), 5*time.Minute, 15*time.Minute), + targets: targets.NewGitlabManager(gitlab.New("targetsRepo", "gitlabToken", 1), 5*time.Minute, 15*time.Minute), } sparrow.loader = config.NewLoader(cfg, sparrow.cCfgChecks) @@ -187,3 +189,9 @@ func fanInResults(checkChan chan checks.Result, cResult chan checks.ResultDTO, n } } } + +// GlobalTarget represents a GlobalTarget that can be checked +type GlobalTarget struct { + Url string `json:"url"` + LastSeen time.Time `json:"lastSeen"` +} diff --git a/pkg/sparrow/targets/gitlab.go b/pkg/sparrow/targets/gitlab.go index 3e660da3..5e6b5694 100644 --- a/pkg/sparrow/targets/gitlab.go +++ b/pkg/sparrow/targets/gitlab.go @@ -1,45 +1,22 @@ package targets import ( - "bytes" "context" - "encoding/json" "fmt" - "net/http" - "net/url" "time" - "github.com/caas-team/sparrow/internal/logger" -) + "github.com/caas-team/sparrow/pkg/checks" + "github.com/caas-team/sparrow/pkg/sparrow/gitlab" -var ( - _ Gitlab = &gitlab{} - _ TargetManager = &gitlabTargetManager{} + "github.com/caas-team/sparrow/internal/logger" ) -// Gitlab handles interaction with a gitlab repository containing -// the global targets for the Sparrow instance -type Gitlab interface { - FetchFiles(ctx context.Context) ([]globalTarget, error) - PutFile(ctx context.Context, file GitlabFile) error - PostFile(ctx context.Context, file GitlabFile) error -} - -// gitlab implements Gitlab -type gitlab struct { - // the base URL of the gitlab instance - baseUrl string - // the ID of the project containing the global targets - projectID int - // the token used to authenticate with the gitlab instance - token string - client *http.Client -} +var _ TargetManager = &gitlabTargetManager{} // gitlabTargetManager implements TargetManager type gitlabTargetManager struct { - targets []globalTarget - gitlab Gitlab + targets []checks.GlobalTarget + gitlab gitlab.Gitlab // the DNS name used for self-registration name string // the interval for the target reconciliation process @@ -54,38 +31,24 @@ type gitlabTargetManager struct { } // NewGitlabManager creates a new gitlabTargetManager -func NewGitlabManager(g Gitlab, checkInterval, unhealthyThreshold time.Duration) TargetManager { +func NewGitlabManager(g gitlab.Gitlab, checkInterval, unhealthyThreshold time.Duration) TargetManager { return &gitlabTargetManager{ - targets: []globalTarget{}, gitlab: g, checkInterval: checkInterval, unhealthyThreshold: unhealthyThreshold, } } -// file represents a file in a gitlab repository -type file struct { - Name string `json:"name"` -} - -func NewGitlab(url, token string) Gitlab { - return &gitlab{ - baseUrl: url, - token: token, - client: &http.Client{}, - } -} - // updateRegistration registers the current instance as a global target func (t *gitlabTargetManager) updateRegistration(ctx context.Context) error { log := logger.FromContext(ctx).With("name", t.name, "registered", t.registered) log.Debug("Updating registration") - f := GitlabFile{ + f := gitlab.File{ Branch: "main", AuthorEmail: fmt.Sprintf("%s@sparrow", t.name), AuthorName: t.name, - Content: globalTarget{Url: fmt.Sprintf("https://%s", t.name), LastSeen: time.Now().UTC()}, + Content: checks.GlobalTarget{Url: fmt.Sprintf("https://%s", t.name), LastSeen: time.Now().UTC()}, } if t.registered { @@ -146,7 +109,7 @@ func (t *gitlabTargetManager) Reconcile(ctx context.Context) { } // GetTargets returns the current targets of the gitlabTargetManager -func (t *gitlabTargetManager) GetTargets() []globalTarget { +func (t *gitlabTargetManager) GetTargets() []checks.GlobalTarget { return t.targets } @@ -154,7 +117,7 @@ func (t *gitlabTargetManager) GetTargets() []globalTarget { // with the latest available healthy targets func (t *gitlabTargetManager) refreshTargets(ctx context.Context) error { log := logger.FromContext(ctx).With("name", "updateGlobalTargets") - var healthyTargets []globalTarget + var healthyTargets []checks.GlobalTarget targets, err := t.gitlab.FetchFiles(ctx) if err != nil { @@ -174,208 +137,3 @@ func (t *gitlabTargetManager) refreshTargets(ctx context.Context) error { log.Debug("Updated global targets", "targets", len(t.targets)) return nil } - -// FetchFiles fetches the files from the global targets repository from the configured gitlab repository -func (g *gitlab) FetchFiles(ctx context.Context) ([]globalTarget, error) { - log := logger.FromContext(ctx).With("name", "FetchFiles") - fl, err := g.fetchFileList(ctx) - if err != nil { - log.Error("Failed to fetch files", "error", err) - return nil, err - } - - result, err := g.fetchFiles(ctx, fl) - if err != nil { - log.Error("Failed to fetch files", "error", err) - return nil, err - } - log.Info("Successfully fetched all target files", "files", len(result)) - return result, nil -} - -// fetchFiles fetches the files from the global targets repository from the configured gitlab repository -func (g *gitlab) fetchFiles(ctx context.Context, fl []string) ([]globalTarget, error) { - var result []globalTarget - log := logger.FromContext(ctx).With("name", "fetchFiles") - log.Debug("Fetching global files") - for _, f := range fl { - // URL encode the name - n := url.PathEscape(f) - req, err := http.NewRequestWithContext(ctx, - http.MethodGet, - fmt.Sprintf("%s/api/v4/projects/%d/repository/files/%s/raw?ref=main", g.baseUrl, g.projectID, n), - http.NoBody, - ) - if err != nil { - log.Error("Failed to create request", "error", err) - return nil, err - } - req.Header.Add("PRIVATE-TOKEN", g.token) - req.Header.Add("Content-Type", "application/json") - - res, err := g.client.Do(req) - if err != nil { - log.Error("Failed to fetch file", "file", f, "error", err) - return nil, err - } - if res.StatusCode != http.StatusOK { - log.Error("Failed to fetch file", "status", res.Status) - return nil, fmt.Errorf("request failed, status is %s", res.Status) - } - - defer res.Body.Close() - var gt globalTarget - err = json.NewDecoder(res.Body).Decode(>) - if err != nil { - log.Error("Failed to decode file after fetching", "file", f, "error", err) - return nil, err - } - - log.Debug("Successfully fetched file", "file", f) - result = append(result, gt) - } - return result, nil -} - -// fetchFileList fetches the files from the global targets repository from the configured gitlab repository -func (g *gitlab) fetchFileList(ctx context.Context) ([]string, error) { - log := logger.FromContext(ctx).With("name", "fetchFileList") - log.Debug("Fetching global files") - type file struct { - Name string `json:"name"` - } - - req, err := http.NewRequestWithContext(ctx, - http.MethodGet, - fmt.Sprintf("%s/api/v4/projects/%d/repository/tree?ref=main", g.baseUrl, g.projectID), - http.NoBody, - ) - if err != nil { - log.Error("Failed to create request", "error", err) - return nil, err - } - - req.Header.Add("PRIVATE-TOKEN", g.token) - req.Header.Add("Content-Type", "application/json") - - res, err := g.client.Do(req) - if err != nil { - log.Error("Failed to fetch file list", "error", err) - return nil, err - } - if res.StatusCode != http.StatusOK { - log.Error("Failed to fetch file list", "status", res.Status) - return nil, fmt.Errorf("request failed, status is %s", res.Status) - } - - defer res.Body.Close() - var fl []file - err = json.NewDecoder(res.Body).Decode(&fl) - if err != nil { - log.Error("Failed to decode file list", "error", err) - return nil, err - } - - var result []string - for _, f := range fl { - result = append(result, f.Name) - } - - log.Debug("Successfully fetched file list", "files", len(result)) - return result, nil -} - -type GitlabFile struct { - Branch string `json:"branch"` - AuthorEmail string `json:"author_email"` - AuthorName string `json:"author_name"` - Content globalTarget `json:"content"` - CommitMessage string `json:"commit_message"` - fileName string -} - -// Bytes returns the bytes of the GitlabFile -func (g GitlabFile) Bytes() ([]byte, error) { - b, err := json.Marshal(g) - return b, err -} - -// PutFile commits the current instance to the configured gitlab repository -// as a global target for other sparrow instances to discover -func (g *gitlab) PutFile(ctx context.Context, body GitlabFile) error { - log := logger.FromContext(ctx).With("name", "AddRegistration") - log.Debug("Registering sparrow instance to gitlab") - - // chose method based on whether the registration has already happened - n := url.PathEscape(body.Content.Url) - b, err := body.Bytes() - if err != nil { - log.Error("Failed to create request", "error", err) - return err - } - req, err := http.NewRequestWithContext(ctx, - http.MethodGet, - fmt.Sprintf("%s/api/v4/projects/%d/repository/files/%s", g.baseUrl, g.projectID, n), - bytes.NewBuffer(b), - ) - if err != nil { - log.Error("Failed to create request", "error", err) - return err - } - - req.Header.Add("PRIVATE-TOKEN", g.token) - req.Header.Add("Content-Type", "application/json") - - resp, err := g.client.Do(req) - if err != nil { - log.Error("Failed to push registration file", "error", err) - return err - } - - if resp.StatusCode != http.StatusAccepted { - log.Error("Failed to push registration file", "status", resp.Status) - return fmt.Errorf("request failed, status is %s", resp.Status) - } - - return nil -} - -// PostFile commits the current instance to the configured gitlab repository -// as a global target for other sparrow instances to discover -func (g *gitlab) PostFile(ctx context.Context, body GitlabFile) error { - log := logger.FromContext(ctx).With("name", "AddRegistration") - log.Debug("Registering sparrow instance to gitlab") - - // chose method based on whether the registration has already happened - n := url.PathEscape(body.Content.Url) - b, err := body.Bytes() - if err != nil { - log.Error("Failed to create request", "error", err) - return err - } - req, err := http.NewRequestWithContext(ctx, - http.MethodPost, - fmt.Sprintf("%s/api/v4/projects/%d/repository/files/%s", g.baseUrl, g.projectID, n), - bytes.NewBuffer(b), - ) - if err != nil { - log.Error("Failed to create request", "error", err) - return err - } - - req.Header.Add("PRIVATE-TOKEN", g.token) - req.Header.Add("Content-Type", "application/json") - - resp, err := g.client.Do(req) - if err != nil { - log.Error("Failed to push registration file", "error", err) - return err - } - - if resp.StatusCode != http.StatusCreated { - log.Error("Failed to push registration file", "status", resp.Status) - return fmt.Errorf("request failed, status is %s", resp.Status) - } - - return nil -} diff --git a/pkg/sparrow/targets/gitlab_test.go b/pkg/sparrow/targets/gitlab_test.go index eebe7313..25fb90c4 100644 --- a/pkg/sparrow/targets/gitlab_test.go +++ b/pkg/sparrow/targets/gitlab_test.go @@ -2,282 +2,37 @@ package targets import ( "context" - "fmt" - "net/http" - "reflect" "testing" "time" - "github.com/jarcoal/httpmock" + "github.com/caas-team/sparrow/pkg/checks" + gitlabmock "github.com/caas-team/sparrow/pkg/sparrow/gitlab/test" ) -func Test_gitlab_fetchFileList(t *testing.T) { - type file struct { - Name string `json:"name"` - } - tests := []struct { - name string - want []string - wantErr bool - mockBody []file - mockCode int - }{ - { - name: "success - 0 targets", - want: nil, - wantErr: false, - mockCode: http.StatusOK, - mockBody: []file{}, - }, - { - name: "success - 1 target", - want: []string{ - "test", - }, - wantErr: false, - mockCode: http.StatusOK, - mockBody: []file{ - { - Name: "test", - }, - }, - }, - { - name: "success - 2 targets", - want: []string{ - "test", - "test2", - }, - wantErr: false, - mockCode: http.StatusOK, - mockBody: []file{ - { - Name: "test", - }, - { - Name: "test2", - }, - }, - }, - { - name: "failure - API error", - want: nil, - wantErr: true, - mockCode: http.StatusInternalServerError, - }, - } - - httpmock.Activate() - defer httpmock.DeactivateAndReset() - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - resp, err := httpmock.NewJsonResponder(tt.mockCode, tt.mockBody) - if err != nil { - t.Fatalf("error creating mock response: %v", err) - } - httpmock.RegisterResponder("GET", "http://test/api/v4/projects/1/repository/tree?ref=main", resp) - - g := &gitlab{ - baseUrl: "http://test", - projectID: 1, - token: "test", - client: http.DefaultClient, - } - got, err := g.fetchFileList(context.Background()) - if (err != nil) != tt.wantErr { - t.Errorf("FetchFiles() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("FetchFiles() got = %v, want %v", got, tt.want) - } - }) - } -} - -// The filelist and url are the same, so we HTTP responders can -// be created without much hassle -func Test_gitlab_fetchFiles(t *testing.T) { - tests := []struct { - name string - want []globalTarget - fileList []string - wantErr bool - mockCode int - }{ - { - name: "success - 0 targets", - want: nil, - wantErr: false, - mockCode: http.StatusOK, - }, - { - name: "success - 1 target", - want: []globalTarget{ - { - Url: "test", - LastSeen: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), - }, - }, - fileList: []string{ - "test", - }, - wantErr: false, - mockCode: http.StatusOK, - }, - { - name: "success - 2 targets", - want: []globalTarget{ - { - Url: "test", - LastSeen: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), - }, - { - Url: "test2", - LastSeen: time.Date(2021, 2, 1, 0, 0, 0, 0, time.UTC), - }, - }, - fileList: []string{ - "test", - "test2", - }, - wantErr: false, - mockCode: http.StatusOK, - }, - } - - httpmock.Activate() - defer httpmock.DeactivateAndReset() - g := &gitlab{ - baseUrl: "http://test", - projectID: 1, - token: "test", - client: http.DefaultClient, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // setup mock responses - for i, target := range tt.want { - resp, err := httpmock.NewJsonResponder(tt.mockCode, target) - if err != nil { - t.Fatalf("error creating mock response: %v", err) - } - httpmock.RegisterResponder("GET", fmt.Sprintf("http://test/api/v4/projects/1/repository/files/%s/raw?ref=main", tt.fileList[i]), resp) - } - - got, err := g.fetchFiles(context.Background(), tt.fileList) - if (err != nil) != tt.wantErr { - t.Fatalf("FetchFiles() error = %v, wantErr %v", err, tt.wantErr) - } - if !reflect.DeepEqual(got, tt.want) { - t.Fatalf("FetchFiles() got = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_gitlab_fetchFiles_error_cases(t *testing.T) { - type mockResponses struct { - response globalTarget - err bool - } - - tests := []struct { - name string - mockResponses []mockResponses - fileList []string - }{ - { - name: "failure - direct API error", - mockResponses: []mockResponses{ - { - err: true, - }, - }, - fileList: []string{ - "test", - }, - }, - { - name: "failure - API error after one successful request", - mockResponses: []mockResponses{ - { - response: globalTarget{ - Url: "test", - LastSeen: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), - }, - err: false, - }, - { - response: globalTarget{}, - err: true, - }, - }, - fileList: []string{ - "test", - "test2-will-fail", - }, - }, - } - - httpmock.Activate() - defer httpmock.DeactivateAndReset() - g := &gitlab{ - baseUrl: "http://test", - projectID: 1, - token: "test", - client: http.DefaultClient, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - for i, target := range tt.mockResponses { - if target.err { - errResp := httpmock.NewStringResponder(http.StatusInternalServerError, "") - httpmock.RegisterResponder("GET", fmt.Sprintf("http://test/api/v4/projects/1/repository/files/%s/raw?ref=main", tt.fileList[i]), errResp) - continue - } - resp, err := httpmock.NewJsonResponder(http.StatusOK, target) - if err != nil { - t.Fatalf("error creating mock response: %v", err) - } - httpmock.RegisterResponder("GET", fmt.Sprintf("http://test/api/v4/projects/1/repository/files/%s/raw?ref=main", tt.fileList[i]), resp) - } - - _, err := g.fetchFiles(context.Background(), tt.fileList) - if err == nil { - t.Fatalf("Expected error but got none.") - } - }) - } -} - func Test_gitlabTargetManager_refreshTargets(t *testing.T) { now := time.Now() tooOld := now.Add(-time.Hour * 2) tests := []struct { name string - mockTargets []globalTarget - expectedHealthy []globalTarget + mockTargets []checks.GlobalTarget + expectedHealthy []checks.GlobalTarget wantErr bool }{ { name: "success with 0 targets", - mockTargets: []globalTarget{}, - expectedHealthy: []globalTarget{}, + mockTargets: []checks.GlobalTarget{}, + expectedHealthy: []checks.GlobalTarget{}, }, { name: "success with 1 healthy target", - mockTargets: []globalTarget{ + mockTargets: []checks.GlobalTarget{ { Url: "test", LastSeen: now, }, }, - expectedHealthy: []globalTarget{ + expectedHealthy: []checks.GlobalTarget{ { Url: "test", LastSeen: now, @@ -286,7 +41,7 @@ func Test_gitlabTargetManager_refreshTargets(t *testing.T) { }, { name: "success with 1 unhealthy target", - mockTargets: []globalTarget{ + mockTargets: []checks.GlobalTarget{ { Url: "test", LastSeen: tooOld, @@ -295,7 +50,7 @@ func Test_gitlabTargetManager_refreshTargets(t *testing.T) { }, { name: "success with 1 healthy and 1 unhealthy targets", - mockTargets: []globalTarget{ + mockTargets: []checks.GlobalTarget{ { Url: "test", LastSeen: now, @@ -305,7 +60,7 @@ func Test_gitlabTargetManager_refreshTargets(t *testing.T) { LastSeen: tooOld, }, }, - expectedHealthy: []globalTarget{ + expectedHealthy: []checks.GlobalTarget{ { Url: "test", LastSeen: now, @@ -323,7 +78,7 @@ func Test_gitlabTargetManager_refreshTargets(t *testing.T) { t.Run(tt.name, func(t *testing.T) { gtm := &gitlabTargetManager{ targets: nil, - gitlab: newMockGitlab(tt.mockTargets, tt.wantErr), + gitlab: gitlabmock.New(tt.mockTargets, tt.wantErr), name: "test", unhealthyThreshold: time.Hour, } @@ -333,35 +88,3 @@ func Test_gitlabTargetManager_refreshTargets(t *testing.T) { }) } } - -type mockGitlab struct { - targets []globalTarget - err error -} - -func (m mockGitlab) PutFile(ctx context.Context, file GitlabFile) error { - panic("implement me") -} - -func (m mockGitlab) PostFile(ctx context.Context, f GitlabFile) error { - panic("implement me") -} - -func (m mockGitlab) FetchFiles(ctx context.Context) ([]globalTarget, error) { - return m.targets, m.err -} - -func (m mockGitlab) FetchFileList(ctx context.Context) ([]string, error) { - panic("implement me") -} - -func newMockGitlab(targets []globalTarget, err bool) Gitlab { - var e error - if err { - e = fmt.Errorf("error") - } - return &mockGitlab{ - targets: targets, - err: e, - } -} diff --git a/pkg/sparrow/targets/targetmanager.go b/pkg/sparrow/targets/targetmanager.go new file mode 100644 index 00000000..d7ac279c --- /dev/null +++ b/pkg/sparrow/targets/targetmanager.go @@ -0,0 +1,14 @@ +package targets + +import ( + "context" + + "github.com/caas-team/sparrow/pkg/checks" +) + +// TargetManager handles the management of globalTargets for +// a Sparrow instance +type TargetManager interface { + Reconcile(ctx context.Context) + GetTargets() []checks.GlobalTarget +} diff --git a/pkg/sparrow/targets/targets.go b/pkg/sparrow/targets/targets.go deleted file mode 100644 index 5f1eb2ad..00000000 --- a/pkg/sparrow/targets/targets.go +++ /dev/null @@ -1,19 +0,0 @@ -package targets - -import ( - "context" - "time" -) - -// globalTarget represents a globalTarget that can be checked -type globalTarget struct { - Url string `json:"url"` - LastSeen time.Time `json:"lastSeen"` -} - -// TargetManager handles the management of globalTargets for -// a Sparrow instance -type TargetManager interface { - Reconcile(ctx context.Context) - GetTargets() []globalTarget -}