From 2c85610000677d2a54895b1e319c091f3f22c797 Mon Sep 17 00:00:00 2001 From: Reto Galante Date: Mon, 11 Nov 2024 14:54:37 +0100 Subject: [PATCH 01/47] build and lint using dagger --- ci/.gitattributes | 4 +++ ci/.gitignore | 4 +++ ci/go.mod | 47 +++++++++++++++++++++++++++++ ci/go.sum | 77 +++++++++++++++++++++++++++++++++++++++++++++++ ci/main.go | 39 ++++++++++++++++++++++++ dagger.json | 6 ++++ 6 files changed, 177 insertions(+) create mode 100644 ci/.gitattributes create mode 100644 ci/.gitignore create mode 100644 ci/go.mod create mode 100644 ci/go.sum create mode 100644 ci/main.go create mode 100644 dagger.json diff --git a/ci/.gitattributes b/ci/.gitattributes new file mode 100644 index 00000000..3a454933 --- /dev/null +++ b/ci/.gitattributes @@ -0,0 +1,4 @@ +/dagger.gen.go linguist-generated +/internal/dagger/** linguist-generated +/internal/querybuilder/** linguist-generated +/internal/telemetry/** linguist-generated diff --git a/ci/.gitignore b/ci/.gitignore new file mode 100644 index 00000000..7ebabcc1 --- /dev/null +++ b/ci/.gitignore @@ -0,0 +1,4 @@ +/dagger.gen.go +/internal/dagger +/internal/querybuilder +/internal/telemetry diff --git a/ci/go.mod b/ci/go.mod new file mode 100644 index 00000000..0d019350 --- /dev/null +++ b/ci/go.mod @@ -0,0 +1,47 @@ +module dagger/ci + +go 1.23.2 + +require ( + github.com/99designs/gqlgen v0.17.49 + github.com/Khan/genqlient v0.7.0 + github.com/vektah/gqlparser/v2 v2.5.16 + go.opentelemetry.io/otel v1.27.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240518090000-14441aefdf88 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.3.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 + go.opentelemetry.io/otel/log v0.3.0 + go.opentelemetry.io/otel/sdk v1.27.0 + go.opentelemetry.io/otel/sdk/log v0.3.0 + go.opentelemetry.io/otel/trace v1.27.0 + go.opentelemetry.io/proto/otlp v1.3.1 + golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa + golang.org/x/sync v0.8.0 + google.golang.org/grpc v1.65.0 +) + +require ( + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect + github.com/sosodev/duration v1.3.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect + go.opentelemetry.io/otel/metric v1.27.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect + google.golang.org/protobuf v1.34.2 // indirect +) + +replace go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc => go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240518090000-14441aefdf88 + +replace go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp => go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.3.0 + +replace go.opentelemetry.io/otel/log => go.opentelemetry.io/otel/log v0.3.0 + +replace go.opentelemetry.io/otel/sdk/log => go.opentelemetry.io/otel/sdk/log v0.3.0 diff --git a/ci/go.sum b/ci/go.sum new file mode 100644 index 00000000..f6bb5391 --- /dev/null +++ b/ci/go.sum @@ -0,0 +1,77 @@ +github.com/99designs/gqlgen v0.17.49 h1:b3hNGexHd33fBSAd4NDT/c3NCcQzcAVkknhN9ym36YQ= +github.com/99designs/gqlgen v0.17.49/go.mod h1:tC8YFVZMed81x7UJ7ORUwXF4Kn6SXuucFqQBhN8+BU0= +github.com/Khan/genqlient v0.7.0 h1:GZ1meyRnzcDTK48EjqB8t3bcfYvHArCUUvgOwpz1D4w= +github.com/Khan/genqlient v0.7.0/go.mod h1:HNyy3wZvuYwmW3Y7mkoQLZsa/R5n5yIRajS1kPBvSFM= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/sosodev/duration v1.3.1 h1:qtHBDMQ6lvMQsL15g4aopM4HEfOaYuhWBw3NPTtlqq4= +github.com/sosodev/duration v1.3.1/go.mod h1:RQIBBX0+fMLc/D9+Jb/fwvVmo0eZvDDEERAikUR6SDg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0vy5p8= +github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= +go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= +go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240518090000-14441aefdf88 h1:oM0GTNKGlc5qHctWeIGTVyda4iFFalOzMZ3Ehj5rwB4= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240518090000-14441aefdf88/go.mod h1:JGG8ebaMO5nXOPnvKEl+DiA4MGwFjCbjsxT1WHIEBPY= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.3.0 h1:ccBrA8nCY5mM0y5uO7FT0ze4S0TuFcWdDB2FxGMTjkI= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.3.0/go.mod h1:/9pb6634zi2Lk8LYg9Q0X8Ar6jka4dkFOylBLbVQPCE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 h1:qFffATk0X+HD+f1Z8lswGiOQYKHRlzfmdJm0wEaVrFA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0/go.mod h1:MOiCmryaYtc+V0Ei+Tx9o5S1ZjA7kzLucuVuyzBZloQ= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 h1:QY7/0NeRPKlzusf40ZE4t1VlMKbqSNT7cJRYzWuja0s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0/go.mod h1:HVkSiDhTM9BoUJU8qE6j2eSWLLXvi1USXjyd2BXT8PY= +go.opentelemetry.io/otel/log v0.3.0 h1:kJRFkpUFYtny37NQzL386WbznUByZx186DpEMKhEGZs= +go.opentelemetry.io/otel/log v0.3.0/go.mod h1:ziCwqZr9soYDwGNbIL+6kAvQC+ANvjgG367HVcyR/ys= +go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= +go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= +go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= +go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= +go.opentelemetry.io/otel/sdk/log v0.3.0 h1:GEjJ8iftz2l+XO1GF2856r7yYVh74URiF9JMcAacr5U= +go.opentelemetry.io/otel/sdk/log v0.3.0/go.mod h1:BwCxtmux6ACLuys1wlbc0+vGBd+xytjmjajwqqIul2g= +go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= +go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= +golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 h1:wKguEg1hsxI2/L3hUYrpo1RVi48K+uTyzKqprwLXsb8= +google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142/go.mod h1:d6be+8HhtEtucleCbxpPW9PA9XwISACu8nvpPqF0BVo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 h1:e7S5W7MGGLaSu8j3YjdezkZ+m1/Nm0uRVRMEMGk26Xs= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/ci/main.go b/ci/main.go new file mode 100644 index 00000000..bafbc085 --- /dev/null +++ b/ci/main.go @@ -0,0 +1,39 @@ +// A generated module for Ci functions +// +// This module has been generated via dagger init and serves as a reference to +// basic module structure as you get started with Dagger. +// +// Two functions have been pre-created. You can modify, delete, or add to them, +// as needed. They demonstrate usage of arguments and return types using simple +// echo and grep commands. The functions can be called from the dagger CLI or +// from one of the SDKs. +// +// The first line in this comment block is a short description line and the +// rest is a long description with more detail on the module's purpose or usage, +// if appropriate. All modules should have a short description. + +package main + +import ( + "context" + "dagger/ci/internal/dagger" +) + +type Ci struct{} + +// Returns a Container with installed gems from the gemfile in the provided Directory +func (m *Ci) Build(ctx context.Context, directory *dagger.Directory) *dagger.Container { + return dag.Container(). + From("ruby:latest"). + WithMountedDirectory("/mnt", directory). + WithWorkdir("/mnt"). + WithExec([]string{"gem", "install", "rails"}). + WithExec([]string{"bundle", "config", "bundle", "install"}) +} + +func (m *Ci) Lint(ctx context.Context, directory *dagger.Directory) *dagger.Container { + return m.Build(ctx, directory). + WithExec([]string{"gem", "install", "haml-lint"}). + WithExec([]string{"haml-lint"}) + //WithExec([]string{"haml-lint", "|", "reviewdog", "-efm=\"%f:%l %m\"", "-name=\"HAML-Lint\"", "-reporter=github-pr-review", "-level=error", "-diff=\"git diff\""}) +} diff --git a/dagger.json b/dagger.json new file mode 100644 index 00000000..ff2596a8 --- /dev/null +++ b/dagger.json @@ -0,0 +1,6 @@ +{ + "name": "ci", + "sdk": "go", + "source": "ci", + "engineVersion": "v0.13.5" +} From dabf83960981217e202b2750abc9fe930c4ca8e6 Mon Sep 17 00:00:00 2001 From: Reto Galante Date: Mon, 11 Nov 2024 15:07:54 +0100 Subject: [PATCH 02/47] add missing file, build from dockerfile --- ci/dagger.gen.go | 204 +++++++++++++++++++++++++++++++++++++++++++++++ ci/main.go | 12 +-- 2 files changed, 210 insertions(+), 6 deletions(-) create mode 100644 ci/dagger.gen.go diff --git a/ci/dagger.gen.go b/ci/dagger.gen.go new file mode 100644 index 00000000..ffe454aa --- /dev/null +++ b/ci/dagger.gen.go @@ -0,0 +1,204 @@ +// Code generated by dagger. DO NOT EDIT. + +package main + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "os" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/sdk/resource" + semconv "go.opentelemetry.io/otel/semconv/v1.24.0" + "go.opentelemetry.io/otel/trace" + + "dagger/ci/internal/dagger" + "dagger/ci/internal/querybuilder" + "dagger/ci/internal/telemetry" +) + +var dag = dagger.Connect() + +func Tracer() trace.Tracer { + return otel.Tracer("dagger.io/sdk.go") +} + +// used for local MarshalJSON implementations +var marshalCtx = context.Background() + +// called by main() +func setMarshalContext(ctx context.Context) { + marshalCtx = ctx + dagger.SetMarshalContext(ctx) +} + +type DaggerObject = querybuilder.GraphQLMarshaller + +type ExecError = dagger.ExecError + +// ptr returns a pointer to the given value. +func ptr[T any](v T) *T { + return &v +} + +// convertSlice converts a slice of one type to a slice of another type using a +// converter function +func convertSlice[I any, O any](in []I, f func(I) O) []O { + out := make([]O, len(in)) + for i, v := range in { + out[i] = f(v) + } + return out +} + +func (r Ci) MarshalJSON() ([]byte, error) { + var concrete struct{} + return json.Marshal(&concrete) +} + +func (r *Ci) UnmarshalJSON(bs []byte) error { + var concrete struct{} + err := json.Unmarshal(bs, &concrete) + if err != nil { + return err + } + return nil +} + +func main() { + ctx := context.Background() + + // Direct slog to the new stderr. This is only for dev time debugging, and + // runtime errors/warnings. + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ + Level: slog.LevelWarn, + }))) + + if err := dispatch(ctx); err != nil { + fmt.Println(err.Error()) + os.Exit(2) + } +} + +func dispatch(ctx context.Context) error { + ctx = telemetry.InitEmbedded(ctx, resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String("dagger-go-sdk"), + // TODO version? + )) + defer telemetry.Close() + + // A lot of the "work" actually happens when we're marshalling the return + // value, which entails getting object IDs, which happens in MarshalJSON, + // which has no ctx argument, so we use this lovely global variable. + setMarshalContext(ctx) + + fnCall := dag.CurrentFunctionCall() + parentName, err := fnCall.ParentName(ctx) + if err != nil { + return fmt.Errorf("get parent name: %w", err) + } + fnName, err := fnCall.Name(ctx) + if err != nil { + return fmt.Errorf("get fn name: %w", err) + } + parentJson, err := fnCall.Parent(ctx) + if err != nil { + return fmt.Errorf("get fn parent: %w", err) + } + fnArgs, err := fnCall.InputArgs(ctx) + if err != nil { + return fmt.Errorf("get fn args: %w", err) + } + + inputArgs := map[string][]byte{} + for _, fnArg := range fnArgs { + argName, err := fnArg.Name(ctx) + if err != nil { + return fmt.Errorf("get fn arg name: %w", err) + } + argValue, err := fnArg.Value(ctx) + if err != nil { + return fmt.Errorf("get fn arg value: %w", err) + } + inputArgs[argName] = []byte(argValue) + } + + result, err := invoke(ctx, []byte(parentJson), parentName, fnName, inputArgs) + if err != nil { + return fmt.Errorf("invoke: %w", err) + } + resultBytes, err := json.Marshal(result) + if err != nil { + return fmt.Errorf("marshal: %w", err) + } + if err = fnCall.ReturnValue(ctx, dagger.JSON(resultBytes)); err != nil { + return fmt.Errorf("store return value: %w", err) + } + return nil +} +func invoke(ctx context.Context, parentJSON []byte, parentName string, fnName string, inputArgs map[string][]byte) (_ any, err error) { + _ = inputArgs + switch parentName { + case "Ci": + switch fnName { + case "ContainerEcho": + var parent Ci + err = json.Unmarshal(parentJSON, &parent) + if err != nil { + panic(fmt.Errorf("%s: %w", "failed to unmarshal parent object", err)) + } + var stringArg string + if inputArgs["stringArg"] != nil { + err = json.Unmarshal([]byte(inputArgs["stringArg"]), &stringArg) + if err != nil { + panic(fmt.Errorf("%s: %w", "failed to unmarshal input arg stringArg", err)) + } + } + return (*Ci).ContainerEcho(&parent, stringArg), nil + case "GrepDir": + var parent Ci + err = json.Unmarshal(parentJSON, &parent) + if err != nil { + panic(fmt.Errorf("%s: %w", "failed to unmarshal parent object", err)) + } + var directoryArg *dagger.Directory + if inputArgs["directoryArg"] != nil { + err = json.Unmarshal([]byte(inputArgs["directoryArg"]), &directoryArg) + if err != nil { + panic(fmt.Errorf("%s: %w", "failed to unmarshal input arg directoryArg", err)) + } + } + var pattern string + if inputArgs["pattern"] != nil { + err = json.Unmarshal([]byte(inputArgs["pattern"]), &pattern) + if err != nil { + panic(fmt.Errorf("%s: %w", "failed to unmarshal input arg pattern", err)) + } + } + return (*Ci).GrepDir(&parent, ctx, directoryArg, pattern) + default: + return nil, fmt.Errorf("unknown function %s", fnName) + } + case "": + return dag.Module(). + WithDescription("A generated module for Ci functions\n\nThis module has been generated via dagger init and serves as a reference to\nbasic module structure as you get started with Dagger.\n\nTwo functions have been pre-created. You can modify, delete, or add to them,\nas needed. They demonstrate usage of arguments and return types using simple\necho and grep commands. The functions can be called from the dagger CLI or\nfrom one of the SDKs.\n\nThe first line in this comment block is a short description line and the\nrest is a long description with more detail on the module's purpose or usage,\nif appropriate. All modules should have a short description.\n"). + WithObject( + dag.TypeDef().WithObject("Ci"). + WithFunction( + dag.Function("ContainerEcho", + dag.TypeDef().WithObject("Container")). + WithDescription("Returns a container that echoes whatever string argument is provided"). + WithArg("stringArg", dag.TypeDef().WithKind(dagger.StringKind))). + WithFunction( + dag.Function("GrepDir", + dag.TypeDef().WithKind(dagger.StringKind)). + WithDescription("Returns lines that match a pattern in the files of the provided Directory"). + WithArg("directoryArg", dag.TypeDef().WithObject("Directory")). + WithArg("pattern", dag.TypeDef().WithKind(dagger.StringKind)))), nil + default: + return nil, fmt.Errorf("unknown object %s", parentName) + } +} diff --git a/ci/main.go b/ci/main.go index bafbc085..7dad8ed5 100644 --- a/ci/main.go +++ b/ci/main.go @@ -23,12 +23,12 @@ type Ci struct{} // Returns a Container with installed gems from the gemfile in the provided Directory func (m *Ci) Build(ctx context.Context, directory *dagger.Directory) *dagger.Container { - return dag.Container(). - From("ruby:latest"). - WithMountedDirectory("/mnt", directory). - WithWorkdir("/mnt"). - WithExec([]string{"gem", "install", "rails"}). - WithExec([]string{"bundle", "config", "bundle", "install"}) + return dag.Container().Build(directory) +// From("ruby:latest"). +// WithMountedDirectory("/mnt", directory). +// WithWorkdir("/mnt"). +// WithExec([]string{"gem", "install", "rails"}). +// WithExec([]string{"bundle", "config", "bundle", "install"}) } func (m *Ci) Lint(ctx context.Context, directory *dagger.Directory) *dagger.Container { From eb548519507d1c560ead020cb77fe29edf5ea69e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20B=C3=BChlmann?= Date: Mon, 11 Nov 2024 15:36:03 +0100 Subject: [PATCH 03/47] Removes generated dagger.gen.go --- ci/dagger.gen.go | 204 ----------------------------------------------- 1 file changed, 204 deletions(-) delete mode 100644 ci/dagger.gen.go diff --git a/ci/dagger.gen.go b/ci/dagger.gen.go deleted file mode 100644 index ffe454aa..00000000 --- a/ci/dagger.gen.go +++ /dev/null @@ -1,204 +0,0 @@ -// Code generated by dagger. DO NOT EDIT. - -package main - -import ( - "context" - "encoding/json" - "fmt" - "log/slog" - "os" - - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/sdk/resource" - semconv "go.opentelemetry.io/otel/semconv/v1.24.0" - "go.opentelemetry.io/otel/trace" - - "dagger/ci/internal/dagger" - "dagger/ci/internal/querybuilder" - "dagger/ci/internal/telemetry" -) - -var dag = dagger.Connect() - -func Tracer() trace.Tracer { - return otel.Tracer("dagger.io/sdk.go") -} - -// used for local MarshalJSON implementations -var marshalCtx = context.Background() - -// called by main() -func setMarshalContext(ctx context.Context) { - marshalCtx = ctx - dagger.SetMarshalContext(ctx) -} - -type DaggerObject = querybuilder.GraphQLMarshaller - -type ExecError = dagger.ExecError - -// ptr returns a pointer to the given value. -func ptr[T any](v T) *T { - return &v -} - -// convertSlice converts a slice of one type to a slice of another type using a -// converter function -func convertSlice[I any, O any](in []I, f func(I) O) []O { - out := make([]O, len(in)) - for i, v := range in { - out[i] = f(v) - } - return out -} - -func (r Ci) MarshalJSON() ([]byte, error) { - var concrete struct{} - return json.Marshal(&concrete) -} - -func (r *Ci) UnmarshalJSON(bs []byte) error { - var concrete struct{} - err := json.Unmarshal(bs, &concrete) - if err != nil { - return err - } - return nil -} - -func main() { - ctx := context.Background() - - // Direct slog to the new stderr. This is only for dev time debugging, and - // runtime errors/warnings. - slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ - Level: slog.LevelWarn, - }))) - - if err := dispatch(ctx); err != nil { - fmt.Println(err.Error()) - os.Exit(2) - } -} - -func dispatch(ctx context.Context) error { - ctx = telemetry.InitEmbedded(ctx, resource.NewWithAttributes( - semconv.SchemaURL, - semconv.ServiceNameKey.String("dagger-go-sdk"), - // TODO version? - )) - defer telemetry.Close() - - // A lot of the "work" actually happens when we're marshalling the return - // value, which entails getting object IDs, which happens in MarshalJSON, - // which has no ctx argument, so we use this lovely global variable. - setMarshalContext(ctx) - - fnCall := dag.CurrentFunctionCall() - parentName, err := fnCall.ParentName(ctx) - if err != nil { - return fmt.Errorf("get parent name: %w", err) - } - fnName, err := fnCall.Name(ctx) - if err != nil { - return fmt.Errorf("get fn name: %w", err) - } - parentJson, err := fnCall.Parent(ctx) - if err != nil { - return fmt.Errorf("get fn parent: %w", err) - } - fnArgs, err := fnCall.InputArgs(ctx) - if err != nil { - return fmt.Errorf("get fn args: %w", err) - } - - inputArgs := map[string][]byte{} - for _, fnArg := range fnArgs { - argName, err := fnArg.Name(ctx) - if err != nil { - return fmt.Errorf("get fn arg name: %w", err) - } - argValue, err := fnArg.Value(ctx) - if err != nil { - return fmt.Errorf("get fn arg value: %w", err) - } - inputArgs[argName] = []byte(argValue) - } - - result, err := invoke(ctx, []byte(parentJson), parentName, fnName, inputArgs) - if err != nil { - return fmt.Errorf("invoke: %w", err) - } - resultBytes, err := json.Marshal(result) - if err != nil { - return fmt.Errorf("marshal: %w", err) - } - if err = fnCall.ReturnValue(ctx, dagger.JSON(resultBytes)); err != nil { - return fmt.Errorf("store return value: %w", err) - } - return nil -} -func invoke(ctx context.Context, parentJSON []byte, parentName string, fnName string, inputArgs map[string][]byte) (_ any, err error) { - _ = inputArgs - switch parentName { - case "Ci": - switch fnName { - case "ContainerEcho": - var parent Ci - err = json.Unmarshal(parentJSON, &parent) - if err != nil { - panic(fmt.Errorf("%s: %w", "failed to unmarshal parent object", err)) - } - var stringArg string - if inputArgs["stringArg"] != nil { - err = json.Unmarshal([]byte(inputArgs["stringArg"]), &stringArg) - if err != nil { - panic(fmt.Errorf("%s: %w", "failed to unmarshal input arg stringArg", err)) - } - } - return (*Ci).ContainerEcho(&parent, stringArg), nil - case "GrepDir": - var parent Ci - err = json.Unmarshal(parentJSON, &parent) - if err != nil { - panic(fmt.Errorf("%s: %w", "failed to unmarshal parent object", err)) - } - var directoryArg *dagger.Directory - if inputArgs["directoryArg"] != nil { - err = json.Unmarshal([]byte(inputArgs["directoryArg"]), &directoryArg) - if err != nil { - panic(fmt.Errorf("%s: %w", "failed to unmarshal input arg directoryArg", err)) - } - } - var pattern string - if inputArgs["pattern"] != nil { - err = json.Unmarshal([]byte(inputArgs["pattern"]), &pattern) - if err != nil { - panic(fmt.Errorf("%s: %w", "failed to unmarshal input arg pattern", err)) - } - } - return (*Ci).GrepDir(&parent, ctx, directoryArg, pattern) - default: - return nil, fmt.Errorf("unknown function %s", fnName) - } - case "": - return dag.Module(). - WithDescription("A generated module for Ci functions\n\nThis module has been generated via dagger init and serves as a reference to\nbasic module structure as you get started with Dagger.\n\nTwo functions have been pre-created. You can modify, delete, or add to them,\nas needed. They demonstrate usage of arguments and return types using simple\necho and grep commands. The functions can be called from the dagger CLI or\nfrom one of the SDKs.\n\nThe first line in this comment block is a short description line and the\nrest is a long description with more detail on the module's purpose or usage,\nif appropriate. All modules should have a short description.\n"). - WithObject( - dag.TypeDef().WithObject("Ci"). - WithFunction( - dag.Function("ContainerEcho", - dag.TypeDef().WithObject("Container")). - WithDescription("Returns a container that echoes whatever string argument is provided"). - WithArg("stringArg", dag.TypeDef().WithKind(dagger.StringKind))). - WithFunction( - dag.Function("GrepDir", - dag.TypeDef().WithKind(dagger.StringKind)). - WithDescription("Returns lines that match a pattern in the files of the provided Directory"). - WithArg("directoryArg", dag.TypeDef().WithObject("Directory")). - WithArg("pattern", dag.TypeDef().WithKind(dagger.StringKind)))), nil - default: - return nil, fmt.Errorf("unknown object %s", parentName) - } -} From a54acc8d4b5f0156728dbf79a91afd831f52aa58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20B=C3=BChlmann?= Date: Mon, 11 Nov 2024 15:56:41 +0100 Subject: [PATCH 04/47] Call linter by dagger --- .github/workflows/dagger-lint.yaml | 14 ++++++++++ .github/workflows/reusable-lint.yaml | 38 ---------------------------- ci/go.mod | 11 +++++--- ci/go.sum | 18 ++++++++----- dagger.json | 4 +-- 5 files changed, 35 insertions(+), 50 deletions(-) create mode 100644 .github/workflows/dagger-lint.yaml delete mode 100644 .github/workflows/reusable-lint.yaml diff --git a/.github/workflows/dagger-lint.yaml b/.github/workflows/dagger-lint.yaml new file mode 100644 index 00000000..e80a795a --- /dev/null +++ b/.github/workflows/dagger-lint.yaml @@ -0,0 +1,14 @@ +name: 'Code Style Review' +on: + workflow_call: +jobs: + lint: + runs-on: 'ubuntu-latest' + steps: + - name: 'lint' + uses: dagger/dagger-for-github@v6 + with: + verb: call + args: lint --directory=./ + # TODO: add the token as github secret + #cloud-token: ${{ secrets.DAGGER_CLOUD_TOKEN }} diff --git a/.github/workflows/reusable-lint.yaml b/.github/workflows/reusable-lint.yaml deleted file mode 100644 index 96e86a7c..00000000 --- a/.github/workflows/reusable-lint.yaml +++ /dev/null @@ -1,38 +0,0 @@ -name: 'Code Style Review' - -on: - workflow_call: - -jobs: - lint: - runs-on: 'ubuntu-latest' - - steps: - - uses: actions/checkout@v4 - - - uses: ruby/setup-ruby@v1 - with: - bundler-cache: true - - - uses: reviewdog/action-rubocop@v2 - with: - rubocop_version: gemfile - rubocop_extensions: rubocop-minitest:gemfile rubocop-performance:gemfile rubocop-rails:gemfile - # reporter: github-pr-review - level: error - - - run: 'gem install haml-lint' - - - uses: reviewdog/action-setup@v1 - - - name: 'Run Reviewdog HAML-Lint' - env: - REVIEWDOG_GITHUB_API_TOKEN: ${{ github.token }} - run: | - haml-lint | - reviewdog \ - -efm="%f:%l %m" \ - -name="HAML-Lint" \ - -reporter=github-pr-review \ - -level=error \ - -diff="git diff $DIFF_BRANCH" diff --git a/ci/go.mod b/ci/go.mod index 0d019350..2ad00529 100644 --- a/ci/go.mod +++ b/ci/go.mod @@ -3,9 +3,9 @@ module dagger/ci go 1.23.2 require ( - github.com/99designs/gqlgen v0.17.49 + github.com/99designs/gqlgen v0.17.55 github.com/Khan/genqlient v0.7.0 - github.com/vektah/gqlparser/v2 v2.5.16 + github.com/vektah/gqlparser/v2 v2.5.17 go.opentelemetry.io/otel v1.27.0 go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240518090000-14441aefdf88 go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.3.0 @@ -28,10 +28,13 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect github.com/sosodev/duration v1.3.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.27.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect - go.opentelemetry.io/otel/metric v1.27.0 // indirect + go.opentelemetry.io/otel/metric v1.27.0 + go.opentelemetry.io/otel/sdk/metric v1.27.0 golang.org/x/net v0.29.0 // indirect - golang.org/x/sys v0.25.0 // indirect + golang.org/x/sys v0.26.0 // indirect golang.org/x/text v0.18.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect diff --git a/ci/go.sum b/ci/go.sum index f6bb5391..c52dd24b 100644 --- a/ci/go.sum +++ b/ci/go.sum @@ -1,5 +1,5 @@ -github.com/99designs/gqlgen v0.17.49 h1:b3hNGexHd33fBSAd4NDT/c3NCcQzcAVkknhN9ym36YQ= -github.com/99designs/gqlgen v0.17.49/go.mod h1:tC8YFVZMed81x7UJ7ORUwXF4Kn6SXuucFqQBhN8+BU0= +github.com/99designs/gqlgen v0.17.55 h1:3vzrNWYyzSZjGDFo68e5j9sSauLxfKvLp+6ioRokVtM= +github.com/99designs/gqlgen v0.17.55/go.mod h1:3Bq768f8hgVPGZxL8aY9MaYmbxa6llPM/qu1IGH1EJo= github.com/Khan/genqlient v0.7.0 h1:GZ1meyRnzcDTK48EjqB8t3bcfYvHArCUUvgOwpz1D4w= github.com/Khan/genqlient v0.7.0/go.mod h1:HNyy3wZvuYwmW3Y7mkoQLZsa/R5n5yIRajS1kPBvSFM= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= @@ -27,14 +27,18 @@ github.com/sosodev/duration v1.3.1 h1:qtHBDMQ6lvMQsL15g4aopM4HEfOaYuhWBw3NPTtlqq github.com/sosodev/duration v1.3.1/go.mod h1:RQIBBX0+fMLc/D9+Jb/fwvVmo0eZvDDEERAikUR6SDg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0vy5p8= -github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= +github.com/vektah/gqlparser/v2 v2.5.17 h1:9At7WblLV7/36nulgekUgIaqHZWn5hxqluxrxGUhOmI= +github.com/vektah/gqlparser/v2 v2.5.17/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240518090000-14441aefdf88 h1:oM0GTNKGlc5qHctWeIGTVyda4iFFalOzMZ3Ehj5rwB4= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240518090000-14441aefdf88/go.mod h1:JGG8ebaMO5nXOPnvKEl+DiA4MGwFjCbjsxT1WHIEBPY= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.3.0 h1:ccBrA8nCY5mM0y5uO7FT0ze4S0TuFcWdDB2FxGMTjkI= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.3.0/go.mod h1:/9pb6634zi2Lk8LYg9Q0X8Ar6jka4dkFOylBLbVQPCE= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.27.0 h1:bFgvUr3/O4PHj3VQcFEuYKvRZJX1SJDQ+11JXuSB3/w= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.27.0/go.mod h1:xJntEd2KL6Qdg5lwp97HMLQDVeAhrYxmzFseAMDPQ8I= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0 h1:CIHWikMsN3wO+wq1Tp5VGdVRTcON+DmOJSfDjXypKOc= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0/go.mod h1:TNupZ6cxqyFEpLXAZW7On+mLFL0/g0TE3unIYL91xWc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 h1:qFffATk0X+HD+f1Z8lswGiOQYKHRlzfmdJm0wEaVrFA= @@ -49,6 +53,8 @@ go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kT go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= go.opentelemetry.io/otel/sdk/log v0.3.0 h1:GEjJ8iftz2l+XO1GF2856r7yYVh74URiF9JMcAacr5U= go.opentelemetry.io/otel/sdk/log v0.3.0/go.mod h1:BwCxtmux6ACLuys1wlbc0+vGBd+xytjmjajwqqIul2g= +go.opentelemetry.io/otel/sdk/metric v1.27.0 h1:5uGNOlpXi+Hbo/DRoI31BSb1v+OGcpv2NemcCrOL8gI= +go.opentelemetry.io/otel/sdk/metric v1.27.0/go.mod h1:we7jJVrYN2kh3mVBlswtPU22K0SA+769l93J6bsyvqw= go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= @@ -61,8 +67,8 @@ golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 h1:wKguEg1hsxI2/L3hUYrpo1RVi48K+uTyzKqprwLXsb8= diff --git a/dagger.json b/dagger.json index ff2596a8..131359a3 100644 --- a/dagger.json +++ b/dagger.json @@ -1,6 +1,6 @@ { "name": "ci", + "engineVersion": "v0.14.0", "sdk": "go", - "source": "ci", - "engineVersion": "v0.13.5" + "source": "ci" } From a02b4922ae67d917fedc31b9c7dec5b91cd330ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20B=C3=BChlmann?= Date: Mon, 11 Nov 2024 16:01:19 +0100 Subject: [PATCH 05/47] Run lint on push to branch dagger-ci --- .github/workflows/dagger-lint.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dagger-lint.yaml b/.github/workflows/dagger-lint.yaml index e80a795a..4a21b597 100644 --- a/.github/workflows/dagger-lint.yaml +++ b/.github/workflows/dagger-lint.yaml @@ -1,6 +1,8 @@ name: 'Code Style Review' on: - workflow_call: + push: + branches: + - dagger-ci jobs: lint: runs-on: 'ubuntu-latest' From 7d24052c45deab9604dcc8cfbe427203bc65f8dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20B=C3=BChlmann?= Date: Mon, 11 Nov 2024 16:06:02 +0100 Subject: [PATCH 06/47] Use dagger engine v0.14.0 --- .github/workflows/dagger-lint.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/dagger-lint.yaml b/.github/workflows/dagger-lint.yaml index 4a21b597..75c8afd3 100644 --- a/.github/workflows/dagger-lint.yaml +++ b/.github/workflows/dagger-lint.yaml @@ -12,5 +12,6 @@ jobs: with: verb: call args: lint --directory=./ + version: '0.14.0' # TODO: add the token as github secret #cloud-token: ${{ secrets.DAGGER_CLOUD_TOKEN }} From ac5d073cce0f09c05475f840ceae30f6a12f0780 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20B=C3=BChlmann?= Date: Mon, 11 Nov 2024 16:19:11 +0100 Subject: [PATCH 07/47] Add module to lint action --- .github/workflows/dagger-lint.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/dagger-lint.yaml b/.github/workflows/dagger-lint.yaml index 75c8afd3..c07ce8b6 100644 --- a/.github/workflows/dagger-lint.yaml +++ b/.github/workflows/dagger-lint.yaml @@ -11,6 +11,7 @@ jobs: uses: dagger/dagger-for-github@v6 with: verb: call + module: ci args: lint --directory=./ version: '0.14.0' # TODO: add the token as github secret From 98c254342794f69a2fab27abf3a13f86babfd37f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20B=C3=BChlmann?= Date: Mon, 11 Nov 2024 16:22:31 +0100 Subject: [PATCH 08/47] Set working directory --- .github/workflows/dagger-lint.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/dagger-lint.yaml b/.github/workflows/dagger-lint.yaml index c07ce8b6..db349900 100644 --- a/.github/workflows/dagger-lint.yaml +++ b/.github/workflows/dagger-lint.yaml @@ -9,6 +9,7 @@ jobs: steps: - name: 'lint' uses: dagger/dagger-for-github@v6 + working-directory: './' with: verb: call module: ci From e2e67ad80460b72161164217c4f415a36a5d6e6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20B=C3=BChlmann?= Date: Mon, 11 Nov 2024 16:32:59 +0100 Subject: [PATCH 09/47] Add checkout action --- .github/workflows/dagger-lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dagger-lint.yaml b/.github/workflows/dagger-lint.yaml index db349900..900875bd 100644 --- a/.github/workflows/dagger-lint.yaml +++ b/.github/workflows/dagger-lint.yaml @@ -7,9 +7,9 @@ jobs: lint: runs-on: 'ubuntu-latest' steps: + - uses: actions/checkout@v4 - name: 'lint' uses: dagger/dagger-for-github@v6 - working-directory: './' with: verb: call module: ci From c494a78093890ab2ad4482b20edb4aaccf0d71b6 Mon Sep 17 00:00:00 2001 From: Reto Galante Date: Mon, 11 Nov 2024 17:11:06 +0100 Subject: [PATCH 10/47] fix lint step --- ci/main.go | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/ci/main.go b/ci/main.go index 7dad8ed5..f0de7c29 100644 --- a/ci/main.go +++ b/ci/main.go @@ -21,19 +21,18 @@ import ( type Ci struct{} -// Returns a Container with installed gems from the gemfile in the provided Directory +// Returns a Container built from the Dockerfile in the provided Directory func (m *Ci) Build(ctx context.Context, directory *dagger.Directory) *dagger.Container { return dag.Container().Build(directory) -// From("ruby:latest"). -// WithMountedDirectory("/mnt", directory). -// WithWorkdir("/mnt"). -// WithExec([]string{"gem", "install", "rails"}). -// WithExec([]string{"bundle", "config", "bundle", "install"}) } -func (m *Ci) Lint(ctx context.Context, directory *dagger.Directory) *dagger.Container { - return m.Build(ctx, directory). - WithExec([]string{"gem", "install", "haml-lint"}). - WithExec([]string{"haml-lint"}) - //WithExec([]string{"haml-lint", "|", "reviewdog", "-efm=\"%f:%l %m\"", "-name=\"HAML-Lint\"", "-reporter=github-pr-review", "-level=error", "-diff=\"git diff\""}) +// Returns the result of haml-lint run against the sources in the provided Directory +func (m *Ci) Lint(ctx context.Context, directory *dagger.Directory) (string, error) { + return dag.Container(). + From("ruby:latest"). + WithMountedDirectory("/mnt", directory). + WithWorkdir("/mnt"). + WithExec([]string{"gem", "install", "haml-lint"}). + WithExec([]string{"haml-lint", "--reporter", "json", "."}). + Stdout(ctx) } From e16d612c6a1154edb2d07cf9b033fab1bf416a9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20B=C3=BChlmann?= Date: Mon, 11 Nov 2024 18:13:30 +0100 Subject: [PATCH 11/47] Add services for memcached and postgresql --- ci/main.go | 68 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/ci/main.go b/ci/main.go index f0de7c29..36ed70ad 100644 --- a/ci/main.go +++ b/ci/main.go @@ -17,22 +17,70 @@ package main import ( "context" "dagger/ci/internal/dagger" + "fmt" ) type Ci struct{} // Returns a Container built from the Dockerfile in the provided Directory -func (m *Ci) Build(ctx context.Context, directory *dagger.Directory) *dagger.Container { - return dag.Container().Build(directory) +func (m *Ci) Build(_ context.Context, dir *dagger.Directory) *dagger.Container { + return dag.Container().Build(dir) } // Returns the result of haml-lint run against the sources in the provided Directory -func (m *Ci) Lint(ctx context.Context, directory *dagger.Directory) (string, error) { - return dag.Container(). - From("ruby:latest"). - WithMountedDirectory("/mnt", directory). - WithWorkdir("/mnt"). - WithExec([]string{"gem", "install", "haml-lint"}). - WithExec([]string{"haml-lint", "--reporter", "json", "."}). - Stdout(ctx) +func (m *Ci) Lint(ctx context.Context, dir *dagger.Directory) (string, error) { + return dag.Container(). + From("ruby:latest"). + WithMountedDirectory("/mnt", dir). + WithWorkdir("/mnt"). + WithExec([]string{"gem", "install", "haml-lint"}). + WithExec([]string{"haml-lint", "--reporter", "json", "."}). + Stdout(ctx) +} + +// Creates a PostgreSQL service for local testing based on the official image with the provided version. If no version is provided, 'latest' will be used. +func (m *Ci) Postgres( + _ context.Context, + // +optional + // +default="latest" + version string) *dagger.Service { + + return dag.Container(). + From(fmt.Sprintf("postgres:%s", version)). + WithEnvVariable("POSTGRES_PASSWORD", "postgres"). + WithExposedPort(5432). + AsService() +} + +// Creates a memcached service for local testing based on the official image with the provided version. If no version is provided, 'latest' will be used. +func (m *Ci) Memcached( + _ context.Context, + // +optional + // +default="latest" + version string) *dagger.Service { + + return dag.Container(). + From(fmt.Sprintf("postgres:%s", version)). + WithEnvVariable("POSTGRES_PASSWORD", "postgres"). + WithExposedPort(5432). + AsService() +} + +// Executes the test suite for the Rails application in the provided Directory +func (m *Ci) Test(ctx context.Context, dir *dagger.Directory) *dagger.Container { + return m.Build(ctx, dir).From("ruby:latest"). + WithEnvVariable("RAILS_TEST_DB_NAME", "postgres"). + WithEnvVariable("RAILS_TEST_DB_USERNAME", "postgres"). + WithEnvVariable("RAILS_TEST_DB_PASSWORD", "postgres"). + WithEnvVariable("RAILS_ENV", "test"). + WithEnvVariable("CI", "true"). + WithEnvVariable("PGDATESTYLE", "German"). + WithExec([]string{"sudo", "apt-get", "-yqq", "update"}). + WithExec([]string{"sudo", "apt-get", "-yqq", "install", "libpq-dev", "libvips-dev"}). + WithExec([]string{"gem", "install", "bundler", "--version", "'~> 2'"}). + WithExec([]string{"bundle", "install", "--jobs", "4", "--retry", "3"}). + WithExec([]string{"bundle", "exec", "rails", "db:create"}). + WithExec([]string{"bundle", "exec", "rails", "db:migrate"}). + WithExec([]string{"bundle", "exec", "rails", "assets:precompile"}). + WithExec([]string{"bundle", "exec", "rails", "test"}) } From b6dfe30b809dd52387f43f20c21139605fa5ce43 Mon Sep 17 00:00:00 2001 From: Sylvain Gilgen Date: Tue, 12 Nov 2024 08:31:21 +0100 Subject: [PATCH 12/47] add brakeman step --- ci/main.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ci/main.go b/ci/main.go index 36ed70ad..75b362cc 100644 --- a/ci/main.go +++ b/ci/main.go @@ -38,6 +38,17 @@ func (m *Ci) Lint(ctx context.Context, dir *dagger.Directory) (string, error) { Stdout(ctx) } +// Returns the Sast report as a file +func (m *Ci) Sast(ctx context.Context, directory *dagger.Directory) *dagger.File { + return dag.Container(). + From("presidentbeef/brakeman:latest"). + WithMountedDirectory("/app", directory). + WithWorkdir("/app"). + WithExec([]string{"/usr/src/app/bin/brakeman", }). + File("/app/brakeman-output.tabs") +} + + // Creates a PostgreSQL service for local testing based on the official image with the provided version. If no version is provided, 'latest' will be used. func (m *Ci) Postgres( _ context.Context, From 181c36a8066a5d2cf6d6744f46f413feafb89bf6 Mon Sep 17 00:00:00 2001 From: Sylvain Gilgen Date: Tue, 12 Nov 2024 10:24:09 +0100 Subject: [PATCH 13/47] add sbom creation and vulnerability scan --- ci/main.go | 43 ++++++++++++++++++++++++++++++++----------- dagger.json | 7 +++++++ 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/ci/main.go b/ci/main.go index 75b362cc..09129d32 100644 --- a/ci/main.go +++ b/ci/main.go @@ -40,15 +40,14 @@ func (m *Ci) Lint(ctx context.Context, dir *dagger.Directory) (string, error) { // Returns the Sast report as a file func (m *Ci) Sast(ctx context.Context, directory *dagger.Directory) *dagger.File { - return dag.Container(). - From("presidentbeef/brakeman:latest"). - WithMountedDirectory("/app", directory). - WithWorkdir("/app"). - WithExec([]string{"/usr/src/app/bin/brakeman", }). - File("/app/brakeman-output.tabs") + return dag.Container(). + From("presidentbeef/brakeman:latest"). + WithMountedDirectory("/app", directory). + WithWorkdir("/app"). + WithExec([]string{"/usr/src/app/bin/brakeman"}). + File("/app/brakeman-output.tabs") } - // Creates a PostgreSQL service for local testing based on the official image with the provided version. If no version is provided, 'latest' will be used. func (m *Ci) Postgres( _ context.Context, @@ -79,19 +78,41 @@ func (m *Ci) Memcached( // Executes the test suite for the Rails application in the provided Directory func (m *Ci) Test(ctx context.Context, dir *dagger.Directory) *dagger.Container { - return m.Build(ctx, dir).From("ruby:latest"). + return dag.Container().From("ruby:latest"). + WithMountedDirectory("/app", dir). + WithWorkdir("/app"). WithEnvVariable("RAILS_TEST_DB_NAME", "postgres"). WithEnvVariable("RAILS_TEST_DB_USERNAME", "postgres"). WithEnvVariable("RAILS_TEST_DB_PASSWORD", "postgres"). WithEnvVariable("RAILS_ENV", "test"). WithEnvVariable("CI", "true"). WithEnvVariable("PGDATESTYLE", "German"). - WithExec([]string{"sudo", "apt-get", "-yqq", "update"}). - WithExec([]string{"sudo", "apt-get", "-yqq", "install", "libpq-dev", "libvips-dev"}). - WithExec([]string{"gem", "install", "bundler", "--version", "'~> 2'"}). + WithExec([]string{"apt-get", "-yqq", "update"}). + WithExec([]string{"apt-get", "-yqq", "install", "libpq-dev", "libvips-dev"}). + WithExec([]string{"gem", "install", "bundler"}). WithExec([]string{"bundle", "install", "--jobs", "4", "--retry", "3"}). WithExec([]string{"bundle", "exec", "rails", "db:create"}). WithExec([]string{"bundle", "exec", "rails", "db:migrate"}). WithExec([]string{"bundle", "exec", "rails", "assets:precompile"}). WithExec([]string{"bundle", "exec", "rails", "test"}) } + +func (m *Ci) Sbom(ctx context.Context, container *dagger.Container) *dagger.File { + trivy := dag.Trivy(dagger.TrivyOpts{ + DatabaseRepository: "public.ecr.aws/aquasecurity/trivy-db", + }) + + sbom := trivy.Container(container). + Report("spdx-json"). + WithName("spdx.json") + + return sbom +} + +func (m *Ci) Vulnscan(ctx context.Context, sbom *dagger.File) *dagger.File { + trivy := dag.Trivy(dagger.TrivyOpts{ + DatabaseRepository: "public.ecr.aws/aquasecurity/trivy-db", + }) + + return trivy.Sbom(sbom).Report("json") +} diff --git a/dagger.json b/dagger.json index 131359a3..c6e61d7a 100644 --- a/dagger.json +++ b/dagger.json @@ -2,5 +2,12 @@ "name": "ci", "engineVersion": "v0.14.0", "sdk": "go", + "dependencies": [ + { + "name": "trivy", + "source": "github.com/sagikazarmark/daggerverse/trivy@trivy/v0.5.0", + "pin": "5b826062b6bc1bfbd619aa5d0fba117190c85aba" + } + ], "source": "ci" } From e9a1d136fa771d7d293f3063cc6385192c80280b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20B=C3=BChlmann?= Date: Tue, 12 Nov 2024 08:47:10 +0100 Subject: [PATCH 14/47] Use memcached image --- ci/main.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ci/main.go b/ci/main.go index 09129d32..7ab7151d 100644 --- a/ci/main.go +++ b/ci/main.go @@ -70,9 +70,8 @@ func (m *Ci) Memcached( version string) *dagger.Service { return dag.Container(). - From(fmt.Sprintf("postgres:%s", version)). - WithEnvVariable("POSTGRES_PASSWORD", "postgres"). - WithExposedPort(5432). + From(fmt.Sprintf("memcached:%s", version)). + WithExposedPort(11211). AsService() } From 3e49bddedba8dbc2bb97815fb7df0d0ebe8aafd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20B=C3=BChlmann?= Date: Tue, 12 Nov 2024 10:51:15 +0100 Subject: [PATCH 15/47] Implement integration testing --- ci/main.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/ci/main.go b/ci/main.go index 7ab7151d..d4dca238 100644 --- a/ci/main.go +++ b/ci/main.go @@ -39,10 +39,10 @@ func (m *Ci) Lint(ctx context.Context, dir *dagger.Directory) (string, error) { } // Returns the Sast report as a file -func (m *Ci) Sast(ctx context.Context, directory *dagger.Directory) *dagger.File { +func (m *Ci) Sast(ctx context.Context, dir *dagger.Directory) *dagger.File { return dag.Container(). From("presidentbeef/brakeman:latest"). - WithMountedDirectory("/app", directory). + WithMountedDirectory("/app", dir). WithWorkdir("/app"). WithExec([]string{"/usr/src/app/bin/brakeman"}). File("/app/brakeman-output.tabs") @@ -77,18 +77,24 @@ func (m *Ci) Memcached( // Executes the test suite for the Rails application in the provided Directory func (m *Ci) Test(ctx context.Context, dir *dagger.Directory) *dagger.Container { - return dag.Container().From("ruby:latest"). - WithMountedDirectory("/app", dir). - WithWorkdir("/app"). + + return dag.Container(). + From("ruby:latest"). + WithServiceBinding("postgresql", m.Postgres(ctx, "11")). + WithServiceBinding("memcached", m.Memcached(ctx, "latest")). + WithMountedDirectory("/mnt", dir). + WithWorkdir("/mnt"). + WithEnvVariable("RAILS_DB_HOST", "postgresql"). // This is the service name of the postgres container called by rails + WithEnvVariable("RAILS_TEST_DB_HOST", "postgresql"). WithEnvVariable("RAILS_TEST_DB_NAME", "postgres"). WithEnvVariable("RAILS_TEST_DB_USERNAME", "postgres"). WithEnvVariable("RAILS_TEST_DB_PASSWORD", "postgres"). WithEnvVariable("RAILS_ENV", "test"). WithEnvVariable("CI", "true"). WithEnvVariable("PGDATESTYLE", "German"). - WithExec([]string{"apt-get", "-yqq", "update"}). + WithExec([]string{"apt-get", "update"}). WithExec([]string{"apt-get", "-yqq", "install", "libpq-dev", "libvips-dev"}). - WithExec([]string{"gem", "install", "bundler"}). + WithExec([]string{"gem", "install", "bundler", "--version", "~> 2"}). WithExec([]string{"bundle", "install", "--jobs", "4", "--retry", "3"}). WithExec([]string{"bundle", "exec", "rails", "db:create"}). WithExec([]string{"bundle", "exec", "rails", "db:migrate"}). From e53efab1b62cea7e3c18cbf4f96f7496af49cdd5 Mon Sep 17 00:00:00 2001 From: Sylvain Gilgen Date: Tue, 12 Nov 2024 10:58:08 +0100 Subject: [PATCH 16/47] add comments, sdd SbomBuild method --- ci/main.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/ci/main.go b/ci/main.go index d4dca238..621109f1 100644 --- a/ci/main.go +++ b/ci/main.go @@ -102,18 +102,25 @@ func (m *Ci) Test(ctx context.Context, dir *dagger.Directory) *dagger.Container WithExec([]string{"bundle", "exec", "rails", "test"}) } +// Creates an SBOM for the container func (m *Ci) Sbom(ctx context.Context, container *dagger.Container) *dagger.File { - trivy := dag.Trivy(dagger.TrivyOpts{ - DatabaseRepository: "public.ecr.aws/aquasecurity/trivy-db", - }) + trivy := dag.Trivy() sbom := trivy.Container(container). - Report("spdx-json"). - WithName("spdx.json") + Report("cyclonedx"). + WithName("cyclonedx.json") return sbom } +// Builds the container and creates an SBOM for it +func (m *Ci) SbomBuild(ctx context.Context, dir *dagger.Directory) *dagger.File { + container := m.Build(ctx, dir) + + return m.Sbom(ctx, container) +} + +// Scans the SBOM for vulnerabilities func (m *Ci) Vulnscan(ctx context.Context, sbom *dagger.File) *dagger.File { trivy := dag.Trivy(dagger.TrivyOpts{ DatabaseRepository: "public.ecr.aws/aquasecurity/trivy-db", From ee5266ebeb1bff4074325c223036b2960852556c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20B=C3=BChlmann?= Date: Tue, 12 Nov 2024 11:24:52 +0100 Subject: [PATCH 17/47] Integrate dagger cloud --- .github/workflows/dagger-lint.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dagger-lint.yaml b/.github/workflows/dagger-lint.yaml index 900875bd..5a6a2cba 100644 --- a/.github/workflows/dagger-lint.yaml +++ b/.github/workflows/dagger-lint.yaml @@ -13,7 +13,6 @@ jobs: with: verb: call module: ci - args: lint --directory=./ + args: lint --dir=./ version: '0.14.0' - # TODO: add the token as github secret - #cloud-token: ${{ secrets.DAGGER_CLOUD_TOKEN }} + cloud-token: ${{ secrets.DAGGER_CLOUD_TOKEN }} From 535d6445e906e6d13635b3a7e4c1d059c95f9271 Mon Sep 17 00:00:00 2001 From: Reto Galante Date: Tue, 12 Nov 2024 14:56:09 +0100 Subject: [PATCH 18/47] add all step --- ci/main.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/ci/main.go b/ci/main.go index 621109f1..03258b9f 100644 --- a/ci/main.go +++ b/ci/main.go @@ -22,6 +22,13 @@ import ( type Ci struct{} +type Results struct { + LintOutput string + SecurityScan *dagger.File + VulnerabilityScan *dagger.File + Image *dagger.Container +} + // Returns a Container built from the Dockerfile in the provided Directory func (m *Ci) Build(_ context.Context, dir *dagger.Directory) *dagger.Container { return dag.Container().Build(dir) @@ -34,7 +41,7 @@ func (m *Ci) Lint(ctx context.Context, dir *dagger.Directory) (string, error) { WithMountedDirectory("/mnt", dir). WithWorkdir("/mnt"). WithExec([]string{"gem", "install", "haml-lint"}). - WithExec([]string{"haml-lint", "--reporter", "json", "."}). + WithExec([]string{"haml-lint", "-r", "json", "."}). Stdout(ctx) } @@ -128,3 +135,17 @@ func (m *Ci) Vulnscan(ctx context.Context, sbom *dagger.File) *dagger.File { return trivy.Sbom(sbom).Report("json") } + +// Executes all the steps and returns a Results object +func (m *Ci) Ci(ctx context.Context, dir *dagger.Directory) *Results { + lintOutput, _ := m.Lint(ctx, dir) + securityScan := m.Sast(ctx, dir) + vulnerabilityScan := m.Vulnscan(ctx, m.SbomBuild(ctx, dir)) + image := m.Build(ctx, dir) + return &Results{ + LintOutput: lintOutput, + SecurityScan: securityScan, + VulnerabilityScan: vulnerabilityScan, + Image: image, + } +} From 05a1fb05645cfcbc2839286a9efe75555c8f2cd7 Mon Sep 17 00:00:00 2001 From: Sylvain Gilgen Date: Tue, 12 Nov 2024 16:07:19 +0100 Subject: [PATCH 19/47] set repository for trivy --- ci/main.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/ci/main.go b/ci/main.go index 03258b9f..856efca8 100644 --- a/ci/main.go +++ b/ci/main.go @@ -110,8 +110,13 @@ func (m *Ci) Test(ctx context.Context, dir *dagger.Directory) *dagger.Container } // Creates an SBOM for the container -func (m *Ci) Sbom(ctx context.Context, container *dagger.Container) *dagger.File { - trivy := dag.Trivy() +func (m *Ci) Sbom(container *dagger.Container) *dagger.File { + trivy_container := dag.Container().From("aquasec/trivy").WithEnvVariable("TRIVY_JAVA_DB_REPOSITORY", "public.ecr.aws/aquasecurity/trivy-java-db") + + trivy := dag.Trivy(dagger.TrivyOpts{ + Container: trivy_container, + DatabaseRepository: "public.ecr.aws/aquasecurity/trivy-db", + }) sbom := trivy.Container(container). Report("cyclonedx"). @@ -124,12 +129,15 @@ func (m *Ci) Sbom(ctx context.Context, container *dagger.Container) *dagger.File func (m *Ci) SbomBuild(ctx context.Context, dir *dagger.Directory) *dagger.File { container := m.Build(ctx, dir) - return m.Sbom(ctx, container) + return m.Sbom(container) } // Scans the SBOM for vulnerabilities -func (m *Ci) Vulnscan(ctx context.Context, sbom *dagger.File) *dagger.File { +func (m *Ci) Vulnscan(sbom *dagger.File) *dagger.File { + trivy_container := dag.Container().From("aquasec/trivy").WithEnvVariable("TRIVY_JAVA_DB_REPOSITORY", "public.ecr.aws/aquasecurity/trivy-java-db") + trivy := dag.Trivy(dagger.TrivyOpts{ + Container: trivy_container, DatabaseRepository: "public.ecr.aws/aquasecurity/trivy-db", }) @@ -140,8 +148,9 @@ func (m *Ci) Vulnscan(ctx context.Context, sbom *dagger.File) *dagger.File { func (m *Ci) Ci(ctx context.Context, dir *dagger.Directory) *Results { lintOutput, _ := m.Lint(ctx, dir) securityScan := m.Sast(ctx, dir) - vulnerabilityScan := m.Vulnscan(ctx, m.SbomBuild(ctx, dir)) image := m.Build(ctx, dir) + sbom := m.Sbom(image) + vulnerabilityScan := m.Vulnscan(sbom) return &Results{ LintOutput: lintOutput, SecurityScan: securityScan, From 612828f7da54558d8def9f1e3bd9e62f7c1a8dd1 Mon Sep 17 00:00:00 2001 From: Christoph Raaflaub Date: Tue, 12 Nov 2024 16:24:31 +0100 Subject: [PATCH 20/47] run steps in parallel --- ci/main.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/ci/main.go b/ci/main.go index 856efca8..30f0f9c4 100644 --- a/ci/main.go +++ b/ci/main.go @@ -18,6 +18,7 @@ import ( "context" "dagger/ci/internal/dagger" "fmt" + "sync" ) type Ci struct{} @@ -158,3 +159,36 @@ func (m *Ci) Ci(ctx context.Context, dir *dagger.Directory) *Results { Image: image, } } + +// Executes all the steps and returns a Results object +func (m *Ci) CiIntegration(ctx context.Context, dir *dagger.Directory) *Results { + var wg sync.WaitGroup + wg.Add(3) + + var lintOutput, _ = func() (string, error) { + defer wg.Done() + return "empty", error(nil) //m.Lint(ctx, dir) + }() + + var securityScan = func() *dagger.File { + defer wg.Done() + return m.Sast(ctx, dir) + }() + + //vulnerabilityScan := m.Vulnscan(ctx, m.SbomBuild(ctx, dir)) + + var image = func() *dagger.Container { + defer wg.Done() + return m.Build(ctx, dir) + }() + + // This Blocks the execution until its counter become 0 + wg.Wait() + + return &Results{ + LintOutput: lintOutput, + SecurityScan: securityScan, + // VulnerabilityScan: vulnerabilityScan, + Image: image, + } +} From 5d0ea1523c5b7870da2a8e821d41dc649217b4d3 Mon Sep 17 00:00:00 2001 From: Reto Galante Date: Tue, 12 Nov 2024 16:31:32 +0100 Subject: [PATCH 21/47] dedup trivy --- ci/main.go | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/ci/main.go b/ci/main.go index 856efca8..b4b0e594 100644 --- a/ci/main.go +++ b/ci/main.go @@ -111,12 +111,7 @@ func (m *Ci) Test(ctx context.Context, dir *dagger.Directory) *dagger.Container // Creates an SBOM for the container func (m *Ci) Sbom(container *dagger.Container) *dagger.File { - trivy_container := dag.Container().From("aquasec/trivy").WithEnvVariable("TRIVY_JAVA_DB_REPOSITORY", "public.ecr.aws/aquasecurity/trivy-java-db") - - trivy := dag.Trivy(dagger.TrivyOpts{ - Container: trivy_container, - DatabaseRepository: "public.ecr.aws/aquasecurity/trivy-db", - }) + trivy := m.Trivy() sbom := trivy.Container(container). Report("cyclonedx"). @@ -134,14 +129,21 @@ func (m *Ci) SbomBuild(ctx context.Context, dir *dagger.Directory) *dagger.File // Scans the SBOM for vulnerabilities func (m *Ci) Vulnscan(sbom *dagger.File) *dagger.File { - trivy_container := dag.Container().From("aquasec/trivy").WithEnvVariable("TRIVY_JAVA_DB_REPOSITORY", "public.ecr.aws/aquasecurity/trivy-java-db") + trivy := m.Trivy() + + return trivy.Sbom(sbom).Report("json") +} + +// Creates and returns a custom trivy instance +func (m *Ci) Trivy() *dagger.Trivy { + trivy_container := dag.Container(). + From("aquasec/trivy"). + WithEnvVariable("TRIVY_JAVA_DB_REPOSITORY", "public.ecr.aws/aquasecurity/trivy-java-db") - trivy := dag.Trivy(dagger.TrivyOpts{ + return dag.Trivy(dagger.TrivyOpts{ Container: trivy_container, DatabaseRepository: "public.ecr.aws/aquasecurity/trivy-db", }) - - return trivy.Sbom(sbom).Report("json") } // Executes all the steps and returns a Results object From 3dc71f71466e451fd17835e2aed03f22c3d51953 Mon Sep 17 00:00:00 2001 From: Reto Galante Date: Tue, 12 Nov 2024 16:40:31 +0100 Subject: [PATCH 22/47] redup trivy --- ci/main.go | 55 ++++++++++++------------------------------------------ 1 file changed, 12 insertions(+), 43 deletions(-) diff --git a/ci/main.go b/ci/main.go index 96e55c05..b1a52788 100644 --- a/ci/main.go +++ b/ci/main.go @@ -18,7 +18,6 @@ import ( "context" "dagger/ci/internal/dagger" "fmt" - "sync" ) type Ci struct{} @@ -112,7 +111,14 @@ func (m *Ci) Test(ctx context.Context, dir *dagger.Directory) *dagger.Container // Creates an SBOM for the container func (m *Ci) Sbom(container *dagger.Container) *dagger.File { - trivy := m.Trivy() + trivy_container := dag.Container(). + From("aquasec/trivy"). + WithEnvVariable("TRIVY_JAVA_DB_REPOSITORY", "public.ecr.aws/aquasecurity/trivy-java-db") + + trivy := dag.Trivy(dagger.TrivyOpts{ + Container: trivy_container, + DatabaseRepository: "public.ecr.aws/aquasecurity/trivy-db", + }) sbom := trivy.Container(container). Report("cyclonedx"). @@ -130,21 +136,16 @@ func (m *Ci) SbomBuild(ctx context.Context, dir *dagger.Directory) *dagger.File // Scans the SBOM for vulnerabilities func (m *Ci) Vulnscan(sbom *dagger.File) *dagger.File { - trivy := m.Trivy() - - return trivy.Sbom(sbom).Report("json") -} - -// Creates and returns a custom trivy instance -func (m *Ci) Trivy() *dagger.Trivy { trivy_container := dag.Container(). From("aquasec/trivy"). WithEnvVariable("TRIVY_JAVA_DB_REPOSITORY", "public.ecr.aws/aquasecurity/trivy-java-db") - return dag.Trivy(dagger.TrivyOpts{ + trivy := dag.Trivy(dagger.TrivyOpts{ Container: trivy_container, DatabaseRepository: "public.ecr.aws/aquasecurity/trivy-db", }) + + return trivy.Sbom(sbom).Report("json") } // Executes all the steps and returns a Results object @@ -154,43 +155,11 @@ func (m *Ci) Ci(ctx context.Context, dir *dagger.Directory) *Results { image := m.Build(ctx, dir) sbom := m.Sbom(image) vulnerabilityScan := m.Vulnscan(sbom) - return &Results{ - LintOutput: lintOutput, - SecurityScan: securityScan, - VulnerabilityScan: vulnerabilityScan, - Image: image, - } -} - -// Executes all the steps and returns a Results object -func (m *Ci) CiIntegration(ctx context.Context, dir *dagger.Directory) *Results { - var wg sync.WaitGroup - wg.Add(3) - - var lintOutput, _ = func() (string, error) { - defer wg.Done() - return "empty", error(nil) //m.Lint(ctx, dir) - }() - - var securityScan = func() *dagger.File { - defer wg.Done() - return m.Sast(ctx, dir) - }() - - //vulnerabilityScan := m.Vulnscan(ctx, m.SbomBuild(ctx, dir)) - - var image = func() *dagger.Container { - defer wg.Done() - return m.Build(ctx, dir) - }() - - // This Blocks the execution until its counter become 0 - wg.Wait() return &Results{ LintOutput: lintOutput, SecurityScan: securityScan, - // VulnerabilityScan: vulnerabilityScan, + VulnerabilityScan: vulnerabilityScan, Image: image, } } From b988bf75c244d90aa4960fd7ecbdcc0d1cf3ba89 Mon Sep 17 00:00:00 2001 From: Christoph Raaflaub Date: Tue, 12 Nov 2024 21:00:13 +0100 Subject: [PATCH 23/47] add dagger pipeline --- .github/workflows/dagger.yaml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/dagger.yaml diff --git a/.github/workflows/dagger.yaml b/.github/workflows/dagger.yaml new file mode 100644 index 00000000..dbb8fb2f --- /dev/null +++ b/.github/workflows/dagger.yaml @@ -0,0 +1,29 @@ +name: 'Full Dagger Pipeline' +on: + push: + branches: + - dagger-ci +jobs: + lint: + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@v4 + - name: 'pipeline' + uses: dagger/dagger-for-github@v7.0.1 + with: + # Dagger Version + version: 0.14.0 + # Dagger CLI Flags + #dagger-flags: # optional, default is --progress plain + # CLI verb (call, run, download, up, functions, shell, query) + verb: call + # The working directory in which to run the Dagger CLI + workdir: . + # Dagger Cloud Token + cloud-token: ${{ secrets.DAGGER_CLOUD_TOKEN }} + # Dagger module to call. Local or Git + module: . + # Arguments to pass to CLI + args: ci-integration --dir=./ + - name: Test Reporter + uses: dorny/test-reporter@v1.9.1 From c65e9966d9513dfcd2e36b3e3095beff9b787730 Mon Sep 17 00:00:00 2001 From: Christoph Raaflaub Date: Tue, 12 Nov 2024 21:07:52 +0100 Subject: [PATCH 24/47] readd ci-integration function --- .github/workflows/dagger.yaml | 2 +- ci/main.go | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dagger.yaml b/.github/workflows/dagger.yaml index dbb8fb2f..f3829958 100644 --- a/.github/workflows/dagger.yaml +++ b/.github/workflows/dagger.yaml @@ -4,7 +4,7 @@ on: branches: - dagger-ci jobs: - lint: + pipeline: runs-on: 'ubuntu-latest' steps: - uses: actions/checkout@v4 diff --git a/ci/main.go b/ci/main.go index b1a52788..0dd44d96 100644 --- a/ci/main.go +++ b/ci/main.go @@ -18,6 +18,7 @@ import ( "context" "dagger/ci/internal/dagger" "fmt" + "sync" ) type Ci struct{} @@ -163,3 +164,36 @@ func (m *Ci) Ci(ctx context.Context, dir *dagger.Directory) *Results { Image: image, } } + +// Executes all the steps and returns a Results object +func (m *Ci) CiIntegration(ctx context.Context, dir *dagger.Directory) *Results { + var wg sync.WaitGroup + wg.Add(3) + + var lintOutput, _ = func() (string, error) { + defer wg.Done() + return "empty", error(nil) //m.Lint(ctx, dir) + }() + + var securityScan = func() *dagger.File { + defer wg.Done() + return m.Sast(ctx, dir) + }() + + //vulnerabilityScan := m.Vulnscan(ctx, m.SbomBuild(ctx, dir)) + + var image = func() *dagger.Container { + defer wg.Done() + return m.Build(ctx, dir) + }() + + // This Blocks the execution until its counter become 0 + wg.Wait() + + return &Results{ + LintOutput: lintOutput, + SecurityScan: securityScan, + // VulnerabilityScan: vulnerabilityScan, + Image: image, + } +} From e4c5270df4634a1630676ccb598701423d28cc36 Mon Sep 17 00:00:00 2001 From: Reto Galante Date: Wed, 13 Nov 2024 09:52:44 +0100 Subject: [PATCH 25/47] export files, add docu --- ci/main.go | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/ci/main.go b/ci/main.go index 0dd44d96..69f2aec9 100644 --- a/ci/main.go +++ b/ci/main.go @@ -18,16 +18,20 @@ import ( "context" "dagger/ci/internal/dagger" "fmt" - "sync" + "sync" ) type Ci struct{} type Results struct { - LintOutput string - SecurityScan *dagger.File + // haml-lint output as json + LintOutput *dagger.File + // brakeman output as plain text + SecurityScan *dagger.File + // trivy results as json VulnerabilityScan *dagger.File - Image *dagger.Container + // the built image + Image *dagger.Container } // Returns a Container built from the Dockerfile in the provided Directory @@ -36,14 +40,14 @@ func (m *Ci) Build(_ context.Context, dir *dagger.Directory) *dagger.Container { } // Returns the result of haml-lint run against the sources in the provided Directory -func (m *Ci) Lint(ctx context.Context, dir *dagger.Directory) (string, error) { +func (m *Ci) Lint(ctx context.Context, dir *dagger.Directory) *dagger.File { return dag.Container(). From("ruby:latest"). WithMountedDirectory("/mnt", dir). WithWorkdir("/mnt"). WithExec([]string{"gem", "install", "haml-lint"}). - WithExec([]string{"haml-lint", "-r", "json", "."}). - Stdout(ctx) + WithExec([]string{"sh", "-c", "haml-lint -r json . > lint.json || true"}). + File("lint.json") } // Returns the Sast report as a file @@ -151,7 +155,7 @@ func (m *Ci) Vulnscan(sbom *dagger.File) *dagger.File { // Executes all the steps and returns a Results object func (m *Ci) Ci(ctx context.Context, dir *dagger.Directory) *Results { - lintOutput, _ := m.Lint(ctx, dir) + lintOutput := m.Lint(ctx, dir) securityScan := m.Sast(ctx, dir) image := m.Build(ctx, dir) sbom := m.Sbom(image) @@ -170,9 +174,9 @@ func (m *Ci) CiIntegration(ctx context.Context, dir *dagger.Directory) *Results var wg sync.WaitGroup wg.Add(3) - var lintOutput, _ = func() (string, error) { + var lintOutput = func() *dagger.File { defer wg.Done() - return "empty", error(nil) //m.Lint(ctx, dir) + return m.Lint(ctx, dir) }() var securityScan = func() *dagger.File { @@ -188,12 +192,12 @@ func (m *Ci) CiIntegration(ctx context.Context, dir *dagger.Directory) *Results }() // This Blocks the execution until its counter become 0 - wg.Wait() + wg.Wait() return &Results{ - LintOutput: lintOutput, - SecurityScan: securityScan, - // VulnerabilityScan: vulnerabilityScan, - Image: image, + LintOutput: lintOutput, + SecurityScan: securityScan, + // VulnerabilityScan: vulnerabilityScan, + Image: image, } } From cf29681375401029bf6ff60b2e4d790b94ecfae4 Mon Sep 17 00:00:00 2001 From: Reto Galante Date: Wed, 13 Nov 2024 10:16:22 +0100 Subject: [PATCH 26/47] integrate vulnerabilty scan --- ci/main.go | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/ci/main.go b/ci/main.go index 69f2aec9..f3ec6921 100644 --- a/ci/main.go +++ b/ci/main.go @@ -40,7 +40,7 @@ func (m *Ci) Build(_ context.Context, dir *dagger.Directory) *dagger.Container { } // Returns the result of haml-lint run against the sources in the provided Directory -func (m *Ci) Lint(ctx context.Context, dir *dagger.Directory) *dagger.File { +func (m *Ci) Lint(dir *dagger.Directory) *dagger.File { return dag.Container(). From("ruby:latest"). WithMountedDirectory("/mnt", dir). @@ -51,7 +51,7 @@ func (m *Ci) Lint(ctx context.Context, dir *dagger.Directory) *dagger.File { } // Returns the Sast report as a file -func (m *Ci) Sast(ctx context.Context, dir *dagger.Directory) *dagger.File { +func (m *Ci) Sast(dir *dagger.Directory) *dagger.File { return dag.Container(). From("presidentbeef/brakeman:latest"). WithMountedDirectory("/app", dir). @@ -155,8 +155,8 @@ func (m *Ci) Vulnscan(sbom *dagger.File) *dagger.File { // Executes all the steps and returns a Results object func (m *Ci) Ci(ctx context.Context, dir *dagger.Directory) *Results { - lintOutput := m.Lint(ctx, dir) - securityScan := m.Sast(ctx, dir) + lintOutput := m.Lint(dir) + securityScan := m.Sast(dir) image := m.Build(ctx, dir) sbom := m.Sbom(image) vulnerabilityScan := m.Vulnscan(sbom) @@ -172,19 +172,22 @@ func (m *Ci) Ci(ctx context.Context, dir *dagger.Directory) *Results { // Executes all the steps and returns a Results object func (m *Ci) CiIntegration(ctx context.Context, dir *dagger.Directory) *Results { var wg sync.WaitGroup - wg.Add(3) + wg.Add(4) var lintOutput = func() *dagger.File { defer wg.Done() - return m.Lint(ctx, dir) + return m.Lint(dir) }() var securityScan = func() *dagger.File { defer wg.Done() - return m.Sast(ctx, dir) + return m.Sast(dir) }() - //vulnerabilityScan := m.Vulnscan(ctx, m.SbomBuild(ctx, dir)) + var vulnerabilityScan = func() *dagger.File { + defer wg.Done() + return m.Vulnscan(m.Sbom(m.Build(ctx, dir))) + }() var image = func() *dagger.Container { defer wg.Done() @@ -195,9 +198,9 @@ func (m *Ci) CiIntegration(ctx context.Context, dir *dagger.Directory) *Results wg.Wait() return &Results{ - LintOutput: lintOutput, - SecurityScan: securityScan, - // VulnerabilityScan: vulnerabilityScan, - Image: image, + LintOutput: lintOutput, + SecurityScan: securityScan, + VulnerabilityScan: vulnerabilityScan, + Image: image, } } From 228aadbc0b887093cef141ba18c5e0abba932e6a Mon Sep 17 00:00:00 2001 From: Reto Galante Date: Wed, 13 Nov 2024 10:39:28 +0100 Subject: [PATCH 27/47] fix dagger-lint --- .github/workflows/dagger-lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dagger-lint.yaml b/.github/workflows/dagger-lint.yaml index 5a6a2cba..901bc7ed 100644 --- a/.github/workflows/dagger-lint.yaml +++ b/.github/workflows/dagger-lint.yaml @@ -9,7 +9,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: 'lint' - uses: dagger/dagger-for-github@v6 + uses: dagger/dagger-for-github@v7.0.1 with: verb: call module: ci From 995a6049eb68c0d4690ddb066a09e6a234dcc769 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20B=C3=BChlmann?= Date: Wed, 13 Nov 2024 11:09:40 +0100 Subject: [PATCH 28/47] Fixes test job, excluding integration tests --- ci/main.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/ci/main.go b/ci/main.go index f3ec6921..911c0eee 100644 --- a/ci/main.go +++ b/ci/main.go @@ -91,10 +91,11 @@ func (m *Ci) Memcached( func (m *Ci) Test(ctx context.Context, dir *dagger.Directory) *dagger.Container { return dag.Container(). - From("ruby:latest"). + From("registry.puzzle.ch/docker.io/ruby:3.2"). + WithExec([]string{"useradd", "-m", "-u", "1001", "ruby"}). WithServiceBinding("postgresql", m.Postgres(ctx, "11")). WithServiceBinding("memcached", m.Memcached(ctx, "latest")). - WithMountedDirectory("/mnt", dir). + WithMountedDirectory("/mnt", dir, dagger.ContainerWithMountedDirectoryOpts{Owner: "ruby"}). WithWorkdir("/mnt"). WithEnvVariable("RAILS_DB_HOST", "postgresql"). // This is the service name of the postgres container called by rails WithEnvVariable("RAILS_TEST_DB_HOST", "postgresql"). @@ -104,14 +105,18 @@ func (m *Ci) Test(ctx context.Context, dir *dagger.Directory) *dagger.Container WithEnvVariable("RAILS_ENV", "test"). WithEnvVariable("CI", "true"). WithEnvVariable("PGDATESTYLE", "German"). + WithEnvVariable("TZ", "Europe/Zurich"). + WithEnvVariable("DEBIAN_FRONTEND", "noninteractive"). WithExec([]string{"apt-get", "update"}). - WithExec([]string{"apt-get", "-yqq", "install", "libpq-dev", "libvips-dev"}). - WithExec([]string{"gem", "install", "bundler", "--version", "~> 2"}). + WithExec([]string{"apt-get", "-yqq", "install", "libpq-dev", "libvips-dev", "chromium", "graphviz", "imagemagick"}). + WithUser("ruby"). + WithExec([]string{"gem", "update", "--system"}). // update bundler version + WithExec([]string{"gem", "install", "bundler", "--version", `~>2`}). WithExec([]string{"bundle", "install", "--jobs", "4", "--retry", "3"}). WithExec([]string{"bundle", "exec", "rails", "db:create"}). WithExec([]string{"bundle", "exec", "rails", "db:migrate"}). WithExec([]string{"bundle", "exec", "rails", "assets:precompile"}). - WithExec([]string{"bundle", "exec", "rails", "test"}) + WithExec([]string{"bundle", "exec", "rails", "test", "test/controllers", "test/domain", "test/fabricators", "test/fixtures", "test/helpers", "test/mailers", "test/models", "test/presenters", "test/support", "test/tarantula"}) } // Creates an SBOM for the container From 6111d058ce932fffdaa2f4c3e6d1e17d5f2f8e8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20B=C3=BChlmann?= Date: Wed, 13 Nov 2024 11:25:28 +0100 Subject: [PATCH 29/47] Add job for test execution --- .github/workflows/dagger-test.yaml | 18 ++++++++++++++++++ .github/workflows/dagger.yaml | 4 ++++ ci/main.go | 1 + 3 files changed, 23 insertions(+) create mode 100644 .github/workflows/dagger-test.yaml diff --git a/.github/workflows/dagger-test.yaml b/.github/workflows/dagger-test.yaml new file mode 100644 index 00000000..fb829a2d --- /dev/null +++ b/.github/workflows/dagger-test.yaml @@ -0,0 +1,18 @@ +name: 'Test' +on: + push: + branches: + - dagger-ci +jobs: + test: + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@v4 + - name: 'test' + uses: dagger/dagger-for-github@v7.0.1 + with: + verb: call + module: ci + args: test --dir=./ + version: '0.14.0' + cloud-token: ${{ secrets.DAGGER_CLOUD_TOKEN }} diff --git a/.github/workflows/dagger.yaml b/.github/workflows/dagger.yaml index f3829958..8b672245 100644 --- a/.github/workflows/dagger.yaml +++ b/.github/workflows/dagger.yaml @@ -27,3 +27,7 @@ jobs: args: ci-integration --dir=./ - name: Test Reporter uses: dorny/test-reporter@v1.9.1 + with: + name: JEST Tests # Name of the check run which will be created + path: reports/jest-*.xml # Path to test results + reporter: jest-junit # Format of test results diff --git a/ci/main.go b/ci/main.go index 911c0eee..c9700098 100644 --- a/ci/main.go +++ b/ci/main.go @@ -116,6 +116,7 @@ func (m *Ci) Test(ctx context.Context, dir *dagger.Directory) *dagger.Container WithExec([]string{"bundle", "exec", "rails", "db:create"}). WithExec([]string{"bundle", "exec", "rails", "db:migrate"}). WithExec([]string{"bundle", "exec", "rails", "assets:precompile"}). + Terminal(). WithExec([]string{"bundle", "exec", "rails", "test", "test/controllers", "test/domain", "test/fabricators", "test/fixtures", "test/helpers", "test/mailers", "test/models", "test/presenters", "test/support", "test/tarantula"}) } From 324ce47ed3c7b2a490520b3d2943bf95ace9d2ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20B=C3=BChlmann?= Date: Wed, 13 Nov 2024 11:25:44 +0100 Subject: [PATCH 30/47] Fix test function --- ci/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/main.go b/ci/main.go index c9700098..911c0eee 100644 --- a/ci/main.go +++ b/ci/main.go @@ -116,7 +116,6 @@ func (m *Ci) Test(ctx context.Context, dir *dagger.Directory) *dagger.Container WithExec([]string{"bundle", "exec", "rails", "db:create"}). WithExec([]string{"bundle", "exec", "rails", "db:migrate"}). WithExec([]string{"bundle", "exec", "rails", "assets:precompile"}). - Terminal(). WithExec([]string{"bundle", "exec", "rails", "test", "test/controllers", "test/domain", "test/fabricators", "test/fixtures", "test/helpers", "test/mailers", "test/models", "test/presenters", "test/support", "test/tarantula"}) } From 608ca5640ab14203f1ae7a1237eb208959a6a2fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20B=C3=BChlmann?= Date: Wed, 13 Nov 2024 13:51:41 +0100 Subject: [PATCH 31/47] Add test step to ci job --- ci/main.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ci/main.go b/ci/main.go index 911c0eee..74f1c5ee 100644 --- a/ci/main.go +++ b/ci/main.go @@ -116,6 +116,7 @@ func (m *Ci) Test(ctx context.Context, dir *dagger.Directory) *dagger.Container WithExec([]string{"bundle", "exec", "rails", "db:create"}). WithExec([]string{"bundle", "exec", "rails", "db:migrate"}). WithExec([]string{"bundle", "exec", "rails", "assets:precompile"}). + Terminal(). WithExec([]string{"bundle", "exec", "rails", "test", "test/controllers", "test/domain", "test/fabricators", "test/fixtures", "test/helpers", "test/mailers", "test/models", "test/presenters", "test/support", "test/tarantula"}) } @@ -177,7 +178,7 @@ func (m *Ci) Ci(ctx context.Context, dir *dagger.Directory) *Results { // Executes all the steps and returns a Results object func (m *Ci) CiIntegration(ctx context.Context, dir *dagger.Directory) *Results { var wg sync.WaitGroup - wg.Add(4) + wg.Add(5) var lintOutput = func() *dagger.File { defer wg.Done() @@ -199,6 +200,11 @@ func (m *Ci) CiIntegration(ctx context.Context, dir *dagger.Directory) *Results return m.Build(ctx, dir) }() + go func() { + defer wg.Done() + m.Test(ctx, dir) + }() + // This Blocks the execution until its counter become 0 wg.Wait() From ae0c986457f949877067d9a3b83e02554efe7306 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20B=C3=BChlmann?= Date: Wed, 13 Nov 2024 14:33:08 +0100 Subject: [PATCH 32/47] Remove test reporter --- .github/workflows/dagger.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/dagger.yaml b/.github/workflows/dagger.yaml index 8b672245..c446f30f 100644 --- a/.github/workflows/dagger.yaml +++ b/.github/workflows/dagger.yaml @@ -25,9 +25,9 @@ jobs: module: . # Arguments to pass to CLI args: ci-integration --dir=./ - - name: Test Reporter - uses: dorny/test-reporter@v1.9.1 - with: - name: JEST Tests # Name of the check run which will be created - path: reports/jest-*.xml # Path to test results - reporter: jest-junit # Format of test results +# - name: Test Reporter +# uses: dorny/test-reporter@v1.9.1 +# with: +# name: JEST Tests # Name of the check run which will be created +# path: reports/jest-*.xml # Path to test results +# reporter: jest-junit # Format of test results From 75253119006188df8fcfe91b5c3a2664bfa48bb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20B=C3=BChlmann?= Date: Wed, 13 Nov 2024 14:38:35 +0100 Subject: [PATCH 33/47] Remove terminal stmt --- ci/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/main.go b/ci/main.go index 74f1c5ee..cb6e3da4 100644 --- a/ci/main.go +++ b/ci/main.go @@ -116,7 +116,6 @@ func (m *Ci) Test(ctx context.Context, dir *dagger.Directory) *dagger.Container WithExec([]string{"bundle", "exec", "rails", "db:create"}). WithExec([]string{"bundle", "exec", "rails", "db:migrate"}). WithExec([]string{"bundle", "exec", "rails", "assets:precompile"}). - Terminal(). WithExec([]string{"bundle", "exec", "rails", "test", "test/controllers", "test/domain", "test/fabricators", "test/fixtures", "test/helpers", "test/mailers", "test/models", "test/presenters", "test/support", "test/tarantula"}) } From e8f231095e6c2a73390e1ed6b2e7d661a604146d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20B=C3=BChlmann?= Date: Wed, 13 Nov 2024 16:43:01 +0100 Subject: [PATCH 34/47] Integrate Minitest reports --- .github/workflows/dagger.yaml | 12 +++--- Gemfile | 3 ++ ci/main.go | 79 ++++++++++++++++++++--------------- test/test_helper.rb | 26 ++++++------ 4 files changed, 67 insertions(+), 53 deletions(-) diff --git a/.github/workflows/dagger.yaml b/.github/workflows/dagger.yaml index c446f30f..8a5efc36 100644 --- a/.github/workflows/dagger.yaml +++ b/.github/workflows/dagger.yaml @@ -25,9 +25,9 @@ jobs: module: . # Arguments to pass to CLI args: ci-integration --dir=./ -# - name: Test Reporter -# uses: dorny/test-reporter@v1.9.1 -# with: -# name: JEST Tests # Name of the check run which will be created -# path: reports/jest-*.xml # Path to test results -# reporter: jest-junit # Format of test results + - name: Test Reporter + uses: dorny/test-reporter@v1.9.1 + with: + name: Minitests + path: test/reports/*.xml + reporter: java-junit diff --git a/Gemfile b/Gemfile index 8a5a64b5..0585de59 100644 --- a/Gemfile +++ b/Gemfile @@ -119,4 +119,7 @@ group :test do gem 'selenium-webdriver' gem 'webdrivers' gem 'webmock' + gem 'simplecov' + gem 'simplecov-rcov' + gem 'minitest' end diff --git a/ci/main.go b/ci/main.go index cb6e3da4..dad0de2f 100644 --- a/ci/main.go +++ b/ci/main.go @@ -32,6 +32,8 @@ type Results struct { VulnerabilityScan *dagger.File // the built image Image *dagger.Container + // the test reports + TestReports *dagger.Directory } // Returns a Container built from the Dockerfile in the provided Directory @@ -87,38 +89,6 @@ func (m *Ci) Memcached( AsService() } -// Executes the test suite for the Rails application in the provided Directory -func (m *Ci) Test(ctx context.Context, dir *dagger.Directory) *dagger.Container { - - return dag.Container(). - From("registry.puzzle.ch/docker.io/ruby:3.2"). - WithExec([]string{"useradd", "-m", "-u", "1001", "ruby"}). - WithServiceBinding("postgresql", m.Postgres(ctx, "11")). - WithServiceBinding("memcached", m.Memcached(ctx, "latest")). - WithMountedDirectory("/mnt", dir, dagger.ContainerWithMountedDirectoryOpts{Owner: "ruby"}). - WithWorkdir("/mnt"). - WithEnvVariable("RAILS_DB_HOST", "postgresql"). // This is the service name of the postgres container called by rails - WithEnvVariable("RAILS_TEST_DB_HOST", "postgresql"). - WithEnvVariable("RAILS_TEST_DB_NAME", "postgres"). - WithEnvVariable("RAILS_TEST_DB_USERNAME", "postgres"). - WithEnvVariable("RAILS_TEST_DB_PASSWORD", "postgres"). - WithEnvVariable("RAILS_ENV", "test"). - WithEnvVariable("CI", "true"). - WithEnvVariable("PGDATESTYLE", "German"). - WithEnvVariable("TZ", "Europe/Zurich"). - WithEnvVariable("DEBIAN_FRONTEND", "noninteractive"). - WithExec([]string{"apt-get", "update"}). - WithExec([]string{"apt-get", "-yqq", "install", "libpq-dev", "libvips-dev", "chromium", "graphviz", "imagemagick"}). - WithUser("ruby"). - WithExec([]string{"gem", "update", "--system"}). // update bundler version - WithExec([]string{"gem", "install", "bundler", "--version", `~>2`}). - WithExec([]string{"bundle", "install", "--jobs", "4", "--retry", "3"}). - WithExec([]string{"bundle", "exec", "rails", "db:create"}). - WithExec([]string{"bundle", "exec", "rails", "db:migrate"}). - WithExec([]string{"bundle", "exec", "rails", "assets:precompile"}). - WithExec([]string{"bundle", "exec", "rails", "test", "test/controllers", "test/domain", "test/fabricators", "test/fixtures", "test/helpers", "test/mailers", "test/models", "test/presenters", "test/support", "test/tarantula"}) -} - // Creates an SBOM for the container func (m *Ci) Sbom(container *dagger.Container) *dagger.File { trivy_container := dag.Container(). @@ -137,6 +107,18 @@ func (m *Ci) Sbom(container *dagger.Container) *dagger.File { return sbom } +// Executes the test suite for the Rails application in the provided Directory +func (m *Ci) Test(ctx context.Context, dir *dagger.Directory) *dagger.Directory { + return m.BaseTestContainer(ctx, dir). + WithServiceBinding("postgresql", m.Postgres(ctx, "11")). + WithServiceBinding("memcached", m.Memcached(ctx, "latest")). + WithExec([]string{"bundle", "exec", "rails", "db:create"}). + WithExec([]string{"bundle", "exec", "rails", "db:migrate"}). + WithExec([]string{"bundle", "exec", "rails", "assets:precompile"}). + WithExec([]string{"sh", "-c", "bundle exec rails test -v test/controllers test/domain test/fabricators test/fixtures test/helpers test/mailers test/models test/presenters test/support test/tarantula"}). + Directory("/mnt/test/reports") +} + // Builds the container and creates an SBOM for it func (m *Ci) SbomBuild(ctx context.Context, dir *dagger.Directory) *dagger.File { container := m.Build(ctx, dir) @@ -158,6 +140,32 @@ func (m *Ci) Vulnscan(sbom *dagger.File) *dagger.File { return trivy.Sbom(sbom).Report("json") } +// Returns a Container with the base setup for running the rails test suite +func (m *Ci) BaseTestContainer(_ context.Context, dir *dagger.Directory) *dagger.Container { + return dag.Container(). + From("registry.puzzle.ch/docker.io/ruby:3.2"). + WithExec([]string{"useradd", "-m", "-u", "1001", "ruby"}). + WithMountedDirectory("/mnt", dir, dagger.ContainerWithMountedDirectoryOpts{Owner: "ruby"}). + WithWorkdir("/mnt"). + WithEnvVariable("RAILS_DB_HOST", "postgresql"). // This is the service name of the postgres container called by rails + WithEnvVariable("RAILS_TEST_DB_HOST", "postgresql"). + WithEnvVariable("RAILS_TEST_DB_NAME", "postgres"). + WithEnvVariable("RAILS_TEST_DB_USERNAME", "postgres"). + WithEnvVariable("RAILS_TEST_DB_PASSWORD", "postgres"). + WithEnvVariable("RAILS_ENV", "test"). + WithEnvVariable("CI", "true"). + WithEnvVariable("PGDATESTYLE", "German"). + WithEnvVariable("TZ", "Europe/Zurich"). + WithEnvVariable("DEBIAN_FRONTEND", "noninteractive"). + WithEnvVariable("TEST_REPORTS", "true"). + WithExec([]string{"apt-get", "update"}). + WithExec([]string{"apt-get", "-yqq", "install", "libpq-dev", "libvips-dev", "chromium", "graphviz", "imagemagick"}). + WithUser("ruby"). + //WithExec([]string{"gem", "update", "--system"}). + WithExec([]string{"gem", "install", "bundler", "--version", `~>2`}). + WithExec([]string{"bundle", "install", "--jobs", "4", "--retry", "3"}) +} + // Executes all the steps and returns a Results object func (m *Ci) Ci(ctx context.Context, dir *dagger.Directory) *Results { lintOutput := m.Lint(dir) @@ -165,8 +173,10 @@ func (m *Ci) Ci(ctx context.Context, dir *dagger.Directory) *Results { image := m.Build(ctx, dir) sbom := m.Sbom(image) vulnerabilityScan := m.Vulnscan(sbom) + testReports := m.Test(ctx, dir) return &Results{ + TestReports: testReports, LintOutput: lintOutput, SecurityScan: securityScan, VulnerabilityScan: vulnerabilityScan, @@ -199,15 +209,16 @@ func (m *Ci) CiIntegration(ctx context.Context, dir *dagger.Directory) *Results return m.Build(ctx, dir) }() - go func() { + var testReports = func() *dagger.Directory { defer wg.Done() - m.Test(ctx, dir) + return m.Test(ctx, dir) }() // This Blocks the execution until its counter become 0 wg.Wait() return &Results{ + TestReports: testReports, LintOutput: lintOutput, SecurityScan: securityScan, VulnerabilityScan: vulnerabilityScan, diff --git a/test/test_helper.rb b/test/test_helper.rb index 77e1e5c4..27211862 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -7,19 +7,19 @@ ENV['RAILS_ENV'] = 'test' -# if ENV['TEST_REPORTS'] -# require 'simplecov' -# require 'simplecov-rcov' -# SimpleCov.coverage_dir 'test/coverage' -# # use this formatter for jenkins compatibility -# SimpleCov.formatter = SimpleCov::Formatter::RcovFormatter -# SimpleCov.command_name 'Unit Tests' -# SimpleCov.start 'rails' - -# require 'minitest/reporters' -# MiniTest::Reporters.use! [MiniTest::Reporters::DefaultReporter.new, -# MiniTest::Reporters::JUnitReporter.new] -# end +if ENV['TEST_REPORTS'] + require 'simplecov' + require 'simplecov-rcov' + SimpleCov.coverage_dir 'test/coverage' + # use this formatter for jenkins compatibility + SimpleCov.formatter = SimpleCov::Formatter::RcovFormatter + SimpleCov.command_name 'Unit Tests' + SimpleCov.start 'rails' + + require 'minitest/reporters' + Minitest::Reporters.use! [Minitest::Reporters::DefaultReporter.new, + Minitest::Reporters::JUnitReporter.new] +end require File.expand_path('../config/environment', __dir__) Rails.env = 'test' From 587ded1b843cbf115ca7c98bf18a184666f82ead Mon Sep 17 00:00:00 2001 From: Reto Galante Date: Wed, 13 Nov 2024 17:11:14 +0100 Subject: [PATCH 35/47] add ignore linter failures --- ci/main.go | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/ci/main.go b/ci/main.go index dad0de2f..c41f4226 100644 --- a/ci/main.go +++ b/ci/main.go @@ -42,13 +42,24 @@ func (m *Ci) Build(_ context.Context, dir *dagger.Directory) *dagger.Container { } // Returns the result of haml-lint run against the sources in the provided Directory -func (m *Ci) Lint(dir *dagger.Directory) *dagger.File { - return dag.Container(). +func (m *Ci) Lint( + dir *dagger.Directory, + // ignore linter failures + // +optional + // +default=false + pass bool) *dagger.File { + container := dag.Container(). From("ruby:latest"). WithMountedDirectory("/mnt", dir). WithWorkdir("/mnt"). - WithExec([]string{"gem", "install", "haml-lint"}). - WithExec([]string{"sh", "-c", "haml-lint -r json . > lint.json || true"}). + WithExec([]string{"gem", "install", "haml-lint"}) + if pass { + return container. + WithExec([]string{"sh", "-c", "haml-lint -r json . > lint.json || true"}). + File("lint.json") + } + return container. + WithExec([]string{"sh", "-c", "haml-lint -r json . > lint.json"}). File("lint.json") } @@ -167,8 +178,14 @@ func (m *Ci) BaseTestContainer(_ context.Context, dir *dagger.Directory) *dagger } // Executes all the steps and returns a Results object -func (m *Ci) Ci(ctx context.Context, dir *dagger.Directory) *Results { - lintOutput := m.Lint(dir) +func (m *Ci) Ci( + ctx context.Context, + dir *dagger.Directory, + // ignore linter failures + // +optional + // +default=false + pass bool) *Results { + lintOutput := m.Lint(dir, pass) securityScan := m.Sast(dir) image := m.Build(ctx, dir) sbom := m.Sbom(image) @@ -185,13 +202,19 @@ func (m *Ci) Ci(ctx context.Context, dir *dagger.Directory) *Results { } // Executes all the steps and returns a Results object -func (m *Ci) CiIntegration(ctx context.Context, dir *dagger.Directory) *Results { +func (m *Ci) CiIntegration( + ctx context.Context, + dir *dagger.Directory, + // ignore linter failures + // +optional + // +default=false + pass bool) *Results { var wg sync.WaitGroup wg.Add(5) var lintOutput = func() *dagger.File { defer wg.Done() - return m.Lint(dir) + return m.Lint(dir, pass) }() var securityScan = func() *dagger.File { From 82995965a023ee3011039d405025abefff320f0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20B=C3=BChlmann?= Date: Wed, 13 Nov 2024 17:23:56 +0100 Subject: [PATCH 36/47] Add test reports to single job --- .github/workflows/dagger-test.yaml | 10 +++++++++- .github/workflows/dagger.yaml | 6 ------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dagger-test.yaml b/.github/workflows/dagger-test.yaml index fb829a2d..49bf3894 100644 --- a/.github/workflows/dagger-test.yaml +++ b/.github/workflows/dagger-test.yaml @@ -8,7 +8,7 @@ jobs: runs-on: 'ubuntu-latest' steps: - uses: actions/checkout@v4 - - name: 'test' + - name: Test uses: dagger/dagger-for-github@v7.0.1 with: verb: call @@ -16,3 +16,11 @@ jobs: args: test --dir=./ version: '0.14.0' cloud-token: ${{ secrets.DAGGER_CLOUD_TOKEN }} + - name: Report + uses: dorny/test-reporter@v1.9.1 + with: + name: Minitests + path: test/reports/*.xml + reporter: java-junit + fail-on-empty: 'false' + fail-on-error: 'true' diff --git a/.github/workflows/dagger.yaml b/.github/workflows/dagger.yaml index 8a5efc36..235bac30 100644 --- a/.github/workflows/dagger.yaml +++ b/.github/workflows/dagger.yaml @@ -25,9 +25,3 @@ jobs: module: . # Arguments to pass to CLI args: ci-integration --dir=./ - - name: Test Reporter - uses: dorny/test-reporter@v1.9.1 - with: - name: Minitests - path: test/reports/*.xml - reporter: java-junit From a0797d32e5114f46cb2efcfecf298092a834a4c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20B=C3=BChlmann?= Date: Wed, 13 Nov 2024 17:34:08 +0100 Subject: [PATCH 37/47] Export test results --- .github/workflows/dagger-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dagger-test.yaml b/.github/workflows/dagger-test.yaml index 49bf3894..78ef0a48 100644 --- a/.github/workflows/dagger-test.yaml +++ b/.github/workflows/dagger-test.yaml @@ -13,7 +13,7 @@ jobs: with: verb: call module: ci - args: test --dir=./ + args: test --dir . export --path test-reports version: '0.14.0' cloud-token: ${{ secrets.DAGGER_CLOUD_TOKEN }} - name: Report From d13e242bcb6b4af717f78561482afca77a3f7193 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20B=C3=BChlmann?= Date: Wed, 13 Nov 2024 17:35:34 +0100 Subject: [PATCH 38/47] Use test-reports/*.xml as path for test reports --- .github/workflows/dagger-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dagger-test.yaml b/.github/workflows/dagger-test.yaml index 78ef0a48..3b4a291d 100644 --- a/.github/workflows/dagger-test.yaml +++ b/.github/workflows/dagger-test.yaml @@ -20,7 +20,7 @@ jobs: uses: dorny/test-reporter@v1.9.1 with: name: Minitests - path: test/reports/*.xml + path: test-reports/*.xml reporter: java-junit fail-on-empty: 'false' fail-on-error: 'true' From f7da10ce43636d727c2fe68201baa07771bbb1ca Mon Sep 17 00:00:00 2001 From: Christoph Raaflaub Date: Wed, 13 Nov 2024 11:55:29 +0100 Subject: [PATCH 39/47] export files --- .gitignore | 1 + ci/main.go | 22 +++++++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 51e3c8d0..139d21bd 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ /log/*.log /tmp !/tmp/pids +out/ test/coverage test/reports diff --git a/ci/main.go b/ci/main.go index c41f4226..e556b508 100644 --- a/ci/main.go +++ b/ci/main.go @@ -208,7 +208,8 @@ func (m *Ci) CiIntegration( // ignore linter failures // +optional // +default=false - pass bool) *Results { + pass bool, +) string { var wg sync.WaitGroup wg.Add(5) @@ -227,10 +228,12 @@ func (m *Ci) CiIntegration( return m.Vulnscan(m.Sbom(m.Build(ctx, dir))) }() + /* var image = func() *dagger.Container { defer wg.Done() return m.Build(ctx, dir) }() + */ var testReports = func() *dagger.Directory { defer wg.Done() @@ -240,6 +243,7 @@ func (m *Ci) CiIntegration( // This Blocks the execution until its counter become 0 wg.Wait() + /* return &Results{ TestReports: testReports, LintOutput: lintOutput, @@ -247,4 +251,20 @@ func (m *Ci) CiIntegration( VulnerabilityScan: vulnerabilityScan, Image: image, } + */ + + // TODO: fail on errors of the functions! + + + result_container := dag.Container(). + From("alpine:latest"). + WithWorkdir("/tmp/out"). + //WithExec([]string{"cat", securityScan, "rails", "assets:precompile"}). + //WithExec([]string{"sh", "-c", `cat lintOutput > /tmp/out/lint/lint.json`}). + WithExec([]string{"/bin/sh", "-c", `echo foo > /tmp/out/foo`}). + WithExec([]string{"/bin/sh", "-c", fmt.Sprintf("echo %s > /tmp/out/foo.log", lintOutput)}). + WithFile("/tmp/out/scan/", securityScan) + //return result_container.Directory(".") + result_container.Directory(".") + return lintOutput } From 9dbef1614ffae4305439285814a040b0618fa39d Mon Sep 17 00:00:00 2001 From: Christoph Raaflaub Date: Wed, 13 Nov 2024 17:40:23 +0100 Subject: [PATCH 40/47] file export --- ci/main.go | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/ci/main.go b/ci/main.go index e556b508..0ca7e779 100644 --- a/ci/main.go +++ b/ci/main.go @@ -201,7 +201,7 @@ func (m *Ci) Ci( } } -// Executes all the steps and returns a Results object +// Executes all the steps and returns a directory with the results func (m *Ci) CiIntegration( ctx context.Context, dir *dagger.Directory, @@ -209,7 +209,7 @@ func (m *Ci) CiIntegration( // +optional // +default=false pass bool, -) string { +) *dagger.Directory { var wg sync.WaitGroup wg.Add(5) @@ -228,12 +228,10 @@ func (m *Ci) CiIntegration( return m.Vulnscan(m.Sbom(m.Build(ctx, dir))) }() - /* var image = func() *dagger.Container { defer wg.Done() return m.Build(ctx, dir) }() - */ var testReports = func() *dagger.Directory { defer wg.Done() @@ -257,14 +255,10 @@ func (m *Ci) CiIntegration( result_container := dag.Container(). - From("alpine:latest"). WithWorkdir("/tmp/out"). - //WithExec([]string{"cat", securityScan, "rails", "assets:precompile"}). - //WithExec([]string{"sh", "-c", `cat lintOutput > /tmp/out/lint/lint.json`}). - WithExec([]string{"/bin/sh", "-c", `echo foo > /tmp/out/foo`}). - WithExec([]string{"/bin/sh", "-c", fmt.Sprintf("echo %s > /tmp/out/foo.log", lintOutput)}). - WithFile("/tmp/out/scan/", securityScan) - //return result_container.Directory(".") - result_container.Directory(".") - return lintOutput + WithFile("/tmp/out/lint/", lintOutput). + WithFile("/tmp/out/scan/", securityScan). + WithFile("/tmp/out/vuln/", vulnerabilityScan) + + return result_container.Directory(".") } From 5aed25cadecdac93a9590e6024917a85704f7a46 Mon Sep 17 00:00:00 2001 From: Christoph Raaflaub Date: Wed, 13 Nov 2024 18:42:18 +0100 Subject: [PATCH 41/47] add export --- ci/main.go | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/ci/main.go b/ci/main.go index 0ca7e779..2731f9d0 100644 --- a/ci/main.go +++ b/ci/main.go @@ -47,7 +47,8 @@ func (m *Ci) Lint( // ignore linter failures // +optional // +default=false - pass bool) *dagger.File { + pass bool, +) *dagger.File { container := dag.Container(). From("ruby:latest"). WithMountedDirectory("/mnt", dir). @@ -184,7 +185,8 @@ func (m *Ci) Ci( // ignore linter failures // +optional // +default=false - pass bool) *Results { + pass bool, +) *Results { lintOutput := m.Lint(dir, pass) securityScan := m.Sast(dir) image := m.Build(ctx, dir) @@ -211,7 +213,7 @@ func (m *Ci) CiIntegration( pass bool, ) *dagger.Directory { var wg sync.WaitGroup - wg.Add(5) + wg.Add(4) var lintOutput = func() *dagger.File { defer wg.Done() @@ -228,10 +230,12 @@ func (m *Ci) CiIntegration( return m.Vulnscan(m.Sbom(m.Build(ctx, dir))) }() + /* var image = func() *dagger.Container { defer wg.Done() return m.Build(ctx, dir) }() + */ var testReports = func() *dagger.Directory { defer wg.Done() @@ -241,24 +245,18 @@ func (m *Ci) CiIntegration( // This Blocks the execution until its counter become 0 wg.Wait() - /* - return &Results{ - TestReports: testReports, - LintOutput: lintOutput, - SecurityScan: securityScan, - VulnerabilityScan: vulnerabilityScan, - Image: image, - } - */ - // TODO: fail on errors of the functions! - + lintOutputName, _ := lintOutput.Name(ctx) + securityScanName, _ := securityScan.Name(ctx) + //vulnerabilityScanName, _ := vulnerabilityScan.Name(ctx) result_container := dag.Container(). WithWorkdir("/tmp/out"). - WithFile("/tmp/out/lint/", lintOutput). - WithFile("/tmp/out/scan/", securityScan). - WithFile("/tmp/out/vuln/", vulnerabilityScan) + WithFile(fmt.Sprintf("/tmp/out/lint/%s", lintOutputName), lintOutput). + WithFile(fmt.Sprintf("/tmp/out/scan/%s", securityScanName), securityScan). + //WithFile("/tmp/out/vuln/", vulnerabilityScan) + WithDirectory("/tmp/out/tests/", testReports) + vulnerabilityScan.Name(ctx) return result_container.Directory(".") } From 465cbb11ecd3c59347b4c54345c4fc83546fc1de Mon Sep 17 00:00:00 2001 From: Christoph Raaflaub Date: Wed, 13 Nov 2024 18:44:40 +0100 Subject: [PATCH 42/47] pipeline export files --- .github/workflows/dagger.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dagger.yaml b/.github/workflows/dagger.yaml index 235bac30..ec6942f5 100644 --- a/.github/workflows/dagger.yaml +++ b/.github/workflows/dagger.yaml @@ -24,4 +24,4 @@ jobs: # Dagger module to call. Local or Git module: . # Arguments to pass to CLI - args: ci-integration --dir=./ + args: ci-integration --pass=true --dir=./ export --path ./out/ From a26a8f79854255fcfd2de3c6912fc55d884f5399 Mon Sep 17 00:00:00 2001 From: Christoph Raaflaub Date: Wed, 13 Nov 2024 18:49:39 +0100 Subject: [PATCH 43/47] do not fail on lint --- .github/workflows/dagger-lint.yaml | 2 +- .github/workflows/dagger-test.yaml | 2 +- .github/workflows/dagger.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dagger-lint.yaml b/.github/workflows/dagger-lint.yaml index 901bc7ed..02f744ef 100644 --- a/.github/workflows/dagger-lint.yaml +++ b/.github/workflows/dagger-lint.yaml @@ -13,6 +13,6 @@ jobs: with: verb: call module: ci - args: lint --dir=./ + args: lint --pass=true --dir=./ version: '0.14.0' cloud-token: ${{ secrets.DAGGER_CLOUD_TOKEN }} diff --git a/.github/workflows/dagger-test.yaml b/.github/workflows/dagger-test.yaml index 3b4a291d..c8e8bf8b 100644 --- a/.github/workflows/dagger-test.yaml +++ b/.github/workflows/dagger-test.yaml @@ -13,7 +13,7 @@ jobs: with: verb: call module: ci - args: test --dir . export --path test-reports + args: test --dir=./ export --path=./test-reports version: '0.14.0' cloud-token: ${{ secrets.DAGGER_CLOUD_TOKEN }} - name: Report diff --git a/.github/workflows/dagger.yaml b/.github/workflows/dagger.yaml index ec6942f5..3c4ab6d6 100644 --- a/.github/workflows/dagger.yaml +++ b/.github/workflows/dagger.yaml @@ -24,4 +24,4 @@ jobs: # Dagger module to call. Local or Git module: . # Arguments to pass to CLI - args: ci-integration --pass=true --dir=./ export --path ./out/ + args: ci-integration --pass=true --dir=./ export --path=./out/ From 36af8958f737c94f230cb8a4b5a361ea3afa1a6d Mon Sep 17 00:00:00 2001 From: Christoph Raaflaub Date: Thu, 14 Nov 2024 06:31:17 +0100 Subject: [PATCH 44/47] vuln check not parallel --- ci/main.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/ci/main.go b/ci/main.go index 2731f9d0..1c321815 100644 --- a/ci/main.go +++ b/ci/main.go @@ -225,12 +225,11 @@ func (m *Ci) CiIntegration( return m.Sast(dir) }() + /* var vulnerabilityScan = func() *dagger.File { defer wg.Done() return m.Vulnscan(m.Sbom(m.Build(ctx, dir))) }() - - /* var image = func() *dagger.Container { defer wg.Done() return m.Build(ctx, dir) @@ -254,9 +253,8 @@ func (m *Ci) CiIntegration( WithWorkdir("/tmp/out"). WithFile(fmt.Sprintf("/tmp/out/lint/%s", lintOutputName), lintOutput). WithFile(fmt.Sprintf("/tmp/out/scan/%s", securityScanName), securityScan). - //WithFile("/tmp/out/vuln/", vulnerabilityScan) WithDirectory("/tmp/out/tests/", testReports) - - vulnerabilityScan.Name(ctx) - return result_container.Directory(".") + //WithFile(fmt.Sprintf("/tmp/out/vuln/%s", vulnerabilityScanName), vulnerabilityScan) + return result_container. + Directory(".") } From 8b3784bf73eed5a7bf471dd7f36942fee249ed8c Mon Sep 17 00:00:00 2001 From: Christoph Raaflaub Date: Thu, 14 Nov 2024 06:40:02 +0100 Subject: [PATCH 45/47] all report -> unit test --- .github/workflows/dagger.yaml | 10 +++++++++- ci/main.go | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dagger.yaml b/.github/workflows/dagger.yaml index 3c4ab6d6..94ee0dfa 100644 --- a/.github/workflows/dagger.yaml +++ b/.github/workflows/dagger.yaml @@ -24,4 +24,12 @@ jobs: # Dagger module to call. Local or Git module: . # Arguments to pass to CLI - args: ci-integration --pass=true --dir=./ export --path=./out/ + args: ci-integration --pass=true --dir=./ export --path=./reports/ + - name: Unittest Report + uses: dorny/test-reporter@v1.9.1 + with: + name: Minitests + path: reports/unit-tests/*.xml + reporter: java-junit + fail-on-empty: 'false' + fail-on-error: 'true' diff --git a/ci/main.go b/ci/main.go index 1c321815..144f9c30 100644 --- a/ci/main.go +++ b/ci/main.go @@ -213,7 +213,7 @@ func (m *Ci) CiIntegration( pass bool, ) *dagger.Directory { var wg sync.WaitGroup - wg.Add(4) + wg.Add(3) var lintOutput = func() *dagger.File { defer wg.Done() @@ -253,7 +253,7 @@ func (m *Ci) CiIntegration( WithWorkdir("/tmp/out"). WithFile(fmt.Sprintf("/tmp/out/lint/%s", lintOutputName), lintOutput). WithFile(fmt.Sprintf("/tmp/out/scan/%s", securityScanName), securityScan). - WithDirectory("/tmp/out/tests/", testReports) + WithDirectory("/tmp/out/unit-tests/", testReports) //WithFile(fmt.Sprintf("/tmp/out/vuln/%s", vulnerabilityScanName), vulnerabilityScan) return result_container. Directory(".") From 215af68e12130deba8ea2fea1d0e6342a9dd08a9 Mon Sep 17 00:00:00 2001 From: Christoph Raaflaub Date: Thu, 14 Nov 2024 07:03:56 +0100 Subject: [PATCH 46/47] archive artifacts --- .github/workflows/dagger.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dagger.yaml b/.github/workflows/dagger.yaml index 94ee0dfa..7124f40c 100644 --- a/.github/workflows/dagger.yaml +++ b/.github/workflows/dagger.yaml @@ -28,8 +28,13 @@ jobs: - name: Unittest Report uses: dorny/test-reporter@v1.9.1 with: - name: Minitests + name: unit-tests path: reports/unit-tests/*.xml reporter: java-junit fail-on-empty: 'false' fail-on-error: 'true' + - name: archive pipeline artifacts + uses: actions/upload-artifact@v4 + with: + name: pipeline-artifacts + path: reports From 09d8effd8a8f7b5c3e520ea802c2e66e8688e6b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20B=C3=BChlmann?= Date: Thu, 14 Nov 2024 09:01:42 +0100 Subject: [PATCH 47/47] Add build step --- Gemfile.lock | 12 ++++++++++++ ci/main.go | 39 ++++++++++++++++++++++++--------------- 2 files changed, 36 insertions(+), 15 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 18a7940b..ac244302 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -221,6 +221,7 @@ GEM railties (>= 4.1.0) responders warden (~> 1.2.3) + docile (1.4.1) drb (2.2.0) ruby2_keywords dry-configurable (1.1.0) @@ -617,6 +618,14 @@ GEM websocket (~> 1.0) sentry-raven (3.1.2) faraday (>= 1.0) + simplecov (0.22.0) + docile (~> 1.1) + simplecov-html (~> 0.11) + simplecov_json_formatter (~> 0.1) + simplecov-html (0.13.1) + simplecov-rcov (0.3.7) + simplecov (>= 0.4.1) + simplecov_json_formatter (0.1.4) simpleidn (0.2.1) unf (~> 0.1.4) snaky_hash (2.0.1) @@ -736,6 +745,7 @@ DEPENDENCIES m matrix mini_racer + minitest minitest-reporters mocha nested_form_fields @@ -778,6 +788,8 @@ DEPENDENCIES selectize-rails selenium-webdriver sentry-raven + simplecov + simplecov-rcov spring swagger-blocks terser diff --git a/ci/main.go b/ci/main.go index 144f9c30..53342b24 100644 --- a/ci/main.go +++ b/ci/main.go @@ -38,7 +38,11 @@ type Results struct { // Returns a Container built from the Dockerfile in the provided Directory func (m *Ci) Build(_ context.Context, dir *dagger.Directory) *dagger.Container { - return dag.Container().Build(dir) + return dag.Container(). + WithDirectory("/src", dir). + WithWorkdir("/src"). + Directory("/src"). + DockerBuild() } // Returns the result of haml-lint run against the sources in the provided Directory @@ -178,6 +182,11 @@ func (m *Ci) BaseTestContainer(_ context.Context, dir *dagger.Directory) *dagger WithExec([]string{"bundle", "install", "--jobs", "4", "--retry", "3"}) } +// Publish the Container built from the Dockerfile in the provided registry +func (m *Ci) Publish(ctx context.Context, dir *dagger.Directory, destImage string) (string, error) { + return m.Build(ctx, dir).Publish(ctx, destImage) +} + // Executes all the steps and returns a Results object func (m *Ci) Ci( ctx context.Context, @@ -226,14 +235,14 @@ func (m *Ci) CiIntegration( }() /* - var vulnerabilityScan = func() *dagger.File { - defer wg.Done() - return m.Vulnscan(m.Sbom(m.Build(ctx, dir))) - }() - var image = func() *dagger.Container { - defer wg.Done() - return m.Build(ctx, dir) - }() + var vulnerabilityScan = func() *dagger.File { + defer wg.Done() + return m.Vulnscan(m.Sbom(m.Build(ctx, dir))) + }() + var image = func() *dagger.Container { + defer wg.Done() + return m.Build(ctx, dir) + }() */ var testReports = func() *dagger.Directory { @@ -250,11 +259,11 @@ func (m *Ci) CiIntegration( securityScanName, _ := securityScan.Name(ctx) //vulnerabilityScanName, _ := vulnerabilityScan.Name(ctx) result_container := dag.Container(). - WithWorkdir("/tmp/out"). - WithFile(fmt.Sprintf("/tmp/out/lint/%s", lintOutputName), lintOutput). - WithFile(fmt.Sprintf("/tmp/out/scan/%s", securityScanName), securityScan). - WithDirectory("/tmp/out/unit-tests/", testReports) - //WithFile(fmt.Sprintf("/tmp/out/vuln/%s", vulnerabilityScanName), vulnerabilityScan) + WithWorkdir("/tmp/out"). + WithFile(fmt.Sprintf("/tmp/out/lint/%s", lintOutputName), lintOutput). + WithFile(fmt.Sprintf("/tmp/out/scan/%s", securityScanName), securityScan). + WithDirectory("/tmp/out/unit-tests/", testReports) + //WithFile(fmt.Sprintf("/tmp/out/vuln/%s", vulnerabilityScanName), vulnerabilityScan) return result_container. - Directory(".") + Directory(".") }