diff --git a/.chainsaw.yaml b/.chainsaw.yaml
index 5123b8747..b003f979e 100755
--- a/.chainsaw.yaml
+++ b/.chainsaw.yaml
@@ -5,8 +5,6 @@ metadata:
   name: configuration
 spec:
   parallel: 1
-  testDirs:
-  - ./
   timeouts:
     apply: 1m30s
     assert: 1m30s
@@ -14,3 +12,4 @@ spec:
     delete: 1m30s
     error: 1m30s
     exec: 1m30s
+  forceTerminationGracePeriod: 5s
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 8a2a7ef7c..b0b5b64d7 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -27,13 +27,13 @@ jobs:
           - name: v1.28
             version: v1.28.0
         tests:
-          - argo
-          - aws
+          # - argo
+          # - aws
           - best-practices
-          - castai
+          # - castai
           # - cert-manager
-          - consul
-          - external-secret-operator
+          # - consul
+          # - external-secret-operator
           - flux
           - istio
           - karpenter
@@ -105,10 +105,10 @@ jobs:
           - argo
           - aws
           # - best-practices
-          # - castai
+          - castai
           - cert-manager
-          # - consul
-          # - external-secret-operator
+          - consul
+          - external-secret-operator
           # - flux
           # - istio
           # - karpenter
diff --git a/castai/add-castai-removal-disabled/.chainsaw-test/chainsaw-test.yaml b/castai/add-castai-removal-disabled/.chainsaw-test/chainsaw-test.yaml
new file mode 100755
index 000000000..e8e7cb56e
--- /dev/null
+++ b/castai/add-castai-removal-disabled/.chainsaw-test/chainsaw-test.yaml
@@ -0,0 +1,23 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  creationTimestamp: null
+  name: add-castai-removal-disabled
+spec:
+  steps:
+  - name: step-01
+    try:
+    - apply:
+        file: ../add-castai-removal-disabled.yaml
+    - assert:
+        file: policy-ready.yaml
+  - name: step-02
+    try:
+    - apply:
+        file: ../.kyverno-test/resources.yaml
+  - name: step-03
+    try:
+    - assert:
+        file: ../.kyverno-test/patched01.yaml
+    - assert:
+        file: ../.kyverno-test/patched02.yaml
diff --git a/castai/add-castai-removal-disabled/policy-ready.yaml b/castai/add-castai-removal-disabled/.chainsaw-test/policy-ready.yaml
similarity index 100%
rename from castai/add-castai-removal-disabled/policy-ready.yaml
rename to castai/add-castai-removal-disabled/.chainsaw-test/policy-ready.yaml
diff --git a/castai/add-castai-removal-disabled/01-policy.yaml b/castai/add-castai-removal-disabled/01-policy.yaml
deleted file mode 100644
index 7ff766e1e..000000000
--- a/castai/add-castai-removal-disabled/01-policy.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-apiVersion: kuttl.dev/v1beta1
-kind: TestStep
-apply:
-- add-castai-removal-disabled.yaml
-assert:
-- policy-ready.yaml
\ No newline at end of file
diff --git a/castai/add-castai-removal-disabled/02-manifests.yaml b/castai/add-castai-removal-disabled/02-manifests.yaml
deleted file mode 100644
index 0f9a29fcf..000000000
--- a/castai/add-castai-removal-disabled/02-manifests.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-apiVersion: kuttl.dev/v1beta1
-kind: TestStep
-apply:
-- file: .kyverno-test/resources.yaml
-  shouldFail: false
\ No newline at end of file
diff --git a/castai/add-castai-removal-disabled/03-mutated.yaml b/castai/add-castai-removal-disabled/03-mutated.yaml
deleted file mode 100644
index 33cb4563f..000000000
--- a/castai/add-castai-removal-disabled/03-mutated.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-apiVersion: kuttl.dev/v1beta1
-kind: TestStep
-assert:
-- .kyverno-test/patched01.yaml
-- .kyverno-test/patched02.yaml
diff --git a/consul/enforce-min-tls-version/00-assert.yaml b/consul/enforce-min-tls-version/.chainsaw-test/chainsaw-step-00-assert-1.yaml
old mode 100644
new mode 100755
similarity index 100%
rename from consul/enforce-min-tls-version/00-assert.yaml
rename to consul/enforce-min-tls-version/.chainsaw-test/chainsaw-step-00-assert-1.yaml
diff --git a/consul/enforce-min-tls-version/.chainsaw-test/chainsaw-test.yaml b/consul/enforce-min-tls-version/.chainsaw-test/chainsaw-test.yaml
new file mode 100755
index 000000000..6355a4f9a
--- /dev/null
+++ b/consul/enforce-min-tls-version/.chainsaw-test/chainsaw-test.yaml
@@ -0,0 +1,26 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  creationTimestamp: null
+  name: enforce-min-tls-version
+spec:
+  steps:
+  - name: step-00
+    try:
+    - assert:
+        file: chainsaw-step-00-assert-1.yaml
+  - name: step-01
+    try:
+    - apply:
+        file: ../enforce-min-tls-version.yaml
+    - assert:
+        file: policy-ready.yaml
+  - name: step-02
+    try:
+    - apply:
+        file: mesh-good.yaml
+    - apply:
+        expect:
+        - check:
+            ($error != null): true
+        file: mesh-bad.yaml
diff --git a/consul/enforce-min-tls-version/mesh-bad.yaml b/consul/enforce-min-tls-version/.chainsaw-test/mesh-bad.yaml
similarity index 100%
rename from consul/enforce-min-tls-version/mesh-bad.yaml
rename to consul/enforce-min-tls-version/.chainsaw-test/mesh-bad.yaml
diff --git a/consul/enforce-min-tls-version/mesh-good.yaml b/consul/enforce-min-tls-version/.chainsaw-test/mesh-good.yaml
similarity index 100%
rename from consul/enforce-min-tls-version/mesh-good.yaml
rename to consul/enforce-min-tls-version/.chainsaw-test/mesh-good.yaml
diff --git a/consul/enforce-min-tls-version/policy-ready.yaml b/consul/enforce-min-tls-version/.chainsaw-test/policy-ready.yaml
similarity index 100%
rename from consul/enforce-min-tls-version/policy-ready.yaml
rename to consul/enforce-min-tls-version/.chainsaw-test/policy-ready.yaml
diff --git a/consul/enforce-min-tls-version/01-policy.yaml b/consul/enforce-min-tls-version/01-policy.yaml
deleted file mode 100644
index 7008388f6..000000000
--- a/consul/enforce-min-tls-version/01-policy.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-apiVersion: kuttl.dev/v1beta1
-kind: TestStep
-apply:
-- enforce-min-tls-version.yaml
-assert:
-- policy-ready.yaml
\ No newline at end of file
diff --git a/consul/enforce-min-tls-version/02-manifests.yaml b/consul/enforce-min-tls-version/02-manifests.yaml
deleted file mode 100644
index c9ce6a588..000000000
--- a/consul/enforce-min-tls-version/02-manifests.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-apiVersion: kuttl.dev/v1beta1
-kind: TestStep
-apply:
-- file: mesh-good.yaml
-  shouldFail: false
-- file: mesh-bad.yaml
-  shouldFail: true
\ No newline at end of file
diff --git a/external-secret-operator/add-external-secret-prefix/01-assert.yaml b/external-secret-operator/add-external-secret-prefix/.chainsaw-test/chainsaw-step-01-assert-1.yaml
old mode 100644
new mode 100755
similarity index 100%
rename from external-secret-operator/add-external-secret-prefix/01-assert.yaml
rename to external-secret-operator/add-external-secret-prefix/.chainsaw-test/chainsaw-step-01-assert-1.yaml
diff --git a/external-secret-operator/add-external-secret-prefix/.chainsaw-test/chainsaw-test.yaml b/external-secret-operator/add-external-secret-prefix/.chainsaw-test/chainsaw-test.yaml
new file mode 100755
index 000000000..d5db10348
--- /dev/null
+++ b/external-secret-operator/add-external-secret-prefix/.chainsaw-test/chainsaw-test.yaml
@@ -0,0 +1,25 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  creationTimestamp: null
+  name: add-external-secret-prefix
+spec:
+  steps:
+  - name: step-01
+    try:
+    - assert:
+        file: chainsaw-step-01-assert-1.yaml
+  - name: step-02
+    try:
+    - apply:
+        file: ns.yaml
+    - apply:
+        file: ../add-external-secret-prefix.yaml
+    - assert:
+        file: policy-ready.yaml
+  - name: step-03
+    try:
+    - apply:
+        file: resource.yaml
+    - assert:
+        file: resource-mutated.yaml
diff --git a/external-secret-operator/add-external-secret-prefix/ns.yaml b/external-secret-operator/add-external-secret-prefix/.chainsaw-test/ns.yaml
similarity index 100%
rename from external-secret-operator/add-external-secret-prefix/ns.yaml
rename to external-secret-operator/add-external-secret-prefix/.chainsaw-test/ns.yaml
diff --git a/external-secret-operator/add-external-secret-prefix/policy-ready.yaml b/external-secret-operator/add-external-secret-prefix/.chainsaw-test/policy-ready.yaml
similarity index 100%
rename from external-secret-operator/add-external-secret-prefix/policy-ready.yaml
rename to external-secret-operator/add-external-secret-prefix/.chainsaw-test/policy-ready.yaml
diff --git a/external-secret-operator/add-external-secret-prefix/resource-mutated.yaml b/external-secret-operator/add-external-secret-prefix/.chainsaw-test/resource-mutated.yaml
similarity index 100%
rename from external-secret-operator/add-external-secret-prefix/resource-mutated.yaml
rename to external-secret-operator/add-external-secret-prefix/.chainsaw-test/resource-mutated.yaml
diff --git a/external-secret-operator/add-external-secret-prefix/kuttlresource.yaml b/external-secret-operator/add-external-secret-prefix/.chainsaw-test/resource.yaml
similarity index 100%
rename from external-secret-operator/add-external-secret-prefix/kuttlresource.yaml
rename to external-secret-operator/add-external-secret-prefix/.chainsaw-test/resource.yaml
diff --git a/external-secret-operator/add-external-secret-prefix/02-policy.yaml b/external-secret-operator/add-external-secret-prefix/02-policy.yaml
deleted file mode 100644
index 6d8781901..000000000
--- a/external-secret-operator/add-external-secret-prefix/02-policy.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-apiVersion: kuttl.dev/v1beta1
-kind: TestStep
-apply:
-- ns.yaml
-- add-external-secret-prefix.yaml
-assert:
-- policy-ready.yaml
diff --git a/external-secret-operator/add-external-secret-prefix/03-resource.yaml b/external-secret-operator/add-external-secret-prefix/03-resource.yaml
deleted file mode 100644
index 0cc05bd3c..000000000
--- a/external-secret-operator/add-external-secret-prefix/03-resource.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-apiVersion: kuttl.dev/v1beta1
-kind: TestStep
-apply:
-- kuttlresource.yaml
-assert:
-- resource-mutated.yaml