Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improved full line #6

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

Improved full line #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
748ff50
improve finding of full fragment by checking preceeding newlines
Kemosabert Sep 25, 2024
4244f53
take multi line secrets into account
Kemosabert Sep 25, 2024
991184b
update comments
Kemosabert Sep 25, 2024
3c2844b
add bunch of unit test
Kemosabert Sep 25, 2024
1cc1dfa
update algo
Kemosabert Sep 26, 2024
5f611db
ensure it gets correct occurence
Kemosabert Sep 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 80 additions & 7 deletions detect/detect.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -270,13 +270,8 @@ func (d *Detector) detectRule(fragment Fragment, rule config.Rule) []report.Find
if matchIndex[1] > loc.endLineIndex {
loc.endLineIndex = matchIndex[1]
}

full_fragment := ""
if( len(fragment.Raw) > 250 ){
full_fragment = strings.TrimSpace(fragment.Raw[0:250])
}else{
full_fragment = strings.TrimSpace(fragment.Raw[0:])
}

full_fragment := findFullLine(fragment, loc, secret)

finding := report.Finding{
Description: rule.Description,
Expand Down Expand Up @@ -413,3 +408,81 @@ func (d *Detector) addFinding(finding report.Finding) {
func (d *Detector) addCommit(commit string) {
d.commitMap[commit] = true
}

// this function knows how to find the full line based on the secret and the newline chars it is between
func findFullLine(fragment Fragment, loc Location, secret string) string {

secretLocation := findRelevantOccurenceOfSecret(fragment.Raw, secret, loc)
if secretLocation == nil {
return ""
}

secretStartingIdx := secretLocation[0] // start of secret in fragment
secretEndingIdx := secretLocation[1] // end of secret in fragment

// find the nearest previous newline
prevNewlineIndex := findIndexAfterPreviousNewline(fragment.Raw, secretStartingIdx-1)
// find the nearest next newline
nextNewlineIndex := findIndexBeforeNextNewline(fragment.Raw, secretEndingIdx+1)

// return substring between indices
return strings.TrimSpace(fragment.Raw[prevNewlineIndex:nextNewlineIndex])
}

// this function checks which occurence of the secret is relevant based on the location provided
func findRelevantOccurenceOfSecret(fragment string, secret string, loc Location) []int {
re := regexp.MustCompile(regexp.QuoteMeta(secret))
matches := re.FindAllStringIndex(fragment, -1)

// Extract the start indices
for _, match := range matches {
if match[0] >= loc.startLineIndex && match[1] <= loc.endLineIndex {
return match
}
}

return nil
}

func findIndexAfterPreviousNewline(fragment string, startIdx int) int {
if startIdx <= 0 {
return 0
}

re, err := regexp.Compile(`\r|\n`)
if err != nil {
return 0
}

for i := startIdx; i >= 0; i-- {
char := fragment[i]

if isNewline := re.Match([]byte{char}); isNewline {
return i
}
}

return 0
}

func findIndexBeforeNextNewline(fragment string, startIdx int) int {
maxIdx := len(fragment)
if startIdx == maxIdx {
return maxIdx
}

re, err := regexp.Compile(`\r|\n`)
if err != nil {
return maxIdx
}

for i := startIdx; i < maxIdx; i++ {
char := fragment[i]

if isNewline := re.Match([]byte{char}); isNewline {
return i
}
}

return maxIdx
}
108 changes: 107 additions & 1 deletion detect/detect_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/zricethezav/gitleaks/v8/config"
"github.com/zricethezav/gitleaks/v8/report"
"github.com/zricethezav/gitleaks/v8/sources"
Expand Down Expand Up @@ -68,6 +67,7 @@ func TestDetect(t *testing.T) {
Match: "AKIALALEMEL33243OKIA",
File: "tmp.go",
Line: `awsToken := \"AKIALALEMEL33243OKIA\"`,
FullLine: `awsToken := \"AKIALALEMEL33243OKIA\"`,
RuleID: "aws-access-key",
Tags: []string{"key", "AWS"},
StartLine: 0,
Expand All @@ -90,6 +90,7 @@ func TestDetect(t *testing.T) {
Secret: "pypi-AgEIcHlwaS5vcmcAAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAAB",
Match: "pypi-AgEIcHlwaS5vcmcAAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAAB",
Line: `pypi-AgEIcHlwaS5vcmcAAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAAB`,
FullLine: `pypi-AgEIcHlwaS5vcmcAAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAA-AAAAAAAAAAB`,
File: "tmp.go",
RuleID: "pypi-upload-token",
Tags: []string{"key", "pypi"},
Expand All @@ -113,6 +114,7 @@ func TestDetect(t *testing.T) {
Secret: "AKIALALEMEL33243OLIA",
Match: "AKIALALEMEL33243OLIA",
Line: `awsToken := \"AKIALALEMEL33243OLIA\"`,
FullLine: `awsToken := \"AKIALALEMEL33243OLIA\"`,
File: "tmp.go",
RuleID: "aws-access-key",
Tags: []string{"key", "AWS"},
Expand All @@ -136,6 +138,7 @@ func TestDetect(t *testing.T) {
Match: "BUNDLE_ENTERPRISE__CONTRIBSYS__COM=cafebabe:deadbeef;",
Secret: "cafebabe:deadbeef",
Line: `export BUNDLE_ENTERPRISE__CONTRIBSYS__COM=cafebabe:deadbeef;`,
FullLine: `export BUNDLE_ENTERPRISE__CONTRIBSYS__COM=cafebabe:deadbeef;`,
File: "tmp.sh",
RuleID: "sidekiq-secret",
Tags: []string{},
Expand All @@ -160,6 +163,7 @@ func TestDetect(t *testing.T) {
Secret: "cafebabe:deadbeef",
File: "tmp.sh",
Line: `echo hello1; export BUNDLE_ENTERPRISE__CONTRIBSYS__COM="cafebabe:deadbeef" && echo hello2`,
FullLine: `echo hello1; export BUNDLE_ENTERPRISE__CONTRIBSYS__COM="cafebabe:deadbeef" && echo hello2`,
RuleID: "sidekiq-secret",
Tags: []string{},
Entropy: 2.6098502,
Expand All @@ -183,6 +187,7 @@ func TestDetect(t *testing.T) {
Secret: "cafeb4b3:d3adb33f",
File: "tmp.sh",
Line: `url = "http://cafeb4b3:[email protected]:80/path?param1=true&param2=false#heading1"`,
FullLine: `url = "http://cafeb4b3:[email protected]:80/path?param1=true&param2=false#heading1"`,
RuleID: "sidekiq-sensitive-url",
Tags: []string{},
Entropy: 2.984234,
Expand Down Expand Up @@ -230,6 +235,7 @@ func TestDetect(t *testing.T) {
Match: "Discord_Public_Key = \"e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5\"",
Secret: "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5",
Line: `const Discord_Public_Key = "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5"`,
FullLine: `const Discord_Public_Key = "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5"`,
File: "tmp.go",
RuleID: "discord-api-key",
Tags: []string{},
Expand Down Expand Up @@ -261,6 +267,7 @@ func TestDetect(t *testing.T) {
Match: "Key = \"e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5\"",
Secret: "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5",
Line: `const Discord_Public_Key = "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5"`,
FullLine: `const Discord_Public_Key = "e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5"`,
File: "tmp.py",
RuleID: "generic-api-key",
Tags: []string{},
Expand Down Expand Up @@ -373,6 +380,7 @@ func TestFromGit(t *testing.T) {
StartColumn: 19,
EndColumn: 38,
Line: "\n awsToken := \"AKIALALEMEL33243OLIA\"",
FullLine: "awsToken := \"AKIALALEMEL33243OLIA\"",
Secret: "AKIALALEMEL33243OLIA",
Match: "AKIALALEMEL33243OLIA",
File: "main.go",
Expand All @@ -395,6 +403,7 @@ func TestFromGit(t *testing.T) {
Secret: "AKIALALEMEL33243OLIA",
Match: "AKIALALEMEL33243OLIA",
Line: "\n\taws_token := \"AKIALALEMEL33243OLIA\"",
FullLine: "aws_token := \"AKIALALEMEL33243OLIA\"",
File: "foo/foo.go",
Date: "2021-11-02T23:48:06Z",
Commit: "491504d5a31946ce75e22554cc34203d8e5ff3ca",
Expand All @@ -421,6 +430,7 @@ func TestFromGit(t *testing.T) {
EndColumn: 36,
Secret: "AKIALALEMEL33243OLIA",
Line: "\n\taws_token := \"AKIALALEMEL33243OLIA\"",
FullLine: "aws_token := \"AKIALALEMEL33243OLIA\"",
Match: "AKIALALEMEL33243OLIA",
Date: "2021-11-02T23:48:06Z",
File: "foo/foo.go",
Expand Down Expand Up @@ -496,6 +506,7 @@ func TestFromGitStaged(t *testing.T) {
StartColumn: 18,
EndColumn: 37,
Line: "\n\taws_token2 := \"AKIALALEMEL33243OLIA\" // this one is not",
FullLine: "aws_token2 := \"AKIALALEMEL33243OLIA\" // this one is not",
Match: "AKIALALEMEL33243OLIA",
Secret: "AKIALALEMEL33243OLIA",
File: "api/api.go",
Expand Down Expand Up @@ -568,6 +579,7 @@ func TestFromFiles(t *testing.T) {
Match: "AKIALALEMEL33243OLIA",
Secret: "AKIALALEMEL33243OLIA",
Line: "\n\tawsToken := \"AKIALALEMEL33243OLIA\"",
FullLine: "awsToken := \"AKIALALEMEL33243OLIA\"",
File: "../testdata/repos/nogit/main.go",
SymlinkFile: "",
RuleID: "aws-access-key",
Expand All @@ -590,6 +602,7 @@ func TestFromFiles(t *testing.T) {
Match: "AKIALALEMEL33243OLIA",
Secret: "AKIALALEMEL33243OLIA",
Line: "\n\tawsToken := \"AKIALALEMEL33243OLIA\"",
FullLine: "awsToken := \"AKIALALEMEL33243OLIA\"",
File: "../testdata/repos/nogit/main.go",
RuleID: "aws-access-key",
Tags: []string{"key", "AWS"},
Expand Down Expand Up @@ -657,6 +670,7 @@ func TestDetectWithSymlinks(t *testing.T) {
Match: "-----BEGIN OPENSSH PRIVATE KEY-----",
Secret: "-----BEGIN OPENSSH PRIVATE KEY-----",
Line: "-----BEGIN OPENSSH PRIVATE KEY-----",
FullLine: "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW",
File: "../testdata/repos/symlinks/source_file/id_ed25519",
SymlinkFile: "../testdata/repos/symlinks/file_symlink/symlinked_id_ed25519",
RuleID: "apkey",
Expand Down Expand Up @@ -716,3 +730,95 @@ func moveDotGit(t *testing.T, from, to string) {
require.NoError(t, err)
}
}

func TestFindSecretLine(t *testing.T) {
tests := []struct {
name string
fragment Fragment
loc Location
secret string
expected string
}{
{
name: "Secret within a line with \\n",
fragment: Fragment{
Raw: "Line 1\nThis is a line with some secret data.\nAnother line follows here.",
},
loc: Location{startLineIndex: 7, endLineIndex: 50},
secret: "secret",
expected: "This is a line with some secret data.",
},
{
name: "Secret within a line with \\r\\n",
fragment: Fragment{
Raw: "Line 1\r\nThis is a line with some secret data.\r\nAnother line follows here.",
},
loc: Location{startLineIndex: 8, endLineIndex: 51},
secret: "secret",
expected: "This is a line with some secret data.",
},
{
name: "Secret at start of string",
fragment: Fragment{
Raw: "secret is at the start\nAnother line follows here.",
},
loc: Location{startLineIndex: 0, endLineIndex: 21},
secret: "secret",
expected: "secret is at the start",
},
{
name: "Secret at end of string",
fragment: Fragment{
Raw: "Line 1\nAnother line follows here with secret",
},
loc: Location{startLineIndex: 29, endLineIndex: 45},
secret: "secret",
expected: "Another line follows here with secret",
},
{
name: "Secret in single line string",
fragment: Fragment{
Raw: "This is a secret line.",
},
loc: Location{startLineIndex: 0, endLineIndex: 23},
secret: "secret",
expected: "This is a secret line.",
},
{
name: "Secret with no newlines around",
fragment: Fragment{
Raw: "This is a line with secret in the middle and no newlines.",
},
loc: Location{startLineIndex: 0, endLineIndex: 57},
secret: "secret",
expected: "This is a line with secret in the middle and no newlines.",
},
{
name: "Secret not found",
fragment: Fragment{
Raw: "This is a line with no secrets.",
},
loc: Location{startLineIndex: 0, endLineIndex: 30},
secret: "hello",
expected: "",
},
{
name: "Multiple newlines",
fragment: Fragment{
Raw: "\n\nThis is a line with a secret in between\n\n",
},
loc: Location{startLineIndex: 2, endLineIndex: 40},
secret: "secret",
expected: "This is a line with a secret in between",
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := findFullLine(tt.fragment, tt.loc, tt.secret)
if result != tt.expected {
t.Errorf("got %q, want %q", result, tt.expected)
}
})
}
}
Loading