diff --git a/go.mod b/go.mod index 1d92b9e..30d1e51 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/MatusOllah/slogcolor v1.4.0 github.com/gravwell/gravwell/v3 v3.8.34 github.com/jasonlvhit/gocron v0.0.1 + github.com/k0kubun/pp v3.0.1+incompatible github.com/kevincobain2000/go-msteams v1.1.1 github.com/mattn/go-isatty v0.0.20 github.com/natefinch/lumberjack v2.0.0+incompatible @@ -17,6 +18,7 @@ require ( github.com/BurntSushi/toml v1.4.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fatih/color v1.16.0 // indirect + github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index 391ed6c..3db2257 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,10 @@ github.com/gravwell/gravwell/v3 v3.8.34/go.mod h1:FsIn6mNCcY7wEswbhxRpLchB9cF5jj github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU= github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= +github.com/k0kubun/pp v3.0.1+incompatible h1:3tqvf7QgUnZ5tXO6pNAZlrvHgl6DvifjDrd9g2S9Z40= +github.com/k0kubun/pp v3.0.1+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= github.com/kevincobain2000/go-msteams v1.1.1 h1:vZ8AYvVmiCdC+VZwsw7RFhb89RG/GasX9kvbdKheFN4= github.com/kevincobain2000/go-msteams v1.1.1/go.mod h1:+HowoQQHg9HLfx3CYQGImGGYw20+kN9rFmUXgxrqBzo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= diff --git a/main.go b/main.go index ae57e49..7a2186f 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,6 @@ package main import ( - "flag" "fmt" "log/slog" "os" @@ -26,9 +25,6 @@ var caches = make(map[string]*cache.Cache) func main() { pkg.Parseflags(&f) pkg.SetupLoggingStdout(f) // nolint: errcheck - flag.VisitAll(func(f *flag.Flag) { - slog.Info(f.Name, slog.String("value", f.Value.String())) - }) parseProxy() wantsVersion() validate() @@ -39,7 +35,7 @@ func main() { } var err error - newFilePaths, err := pkg.FilesByPattern(f.FilePath, 86400) + newFilePaths, err := pkg.FilesByPattern(f.FilePath, f.FileRecentSecs) if err != nil { slog.Error("Error finding files", "error", err.Error()) return @@ -132,7 +128,7 @@ func cron() { func syncFilePaths() { var err error - newFilePaths, err := pkg.FilesByPattern(f.FilePath, 86400) + newFilePaths, err := pkg.FilesByPattern(f.FilePath, f.FileRecentSecs) if err != nil { slog.Error("Error finding files", "error", err.Error()) return @@ -183,19 +179,22 @@ func watch(filePath string) { } func reportResult(result *pkg.ScanResult) { + slog.Info("File info", "filePath", result.FilePath, "size", result.FileInfo.Size(), "modTime", result.FileInfo.ModTime()) slog.Info("Lines read", "count", result.LinesRead) slog.Info("Scanning complete", "filePath", result.FilePath) slog.Info("1st line", "date", result.FirstDate, "line", pkg.Truncate(result.FirstLine, pkg.TruncateMax)) slog.Info("Preview line", "line", pkg.Truncate(result.PreviewLine, pkg.TruncateMax)) slog.Info("Last line", "date", result.LastDate, "line", pkg.Truncate(result.LastLine, pkg.TruncateMax)) slog.Info("Error count", "percent", fmt.Sprintf("%d (%.2f)", result.ErrorCount, result.ErrorPercent)+"%") - slog.Info("Error history", "streaks", result.Streak) + slog.Info("History", "max streak", f.Streak, "current streaks", result.Streak, "symbols", pkg.StreakSymbols(result.Streak, f.Streak, f.Min)) + slog.Info("Scan", "count", result.ScanCount) - if result.ErrorCount < f.Min { + if result.ScanCount == 1 { return } - if pkg.NonStreakZero(result.Streak, f.Streak) { + if !pkg.NonStreakZero(result.Streak, f.Streak, f.Min) { + slog.Info("Streak not met", "streak", f.Streak, "streaks", result.Streak) return } diff --git a/pkg/flags.go b/pkg/flags.go index e2c0270..e01f1d0 100644 --- a/pkg/flags.go +++ b/pkg/flags.go @@ -5,23 +5,24 @@ import ( ) type Flags struct { - FilePath string - FilePathsCap int - Match string - Ignore string - PostCommand string - LogFile string + FilePath string + FilePathsCap int + FileRecentSecs uint64 + Match string + Ignore string + PostCommand string + LogFile string - Min int - Streak int - Every uint64 - Proxy string - LogLevel int - MemLimit int - MSTeamsHook string - MaxBufferSizeMB int - Test bool - Version bool + Min int + Streak int + Every uint64 + Proxy string + LogLevel int + MemLimit int + MSTeamsHook string + MaxBufferMB int + Test bool + Version bool } func Parseflags(f *Flags) { @@ -33,11 +34,12 @@ func Parseflags(f *Flags) { flag.StringVar(&f.PostCommand, "post-cmd", "", "run this shell command after every scan when min errors are found") flag.Uint64Var(&f.Every, "every", 0, "run every n seconds (0 to run once)") flag.IntVar(&f.LogLevel, "log-level", 0, "log level (0=info, -4=debug, 4=warn, 8=error)") - flag.IntVar(&f.MemLimit, "mem-limit", 256, "memory limit in MB (0 to disable)") + flag.IntVar(&f.MemLimit, "mem-limit", 128, "memory limit in MB (0 to disable)") flag.IntVar(&f.FilePathsCap, "file-paths-cap", 100, "max number of file paths to watch") + flag.Uint64Var(&f.FileRecentSecs, "file-recent-secs", 86400, "only files modified in the last n seconds, 0 to disable") flag.IntVar(&f.Min, "min", 1, "on minimum num of matches, it should notify") flag.IntVar(&f.Streak, "streak", 1, "on minimum num of streak matches, it should notify") - flag.IntVar(&f.MaxBufferSizeMB, "max-buffer-size-mb", 0, "max buffer size in MB, default is 0 (not provided) for go's default 64KB") + flag.IntVar(&f.MaxBufferMB, "mbf", 0, "max buffer in MB, default is 0 (not provided) for go's default 64KB") flag.BoolVar(&f.Version, "version", false, "") flag.BoolVar(&f.Test, "test", false, `Quickly test paths or regex # will test if the input matches the regex diff --git a/pkg/notify.go b/pkg/notify.go index 1807d35..552e974 100644 --- a/pkg/notify.go +++ b/pkg/notify.go @@ -4,6 +4,7 @@ import ( "fmt" "log/slog" "os" + "time" gmt "github.com/kevincobain2000/go-msteams/src" ) @@ -62,18 +63,6 @@ func Notify(result *ScanResult, f Flags, version string) { Label: "Ignore Pattern", Message: f.Ignore, }, - { - Label: "Min Errors Threshold", - Message: fmt.Sprintf("%d", f.Min), - }, - { - Label: "Lines Read", - Message: fmt.Sprintf("%d", result.LinesRead), - }, - { - Label: "Total Errors Found", - Message: fmt.Sprintf("%d (%.2f)", result.ErrorCount, result.ErrorPercent) + "%", - }, { Label: "First Line", Message: Truncate(result.FirstLine, TruncateMax), @@ -86,11 +75,40 @@ func Notify(result *ScanResult, f Flags, version string) { Label: "Last Line", Message: Truncate(result.LastLine, TruncateMax), }, + { + Label: "Details", + Message: fmt.Sprintf( + "Min Threshold: %d, Lines Read: %d\n\rMatches Found: %d, Ratio %.2f%%", + f.Min, + result.LinesRead, + result.ErrorCount, + result.ErrorPercent, + ), + }, + { + Label: fmt.Sprintf("Streaks (Max %d)", f.Streak), + Message: StreakSymbols(result.Streak, f.Streak, f.Min), + }, } if result.FirstDate != "" || result.LastDate != "" { + var duration string + if result.FirstDate != "" && result.LastDate != "" { + firstDate, err := time.Parse("2006-01-02 15:04:05", result.FirstDate) + if err != nil { + duration = "X" + } else { + lastDate, err := time.Parse("2006-01-02 15:04:05", result.LastDate) + if err == nil { + duration = lastDate.Sub(firstDate).String() + } else { + duration = "X" + } + } + } + details = append(details, gmt.Details{ - Label: "Time Range", - Message: fmt.Sprintf("%s to %s", result.FirstDate, result.LastDate), + Label: "Range", + Message: fmt.Sprintf("%s to %s (Duration: %s)", result.FirstDate, result.LastDate, duration), }) } diff --git a/pkg/slices.go b/pkg/slices.go index 9849015..11272e7 100644 --- a/pkg/slices.go +++ b/pkg/slices.go @@ -9,13 +9,15 @@ func Capped[T any](cap int, slice []T) []T { return slice[:capped] } -func NonStreakZero(streaks []int, streak int) bool { - nonZeroStreak := make([]int, 0, len(streaks)) - for _, v := range streaks { - if v != 0 { - nonZeroStreak = append(nonZeroStreak, v) +func NonStreakZero(streaks []int, streak int, minimum int) bool { + // check if last three elements are over a minimum + if len(streaks) < streak { + return false + } + for i := 0; i < streak; i++ { + if streaks[len(streaks)-1-i] < minimum { + return false } } - - return len(nonZeroStreak) <= streak + return true } diff --git a/pkg/strings.go b/pkg/strings.go index 9ff1cb0..50806b8 100644 --- a/pkg/strings.go +++ b/pkg/strings.go @@ -4,6 +4,7 @@ import ( "crypto/sha256" "encoding/hex" "log/slog" + "strings" "sync" "github.com/gravwell/gravwell/v3/timegrinder" @@ -65,3 +66,20 @@ func SearchDate(input string) string { } return ts.Format("2006-01-02 15:04:05") } + +func StreakSymbols(arr []int, length int, minimum int) string { + var symbols []string + for _, v := range arr { + if v >= minimum { + symbols = append(symbols, "❌️") + } else { + symbols = append(symbols, "✅️") + } + } + // Fill the rest with grey symbols based on streak length + for i := len(symbols); i < length*StreakMultiplier; i++ { + symbols = append([]string{"🔲"}, symbols...) + } + + return strings.Join(symbols, " ") +} diff --git a/pkg/testit.go b/pkg/testit.go index b4cb3c5..bc3e131 100644 --- a/pkg/testit.go +++ b/pkg/testit.go @@ -6,7 +6,7 @@ import ( ) func TestIt(filepath string, match string) { - fps, err := FilesByPattern(filepath, 86400) + fps, err := FilesByPattern(filepath, 0) if err != nil { slog.Error("Error finding files", "error", err.Error()) } diff --git a/pkg/watcher.go b/pkg/watcher.go index 46404ba..93a4ff0 100644 --- a/pkg/watcher.go +++ b/pkg/watcher.go @@ -16,15 +16,20 @@ type Watcher struct { lastLineKey string lastFileSizeKey string errorHistoryKey string + scanCountKey string matchPattern string ignorePattern string - maxBufferSizeMB int + maxBufferMB int lastLineNum int lastFileSize int64 timestampNow string streak int // Number of error counts to maintain } +const ( + StreakMultiplier = 2 +) + func NewWatcher( filePath string, f Flags, @@ -37,12 +42,13 @@ func NewWatcher( filePath: filePath, matchPattern: f.Match, ignorePattern: f.Ignore, - lastLineKey: "llk-" + filePath, - lastFileSizeKey: "llks-" + filePath, + lastLineKey: "lk-" + filePath, + lastFileSizeKey: "sk-" + filePath, errorHistoryKey: "eh-" + filePath, + scanCountKey: "sc-" + filePath, timestampNow: now.Format("2006-01-02 15:04:05"), - maxBufferSizeMB: f.MaxBufferSizeMB, - streak: f.Streak + 1, + maxBufferMB: f.MaxBufferMB, + streak: f.Streak * StreakMultiplier, } if err := watcher.loadState(); err != nil { return nil, err @@ -63,6 +69,7 @@ type ScanResult struct { LastLine string LastDate string Streak []int // History of error counts for this file path + ScanCount int // Total number of scans performed } func (w *Watcher) Scan() (*ScanResult, error) { @@ -103,9 +110,9 @@ func (w *Watcher) Scan() (*ScanResult, error) { } scanner := bufio.NewScanner(file) - if w.maxBufferSizeMB > 0 { + if w.maxBufferMB > 0 { // For large lines - scanner.Buffer(make([]byte, 0, 64*1024), w.maxBufferSizeMB*1024*1024) + scanner.Buffer(make([]byte, 0, 64*1024), w.maxBufferMB*1024*1024) } currentLineNum := 1 linesRead := 0 @@ -129,8 +136,8 @@ func (w *Watcher) Scan() (*ScanResult, error) { if firstLine == "" { firstLine = lineStr } - if len(previewLine) < 1000 { - previewLine += lineStr + "\n" + if len(previewLine) < 500 { + previewLine += lineStr + "\n\r" } lastLine = lineStr matchCounts++ @@ -154,6 +161,9 @@ func (w *Watcher) Scan() (*ScanResult, error) { w.lastLineNum = currentLineNum w.lastFileSize = bytesRead + // Update scan count + w.incrementScanCount() + // Update error history w.updateErrorHistory(matchCounts) @@ -165,6 +175,9 @@ func (w *Watcher) Scan() (*ScanResult, error) { // Get the error history errorHistory := w.getErrorHistory() + // Get the scan count + scanCount := w.getScanCount() + return &ScanResult{ ErrorCount: matchCounts, FirstDate: SearchDate(firstLine), @@ -177,6 +190,7 @@ func (w *Watcher) Scan() (*ScanResult, error) { ErrorPercent: matchPercentage, LinesRead: linesRead, Streak: errorHistory, + ScanCount: scanCount, }, nil } @@ -197,6 +211,9 @@ func (w *Watcher) saveState() error { } func (w *Watcher) updateErrorHistory(newErrorCount int) { + if w.getScanCount() == 1 { + return + } var history []int if value, found := w.cache.Get(w.errorHistoryKey); found { history = value.([]int) @@ -218,6 +235,22 @@ func (w *Watcher) getErrorHistory() []int { return []int{} } +func (w *Watcher) incrementScanCount() { + count := 0 + if value, found := w.cache.Get(w.scanCountKey); found { + count = value.(int) + } + count++ + w.cache.Set(w.scanCountKey, count, cache.DefaultExpiration) +} + +func (w *Watcher) getScanCount() int { + if value, found := w.cache.Get(w.scanCountKey); found { + return value.(int) + } + return 0 +} + func (w *Watcher) Close() error { return nil }