From 0c93960849820e99cb7766764cab65e79f457f1a Mon Sep 17 00:00:00 2001 From: Pulkit Kathuria Date: Sat, 28 Dec 2024 05:11:07 +0900 Subject: [PATCH] feat: moving --- README.md | 7 +++++ main.go | 63 ++------------------------------------ pkg/database.go | 5 --- pkg/files.go | 14 +++++++++ pkg/flags.go | 56 +++++++++++++++++++++++++++++++++ pkg/{card.go => notify.go} | 47 ++++++++++++++++++++++++++++ pkg/watcher.go | 10 ++++-- 7 files changed, 134 insertions(+), 68 deletions(-) rename pkg/{card.go => notify.go} (69%) diff --git a/README.md b/README.md index b1a4c38..278cdb3 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,8 @@ curl -sL https://raw.githubusercontent.com/rakutentech/go-watch-logs/master/inst ## Examples +### Watching a log file for errors + ```sh # match error patterns and notify on MS Teams go-watch-logs --file-path=my.log --match="error:pattern1|error:pattern2" --ms-teams-hook="https://outlook.office.com/webhook/xxxxx" @@ -50,6 +52,11 @@ go-watch-logs --file-path=my.log --match='HTTP/1.1" 50|HTTP/1.1" 40' --ignore='H go-watch-logs --file-path=my.log --match='HTTP/1.1" 50' --every=60 ``` +### Watching a log file for anomalies + +```sh +``` + **All done!** diff --git a/main.go b/main.go index 881ea49..2cbf7d2 100644 --- a/main.go +++ b/main.go @@ -21,7 +21,7 @@ var filePaths []string var filePathsMutex sync.Mutex func main() { - flags() + pkg.Parseflags(&f) pkg.SetupLoggingStdout(f.LogLevel, f.LogFile) // nolint: errcheck flag.VisitAll(func(f *flag.Flag) { slog.Info(f.Name, slog.String("value", f.Value.String())) @@ -194,11 +194,11 @@ func watch(filePath string) { return } if !f.NotifyOnlyRecent { - notify(result) + pkg.Notify(result, f, version) } if f.NotifyOnlyRecent && pkg.IsRecentlyModified(result.FileInfo, f.Every) { - notify(result) + pkg.Notify(result, f, version) } if f.PostCommand != "" { if _, err := pkg.ExecShell(f.PostCommand); err != nil { @@ -208,63 +208,6 @@ func watch(filePath string) { } } -func notify(result *pkg.ScanResult) { - slog.Info("Sending to MS Teams") - details := pkg.GetAlertDetails(&f, version, result) - - var logDetails []interface{} // nolint: prealloc - for _, detail := range details { - logDetails = append(logDetails, detail.Label, detail.Message) - } - - if f.MSTeamsHook == "" { - slog.Warn("MS Teams hook not set") - return - } - slog.Info("Sending Alert Notify", logDetails...) - - hostname, _ := os.Hostname() - - err := gmt.Send(hostname, details, f.MSTeamsHook, f.Proxy) - if err != nil { - slog.Error("Error sending to Teams", "error", err.Error()) - } else { - slog.Info("Successfully sent to MS Teams") - } -} - -func flags() { - flag.StringVar(&f.FilePath, "file-path", "", "full path to the file to watch") - flag.StringVar(&f.FilePath, "f", "", "(short for --file-path) full path to the file to watch") - flag.StringVar(&f.LogFile, "log-file", "", "full path to output log file") - flag.StringVar(&f.DBPath, "db-path", pkg.GetHomedir()+"/.go-watch-logs.db", "path to store db file. Note dir must exist prior") - flag.StringVar(&f.Match, "match", "", "regex for matching errors (empty to match all lines)") - flag.StringVar(&f.Ignore, "ignore", "", "regex for ignoring errors (empty to ignore none)") - flag.StringVar(&f.PostAlways, "post-always", "", "run this shell command after every scan") - flag.StringVar(&f.PostCommand, "post-cmd", "", "run this shell command after every scan when min errors are found") - flag.Uint64Var(&f.Every, "every", 0, "run every n seconds (0 to run once)") - flag.Uint64Var(&f.HealthCheckEvery, "health-check-every", 0, "run health check every n seconds (0 to disable)") - flag.IntVar(&f.LogLevel, "log-level", 0, "log level (0=info, -4=debug, 4=warn, 8=error)") - flag.IntVar(&f.MemLimit, "mem-limit", 100, "memory limit in MB (0 to disable)") - flag.IntVar(&f.FilePathsCap, "file-paths-cap", 100, "max number of file paths to watch") - flag.IntVar(&f.Min, "min", 1, "on minimum num of matches, it should notify") - flag.BoolVar(&f.Anomaly, "anomaly", false, "") - flag.IntVar(&f.AnomalyWindowDays, "anomaly-window-days", 7, "anomaly window days") - flag.BoolVar(&f.NotifyOnlyRecent, "notify-only-recent", true, "Notify on latest file only by timestamp based on --every") - flag.BoolVar(&f.Version, "version", false, "") - flag.BoolVar(&f.Test, "test", false, `Quickly test paths or regex -# will test if the input matches the regex -echo test123 | go-watch-logs --match=123 --test -# will test if the file paths are found and list them -go-watch-logs --file-path=./ssl_access.*log --test - `) - - flag.StringVar(&f.Proxy, "proxy", "", "http proxy for webhooks") - flag.StringVar(&f.MSTeamsHook, "ms-teams-hook", "", "ms teams webhook") - - flag.Parse() -} - func parseProxy() string { systemProxy := pkg.SystemProxy() if systemProxy != "" && f.Proxy == "" { diff --git a/pkg/database.go b/pkg/database.go index a493fc9..214af1b 100644 --- a/pkg/database.go +++ b/pkg/database.go @@ -38,11 +38,6 @@ func InitDB(dbName string) (*sql.DB, error) { return nil, err } - if err := vaccumIfOver(db, dbName, 100); err != nil { - slog.Error("Error vacuuming database", "error", err.Error()) - return nil, err - } - db.SetMaxOpenConns(5) db.SetMaxIdleConns(5) db.SetConnMaxLifetime(time.Hour) diff --git a/pkg/files.go b/pkg/files.go index 4de9260..6ed717d 100644 --- a/pkg/files.go +++ b/pkg/files.go @@ -91,3 +91,17 @@ func IsRecentlyModified(fileInfo os.FileInfo, within uint64) bool { // Check if the difference is within the adjusted duration return diff <= time.Duration(adjustedWithin)*time.Second } + +func EnsureDirectoryExists(filePath string) error { + // Extract the directory from the file path + dir := filepath.Dir(filePath) + + // Check if the directory exists + if _, err := os.Stat(dir); os.IsNotExist(err) { + // Create the directory if it does not exist + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + return err + } + } + return nil +} diff --git a/pkg/flags.go b/pkg/flags.go index f3ab066..eee43c5 100644 --- a/pkg/flags.go +++ b/pkg/flags.go @@ -1,5 +1,9 @@ package pkg +import ( + "flag" +) + type Flags struct { FilePath string FilePathsCap int @@ -23,3 +27,55 @@ type Flags struct { Test bool Version bool } + +func Parseflags(f *Flags) { + flag.StringVar(&f.FilePath, "file-path", "", "full path to the file to watch") + flag.StringVar(&f.FilePath, "f", "", "(short for --file-path) full path to the file to watch") + flag.StringVar(&f.LogFile, "log-file", "", "full path to output log file") + flag.StringVar(&f.DBPath, "db-path", GetHomedir()+"/.go-watch-logs.db", "path to store db file. Note dir must exist prior") + flag.StringVar(&f.Match, "match", "", "regex for matching errors (empty to match all lines)") + flag.StringVar(&f.Ignore, "ignore", "", "regex for ignoring errors (empty to ignore none)") + flag.StringVar(&f.PostAlways, "post-always", "", "run this shell command after every scan") + flag.StringVar(&f.PostCommand, "post-cmd", "", "run this shell command after every scan when min errors are found") + flag.Uint64Var(&f.Every, "every", 0, "run every n seconds (0 to run once)") + flag.Uint64Var(&f.HealthCheckEvery, "health-check-every", 0, `run health check every n seconds (0 to disable) +sends health check ping to ms teams webhook +`) + flag.IntVar(&f.LogLevel, "log-level", 0, "log level (0=info, -4=debug, 4=warn, 8=error)") + flag.IntVar(&f.MemLimit, "mem-limit", 100, "memory limit in MB (0 to disable)") + flag.IntVar(&f.FilePathsCap, "file-paths-cap", 100, "max number of file paths to watch") + flag.IntVar(&f.Min, "min", 1, "on minimum num of matches, it should notify") + flag.BoolVar(&f.Anomaly, "anomaly", false, "record and watch for anomalies (keeping this true not notify on normal matching errors, only on anomalies)") + flag.IntVar(&f.AnomalyWindowDays, "anomaly-window-days", 7, `anomaly window days +keep data in DB for n days, older data will be deleted +`) + flag.BoolVar(&f.NotifyOnlyRecent, "notify-only-recent", true, "Notify on latest file only by timestamp based on --every") + flag.BoolVar(&f.Version, "version", false, "") + flag.BoolVar(&f.Test, "test", false, `Quickly test paths or regex +# will test if the input matches the regex +echo test123 | go-watch-logs --match=123 --test +# will test if the file paths are found and list them +go-watch-logs --file-path=./ssl_access.*log --test + `) + + flag.StringVar(&f.Proxy, "proxy", "", "http proxy for webhooks") + flag.StringVar(&f.MSTeamsHook, "ms-teams-hook", "", "ms teams webhook") + + flag.Parse() + ParsePostFlags(f) +} + +func ParsePostFlags(f *Flags) { + if f.LogFile != "" { + if err := EnsureDirectoryExists(f.LogFile); err != nil { + panic("Failed to ensure directory for log file: " + err.Error()) + } + } + + if f.DBPath != "" { + if err := EnsureDirectoryExists(f.DBPath); err != nil { + panic("Failed to ensure directory for DB path: " + err.Error()) + } + } + +} diff --git a/pkg/card.go b/pkg/notify.go similarity index 69% rename from pkg/card.go rename to pkg/notify.go index 533ec68..dd4cad8 100644 --- a/pkg/card.go +++ b/pkg/notify.go @@ -2,11 +2,58 @@ package pkg import ( "fmt" + "log/slog" + "os" "runtime" gmt "github.com/kevincobain2000/go-msteams/src" ) +func NotifyOwnError(e error, f Flags) { + hostname, _ := os.Hostname() + details := []gmt.Details{ + { + Label: "Hostname", + Message: hostname, + }, + { + Label: "Error", + Message: e.Error(), + }, + } + err := gmt.Send(hostname, details, f.MSTeamsHook, f.Proxy) + if err != nil { + slog.Error("Error sending to Teams", "error", err.Error()) + } else { + slog.Info("Successfully sent own error to MS Teams") + } +} + +func Notify(result *ScanResult, f Flags, version string) { + slog.Info("Sending to MS Teams") + details := GetAlertDetails(&f, version, result) + + var logDetails []interface{} // nolint: prealloc + for _, detail := range details { + logDetails = append(logDetails, detail.Label, detail.Message) + } + + if f.MSTeamsHook == "" { + slog.Warn("MS Teams hook not set") + return + } + slog.Info("Sending Alert Notify", logDetails...) + + hostname, _ := os.Hostname() + + err := gmt.Send(hostname, details, f.MSTeamsHook, f.Proxy) + if err != nil { + slog.Error("Error sending to Teams", "error", err.Error()) + } else { + slog.Info("Successfully sent to MS Teams") + } +} + func GetPanicDetails(f *Flags, m *runtime.MemStats) []gmt.Details { return []gmt.Details{ { diff --git a/pkg/watcher.go b/pkg/watcher.go index c3d55a0..8b881e4 100644 --- a/pkg/watcher.go +++ b/pkg/watcher.go @@ -124,13 +124,18 @@ func (w *Watcher) Scan() (*ScanResult, error) { if linesRead < 0 { linesRead = -linesRead } - // slog.Debug("Scanning line", "line", string(line), "lineNum", currentLineNum, "linesRead", linesRead) + if w.ignorePattern != "" && regIgnore.Match(line) { continue } // anomaly insertion if w.anomaly { + if w.matchPattern == "" { + slog.Info("Match entire line, incrementing +1 match as empty") + w.anomalizer.MemSafeCount("") + continue + } match := regMatch.FindAllString(string(line), -1) var exactMatch string if len(match) >= 1 { @@ -140,10 +145,9 @@ func (w *Watcher) Scan() (*ScanResult, error) { slog.Info("Match found", "line", string(line), "match", exactMatch) w.anomalizer.MemSafeCount(exactMatch) } - continue // no need to go for match as this is anomaly check only } - if regMatch.Match(line) { + if !w.anomaly && regMatch.Match(line) { lineStr := string(line) if firstLine == "" { firstLine = lineStr