feat: moving

README.md

@@ -36,6 +36,8 @@ curl -sL https://raw.githubusercontent.com/rakutentech/go-watch-logs/master/inst
 
 ## Examples
 
+### Watching a log file for errors
+
 ```sh
 # match error patterns and notify on MS Teams
 go-watch-logs --file-path=my.log --match="error:pattern1|error:pattern2" --ms-teams-hook="https://outlook.office.com/webhook/xxxxx"
@@ -50,6 +52,11 @@ go-watch-logs --file-path=my.log --match='HTTP/1.1" 50|HTTP/1.1" 40' --ignore='H
 go-watch-logs --file-path=my.log --match='HTTP/1.1" 50' --every=60
 ```
 
+### Watching a log file for anomalies
+
+```sh
+```
+
 
 **All done!**
 

main.go

@@ -21,7 +21,7 @@ var filePaths []string
 var filePathsMutex sync.Mutex
 
 func main() {
-	flags()
+	pkg.Parseflags(&f)
 	pkg.SetupLoggingStdout(f.LogLevel, f.LogFile) // nolint: errcheck
 	flag.VisitAll(func(f *flag.Flag) {
 		slog.Info(f.Name, slog.String("value", f.Value.String()))
@@ -194,11 +194,11 @@ func watch(filePath string) {
 			return
 		}
 		if !f.NotifyOnlyRecent {
-			notify(result)
+			pkg.Notify(result, f, version)
 		}
 
 		if f.NotifyOnlyRecent && pkg.IsRecentlyModified(result.FileInfo, f.Every) {
-			notify(result)
+			pkg.Notify(result, f, version)
 		}
 		if f.PostCommand != "" {
 			if _, err := pkg.ExecShell(f.PostCommand); err != nil {
@@ -208,63 +208,6 @@ func watch(filePath string) {
 	}
 }
 
-func notify(result *pkg.ScanResult) {
-	slog.Info("Sending to MS Teams")
-	details := pkg.GetAlertDetails(&f, version, result)
-
-	var logDetails []interface{} // nolint: prealloc
-	for _, detail := range details {
-		logDetails = append(logDetails, detail.Label, detail.Message)
-	}
-
-	if f.MSTeamsHook == "" {
-		slog.Warn("MS Teams hook not set")
-		return
-	}
-	slog.Info("Sending Alert Notify", logDetails...)
-
-	hostname, _ := os.Hostname()
-
-	err := gmt.Send(hostname, details, f.MSTeamsHook, f.Proxy)
-	if err != nil {
-		slog.Error("Error sending to Teams", "error", err.Error())
-	} else {
-		slog.Info("Successfully sent to MS Teams")
-	}
-}
-
-func flags() {
-	flag.StringVar(&f.FilePath, "file-path", "", "full path to the file to watch")
-	flag.StringVar(&f.FilePath, "f", "", "(short for --file-path) full path to the file to watch")
-	flag.StringVar(&f.LogFile, "log-file", "", "full path to output log file")
-	flag.StringVar(&f.DBPath, "db-path", pkg.GetHomedir()+"/.go-watch-logs.db", "path to store db file. Note dir must exist prior")
-	flag.StringVar(&f.Match, "match", "", "regex for matching errors (empty to match all lines)")
-	flag.StringVar(&f.Ignore, "ignore", "", "regex for ignoring errors (empty to ignore none)")
-	flag.StringVar(&f.PostAlways, "post-always", "", "run this shell command after every scan")
-	flag.StringVar(&f.PostCommand, "post-cmd", "", "run this shell command after every scan when min errors are found")
-	flag.Uint64Var(&f.Every, "every", 0, "run every n seconds (0 to run once)")
-	flag.Uint64Var(&f.HealthCheckEvery, "health-check-every", 0, "run health check every n seconds (0 to disable)")
-	flag.IntVar(&f.LogLevel, "log-level", 0, "log level (0=info, -4=debug, 4=warn, 8=error)")
-	flag.IntVar(&f.MemLimit, "mem-limit", 100, "memory limit in MB (0 to disable)")
-	flag.IntVar(&f.FilePathsCap, "file-paths-cap", 100, "max number of file paths to watch")
-	flag.IntVar(&f.Min, "min", 1, "on minimum num of matches, it should notify")
-	flag.BoolVar(&f.Anomaly, "anomaly", false, "")
-	flag.IntVar(&f.AnomalyWindowDays, "anomaly-window-days", 7, "anomaly window days")
-	flag.BoolVar(&f.NotifyOnlyRecent, "notify-only-recent", true, "Notify on latest file only by timestamp based on --every")
-	flag.BoolVar(&f.Version, "version", false, "")
-	flag.BoolVar(&f.Test, "test", false, `Quickly test paths or regex
-# will test if the input matches the regex
-echo test123 | go-watch-logs --match=123 --test
-# will test if the file paths are found and list them
-go-watch-logs --file-path=./ssl_access.*log --test
-	`)
-
-	flag.StringVar(&f.Proxy, "proxy", "", "http proxy for webhooks")
-	flag.StringVar(&f.MSTeamsHook, "ms-teams-hook", "", "ms teams webhook")
-
-	flag.Parse()
-}
-
 func parseProxy() string {
 	systemProxy := pkg.SystemProxy()
 	if systemProxy != "" && f.Proxy == "" {

pkg/database.go

@@ -38,11 +38,6 @@ func InitDB(dbName string) (*sql.DB, error) {
 		return nil, err
 	}
 
-	if err := vaccumIfOver(db, dbName, 100); err != nil {
-		slog.Error("Error vacuuming database", "error", err.Error())
-		return nil, err
-	}
-
 	db.SetMaxOpenConns(5)
 	db.SetMaxIdleConns(5)
 	db.SetConnMaxLifetime(time.Hour)

pkg/files.go

@@ -91,3 +91,17 @@ func IsRecentlyModified(fileInfo os.FileInfo, within uint64) bool {
 	// Check if the difference is within the adjusted duration
 	return diff <= time.Duration(adjustedWithin)*time.Second
 }
+
+func EnsureDirectoryExists(filePath string) error {
+	// Extract the directory from the file path
+	dir := filepath.Dir(filePath)
+
+	// Check if the directory exists
+	if _, err := os.Stat(dir); os.IsNotExist(err) {
+		// Create the directory if it does not exist
+		if err := os.MkdirAll(dir, os.ModePerm); err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/flags.go

@@ -1,5 +1,9 @@
 package pkg
 
+import (
+	"flag"
+)
+
 type Flags struct {
 	FilePath     string
 	FilePathsCap int
@@ -23,3 +27,55 @@ type Flags struct {
 	Test              bool
 	Version           bool
 }
+
+func Parseflags(f *Flags) {
+	flag.StringVar(&f.FilePath, "file-path", "", "full path to the file to watch")
+	flag.StringVar(&f.FilePath, "f", "", "(short for --file-path) full path to the file to watch")
+	flag.StringVar(&f.LogFile, "log-file", "", "full path to output log file")
+	flag.StringVar(&f.DBPath, "db-path", GetHomedir()+"/.go-watch-logs.db", "path to store db file. Note dir must exist prior")
+	flag.StringVar(&f.Match, "match", "", "regex for matching errors (empty to match all lines)")
+	flag.StringVar(&f.Ignore, "ignore", "", "regex for ignoring errors (empty to ignore none)")
+	flag.StringVar(&f.PostAlways, "post-always", "", "run this shell command after every scan")
+	flag.StringVar(&f.PostCommand, "post-cmd", "", "run this shell command after every scan when min errors are found")
+	flag.Uint64Var(&f.Every, "every", 0, "run every n seconds (0 to run once)")
+	flag.Uint64Var(&f.HealthCheckEvery, "health-check-every", 0, `run health check every n seconds (0 to disable)
+sends health check ping to ms teams webhook
+`)
+	flag.IntVar(&f.LogLevel, "log-level", 0, "log level (0=info, -4=debug, 4=warn, 8=error)")
+	flag.IntVar(&f.MemLimit, "mem-limit", 100, "memory limit in MB (0 to disable)")
+	flag.IntVar(&f.FilePathsCap, "file-paths-cap", 100, "max number of file paths to watch")
+	flag.IntVar(&f.Min, "min", 1, "on minimum num of matches, it should notify")
+	flag.BoolVar(&f.Anomaly, "anomaly", false, "record and watch for anomalies (keeping this true not notify on normal matching errors, only on anomalies)")
+	flag.IntVar(&f.AnomalyWindowDays, "anomaly-window-days", 7, `anomaly window days
+keep data in DB for n days, older data will be deleted
+`)
+	flag.BoolVar(&f.NotifyOnlyRecent, "notify-only-recent", true, "Notify on latest file only by timestamp based on --every")
+	flag.BoolVar(&f.Version, "version", false, "")
+	flag.BoolVar(&f.Test, "test", false, `Quickly test paths or regex
+# will test if the input matches the regex
+echo test123 | go-watch-logs --match=123 --test
+# will test if the file paths are found and list them
+go-watch-logs --file-path=./ssl_access.*log --test
+	`)
+
+	flag.StringVar(&f.Proxy, "proxy", "", "http proxy for webhooks")
+	flag.StringVar(&f.MSTeamsHook, "ms-teams-hook", "", "ms teams webhook")
+
+	flag.Parse()
+	ParsePostFlags(f)
+}
+
+func ParsePostFlags(f *Flags) {
+	if f.LogFile != "" {
+		if err := EnsureDirectoryExists(f.LogFile); err != nil {
+			panic("Failed to ensure directory for log file: " + err.Error())
+		}
+	}
+
+	if f.DBPath != "" {
+		if err := EnsureDirectoryExists(f.DBPath); err != nil {
+			panic("Failed to ensure directory for DB path: " + err.Error())
+		}
+	}
+
+}

pkg/card.go → pkg/notify.go

@@ -2,11 +2,58 @@ package pkg
 
 import (
 	"fmt"
+	"log/slog"
+	"os"
 	"runtime"
 
 	gmt "github.com/kevincobain2000/go-msteams/src"
 )
 
+func NotifyOwnError(e error, f Flags) {
+	hostname, _ := os.Hostname()
+	details := []gmt.Details{
+		{
+			Label:   "Hostname",
+			Message: hostname,
+		},
+		{
+			Label:   "Error",
+			Message: e.Error(),
+		},
+	}
+	err := gmt.Send(hostname, details, f.MSTeamsHook, f.Proxy)
+	if err != nil {
+		slog.Error("Error sending to Teams", "error", err.Error())
+	} else {
+		slog.Info("Successfully sent own error to MS Teams")
+	}
+}
+
+func Notify(result *ScanResult, f Flags, version string) {
+	slog.Info("Sending to MS Teams")
+	details := GetAlertDetails(&f, version, result)
+
+	var logDetails []interface{} // nolint: prealloc
+	for _, detail := range details {
+		logDetails = append(logDetails, detail.Label, detail.Message)
+	}
+
+	if f.MSTeamsHook == "" {
+		slog.Warn("MS Teams hook not set")
+		return
+	}
+	slog.Info("Sending Alert Notify", logDetails...)
+
+	hostname, _ := os.Hostname()
+
+	err := gmt.Send(hostname, details, f.MSTeamsHook, f.Proxy)
+	if err != nil {
+		slog.Error("Error sending to Teams", "error", err.Error())
+	} else {
+		slog.Info("Successfully sent to MS Teams")
+	}
+}
+
 func GetPanicDetails(f *Flags, m *runtime.MemStats) []gmt.Details {
 	return []gmt.Details{
 		{

pkg/watcher.go

@@ -124,13 +124,18 @@ func (w *Watcher) Scan() (*ScanResult, error) {
 		if linesRead < 0 {
 			linesRead = -linesRead
 		}
-		// slog.Debug("Scanning line", "line", string(line), "lineNum", currentLineNum, "linesRead", linesRead)
+
 		if w.ignorePattern != "" && regIgnore.Match(line) {
 			continue
 		}
 
 		// anomaly insertion
 		if w.anomaly {
+			if w.matchPattern == "" {
+				slog.Info("Match entire line, incrementing +1 match as empty")
+				w.anomalizer.MemSafeCount("")
+				continue
+			}
 			match := regMatch.FindAllString(string(line), -1)
 			var exactMatch string
 			if len(match) >= 1 {
@@ -140,10 +145,9 @@ func (w *Watcher) Scan() (*ScanResult, error) {
 				slog.Info("Match found", "line", string(line), "match", exactMatch)
 				w.anomalizer.MemSafeCount(exactMatch)
 			}
-			continue // no need to go for match as this is anomaly check only
 		}
 
-		if regMatch.Match(line) {
+		if !w.anomaly && regMatch.Match(line) {
 			lineStr := string(line)
 			if firstLine == "" {
 				firstLine = lineStr