From 0b0803ad2d0e3180f966fc99b94ac91a022fac58 Mon Sep 17 00:00:00 2001 From: Pulkit Kathuria Date: Sat, 28 Dec 2024 19:08:18 +0900 Subject: [PATCH] feat: date and time --- pkg/anomalizer.go | 94 +++++++++++++++++++++++++++++++++++++++++---- pkg/database.go | 3 +- pkg/files.go | 2 +- pkg/flags.go | 4 +- pkg/strings.go | 7 ++++ pkg/watcher.go | 97 ++++++++++++++++++++--------------------------- 6 files changed, 139 insertions(+), 68 deletions(-) diff --git a/pkg/anomalizer.go b/pkg/anomalizer.go index 2c3a59b..60852d9 100644 --- a/pkg/anomalizer.go +++ b/pkg/anomalizer.go @@ -1,21 +1,99 @@ package pkg +import ( + "database/sql" + "time" +) + type Anomalizer struct { - counter map[string]int + f Flags + db *sql.DB + now time.Time + key string + window int + counters []Counter limitCounterKeys int } +type Counter struct { + Match string + Value int +} -func NewAnomalizer() *Anomalizer { +func NewAnomalizer(db *sql.DB, f Flags, now time.Time, key string, window int) *Anomalizer { return &Anomalizer{ - counter: make(map[string]int), + db: db, + f: f, + now: now, + key: key, + window: window, + counters: []Counter{}, limitCounterKeys: 100, } } -func (a *Anomalizer) MemSafeCount(key string) { - tk := Truncate(key, 100) - a.counter[tk]++ - if a.counter[tk] > a.limitCounterKeys { - delete(a.counter, tk) +func (a *Anomalizer) MemSafeCount(match string) { + // Check if the key exists in counters + for i, counter := range a.counters { + if counter.Match == match { + // Increment the counter if the key exists + a.counters[i].Value++ + // If the counter exceeds the limit, remove it + if a.counters[i].Value > a.limitCounterKeys { + a.counters = append(a.counters[:i], a.counters[i+1:]...) + } + return + } + } + + // If the key does not exist, add a new counter + a.counters = append(a.counters, Counter{Match: match, Value: 1}) +} + +type Anomaly struct { + Match string + Value int +} + +func (a *Anomalizer) GetAnomalies(match string) ([]int, error) { + // Query to fetch anomalies within the time range + rows, err := a.db.Query( + `SELECT match, value + FROM anomalies + WHERE key = ? + AND match = ? + AND time = ?`, + a.key, match, a.now.Format("15:04"), + ) + if err != nil { + return nil, err + } + defer rows.Close() + + // Iterate over query results and populate the slice + values := []int{} + for rows.Next() { + var anomaly Anomaly + if err := rows.Scan(&anomaly.Match, &anomaly.Value); err != nil { + return nil, err + } + values = append(values, anomaly.Value) + } + + return values, nil +} + +func (a *Anomalizer) SaveAnomalies() error { + for _, counter := range a.counters { + _, err := a.db.Exec(`INSERT INTO anomalies (key, match, value, date, time) VALUES (?, ?, ?, ?, ?)`, a.key, counter.Match, counter.Value, a.now.Format("2006-01-02"), a.now.Format("15:04")) + if err != nil { + return err + } } + return nil +} + +func (a *Anomalizer) DeleteOldAnomalies() error { + windowAt := a.now.AddDate(0, 0, -a.window).Format("2006-01-02") + _, err := a.db.Exec(`DELETE FROM anomalies WHERE key = ? AND date < ?`, a.key, windowAt) + return err } diff --git a/pkg/database.go b/pkg/database.go index 214af1b..0e793e2 100644 --- a/pkg/database.go +++ b/pkg/database.go @@ -106,7 +106,8 @@ func createTables(db *sql.DB) error { key TEXT KEY, match TEXT, value INTEGER, - created_at DATETIME + date DATE, + time TIME ) `) if err != nil { diff --git a/pkg/files.go b/pkg/files.go index 6ed717d..d41dacd 100644 --- a/pkg/files.go +++ b/pkg/files.go @@ -92,7 +92,7 @@ func IsRecentlyModified(fileInfo os.FileInfo, within uint64) bool { return diff <= time.Duration(adjustedWithin)*time.Second } -func EnsureDirectoryExists(filePath string) error { +func MkdirP(filePath string) error { // Extract the directory from the file path dir := filepath.Dir(filePath) diff --git a/pkg/flags.go b/pkg/flags.go index eee43c5..17edaba 100644 --- a/pkg/flags.go +++ b/pkg/flags.go @@ -67,13 +67,13 @@ go-watch-logs --file-path=./ssl_access.*log --test func ParsePostFlags(f *Flags) { if f.LogFile != "" { - if err := EnsureDirectoryExists(f.LogFile); err != nil { + if err := MkdirP(f.LogFile); err != nil { panic("Failed to ensure directory for log file: " + err.Error()) } } if f.DBPath != "" { - if err := EnsureDirectoryExists(f.DBPath); err != nil { + if err := MkdirP(f.DBPath); err != nil { panic("Failed to ensure directory for DB path: " + err.Error()) } } diff --git a/pkg/strings.go b/pkg/strings.go index 1b2e128..cdb3615 100644 --- a/pkg/strings.go +++ b/pkg/strings.go @@ -26,6 +26,13 @@ func Truncate(s string, n int) string { return s[:n] + "..." } +func LimitString(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] +} + var ( tg *timegrinder.TimeGrinder once sync.Once diff --git a/pkg/watcher.go b/pkg/watcher.go index 72da3ff..7d05634 100644 --- a/pkg/watcher.go +++ b/pkg/watcher.go @@ -12,20 +12,20 @@ import ( ) type Watcher struct { - db *sql.DB - dbName string // full path - filePath string - anomalyKey string - lastLineKey string - lastFileSizeKey string - matchPattern string - ignorePattern string - lastLineNum int - lastFileSize int64 - anomalizer *Anomalizer - anomaly bool - anomalyWindow int - timestampNow string + db *sql.DB + dbName string // full path + filePath string + anomalyKey string + lastLineKey string + lastFileSizeKey string + matchPattern string + ignorePattern string + lastLineNum int + lastFileSize int64 + anomalizer *Anomalizer + anomaly bool + anomalyWindowDays int + timestampNow string } func NewWatcher( @@ -37,21 +37,23 @@ func NewWatcher( if err != nil { return nil, err } + now := time.Now() watcher := &Watcher{ - db: db, - dbName: dbName, - filePath: filePath, - anomaly: f.Anomaly, - anomalizer: NewAnomalizer(), - anomalyWindow: f.AnomalyWindowDays, - matchPattern: f.Match, - ignorePattern: f.Ignore, - anomalyKey: "anm-" + filePath, - lastLineKey: "llk-" + filePath, - lastFileSizeKey: "llks-" + filePath, - timestampNow: time.Now().Format("2006-01-02 15:04:05"), - } + db: db, + dbName: dbName, + filePath: filePath, + anomaly: f.Anomaly, + anomalizer: nil, + anomalyWindowDays: f.AnomalyWindowDays, + matchPattern: f.Match, + ignorePattern: f.Ignore, + anomalyKey: "anm-" + filePath, + lastLineKey: "llk-" + filePath, + lastFileSizeKey: "llks-" + filePath, + timestampNow: now.Format("2006-01-02 15:04:05"), + } + watcher.anomalizer = NewAnomalizer(db, f, now, watcher.anomalyKey, f.AnomalyWindowDays) if err := watcher.loadState(); err != nil { return nil, err } @@ -75,7 +77,7 @@ type ScanResult struct { var lines = []string{} func (w *Watcher) Scan() (*ScanResult, error) { - errorCounts := 0 + matchCounts := 0 firstLine := "" lastLine := "" previewLine := "" @@ -145,7 +147,10 @@ func (w *Watcher) Scan() (*ScanResult, error) { } if exactMatch != "" { slog.Info("Match found", "line", string(line), "match", exactMatch) - w.anomalizer.MemSafeCount(exactMatch) + if len(exactMatch) > 100 { + slog.Warn("Match too long, this impacts DB size, limiting to 100", "match", exactMatch) + } + w.anomalizer.MemSafeCount(LimitString(exactMatch, 100)) } } @@ -158,16 +163,16 @@ func (w *Watcher) Scan() (*ScanResult, error) { previewLine += lineStr + "\n" } lastLine = lineStr - errorCounts++ + matchCounts++ } } if w.anomaly { slog.Info("Saving anomalies") - if err := w.SaveAnomalies(); err != nil { + if err := w.anomalizer.SaveAnomalies(); err != nil { return nil, err } slog.Info("Deleting old anomalies") - if err := w.DeleteOldAnomalies(); err != nil { + if err := w.anomalizer.DeleteOldAnomalies(); err != nil { return nil, err } } @@ -178,7 +183,7 @@ func (w *Watcher) Scan() (*ScanResult, error) { matchPercentage := 0.0 if linesRead > 0 { - matchPercentage = float64(errorCounts) * 100 / float64(linesRead) + matchPercentage = float64(matchCounts) * 100 / float64(linesRead) if matchPercentage > 100 { matchPercentage = 100 } @@ -197,12 +202,12 @@ func (w *Watcher) Scan() (*ScanResult, error) { return nil, err } return &ScanResult{ - ErrorCount: errorCounts, - FirstLine: firstLine, + ErrorCount: matchCounts, FirstDate: SearchDate(firstLine), + LastDate: SearchDate(lastLine), + FirstLine: firstLine, PreviewLine: previewLine, LastLine: lastLine, - LastDate: SearchDate(lastLine), FilePath: w.filePath, FileInfo: fileInfo, ErrorPercent: matchPercentage, @@ -238,26 +243,6 @@ func (w *Watcher) saveState() error { return err } -func (w *Watcher) SaveAnomalies() error { - for match, value := range w.anomalizer.counter { - _, err := w.db.Exec(`INSERT INTO anomalies (key, match, value, created_at) VALUES (?, ?, ?, ?)`, w.anomalyKey, match, value, w.timestampNow) - if err != nil { - return err - } - } - return nil -} - -func (w *Watcher) DeleteOldAnomalies() error { - now, err := time.Parse("2006-01-02 15:04:05", w.timestampNow) - if err != nil { - return err - } - windowAt := now.AddDate(0, 0, -w.anomalyWindow).Format("2006-01-02 15:04:05") - _, err = w.db.Exec(`DELETE FROM anomalies WHERE key = ? AND created_at < ?`, w.anomalyKey, windowAt) - return err -} - func (w *Watcher) Close() error { // return w.db.Close() return nil