feat: date and time

pkg/anomalizer.go

@@ -1,21 +1,99 @@
 package pkg
 
+import (
+	"database/sql"
+	"time"
+)
+
 type Anomalizer struct {
-	counter          map[string]int
+	f                Flags
+	db               *sql.DB
+	now              time.Time
+	key              string
+	window           int
+	counters         []Counter
 	limitCounterKeys int
 }
+type Counter struct {
+	Match string
+	Value int
+}
 
-func NewAnomalizer() *Anomalizer {
+func NewAnomalizer(db *sql.DB, f Flags, now time.Time, key string, window int) *Anomalizer {
 	return &Anomalizer{
-		counter:          make(map[string]int),
+		db:               db,
+		f:                f,
+		now:              now,
+		key:              key,
+		window:           window,
+		counters:         []Counter{},
 		limitCounterKeys: 100,
 	}
 }
 
-func (a *Anomalizer) MemSafeCount(key string) {
-	tk := Truncate(key, 100)
-	a.counter[tk]++
-	if a.counter[tk] > a.limitCounterKeys {
-		delete(a.counter, tk)
+func (a *Anomalizer) MemSafeCount(match string) {
+	// Check if the key exists in counters
+	for i, counter := range a.counters {
+		if counter.Match == match {
+			// Increment the counter if the key exists
+			a.counters[i].Value++
+			// If the counter exceeds the limit, remove it
+			if a.counters[i].Value > a.limitCounterKeys {
+				a.counters = append(a.counters[:i], a.counters[i+1:]...)
+			}
+			return
+		}
+	}
+
+	// If the key does not exist, add a new counter
+	a.counters = append(a.counters, Counter{Match: match, Value: 1})
+}
+
+type Anomaly struct {
+	Match string
+	Value int
+}
+
+func (a *Anomalizer) GetAnomalies(match string) ([]int, error) {
+	// Query to fetch anomalies within the time range
+	rows, err := a.db.Query(
+		`SELECT match, value
+		FROM anomalies
+		WHERE key = ?
+		AND match = ?
+		AND time = ?`,
+		a.key, match, a.now.Format("15:04"),
+	)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	// Iterate over query results and populate the slice
+	values := []int{}
+	for rows.Next() {
+		var anomaly Anomaly
+		if err := rows.Scan(&anomaly.Match, &anomaly.Value); err != nil {
+			return nil, err
+		}
+		values = append(values, anomaly.Value)
+	}
+
+	return values, nil
+}
+
+func (a *Anomalizer) SaveAnomalies() error {
+	for _, counter := range a.counters {
+		_, err := a.db.Exec(`INSERT INTO anomalies (key, match, value, date, time) VALUES (?, ?, ?, ?, ?)`, a.key, counter.Match, counter.Value, a.now.Format("2006-01-02"), a.now.Format("15:04"))
+		if err != nil {
+			return err
+		}
 	}
+	return nil
+}
+
+func (a *Anomalizer) DeleteOldAnomalies() error {
+	windowAt := a.now.AddDate(0, 0, -a.window).Format("2006-01-02")
+	_, err := a.db.Exec(`DELETE FROM anomalies WHERE key = ? AND date < ?`, a.key, windowAt)
+	return err
 }

pkg/database.go

@@ -106,7 +106,8 @@ func createTables(db *sql.DB) error {
 			key TEXT KEY,
 			match TEXT,
 			value INTEGER,
-			created_at DATETIME
+			date DATE,
+			time TIME
 		)
 	`)
 	if err != nil {

pkg/files.go

@@ -92,7 +92,7 @@ func IsRecentlyModified(fileInfo os.FileInfo, within uint64) bool {
 	return diff <= time.Duration(adjustedWithin)*time.Second
 }
 
-func EnsureDirectoryExists(filePath string) error {
+func MkdirP(filePath string) error {
 	// Extract the directory from the file path
 	dir := filepath.Dir(filePath)
 

pkg/flags.go

@@ -67,13 +67,13 @@ go-watch-logs --file-path=./ssl_access.*log --test
 
 func ParsePostFlags(f *Flags) {
 	if f.LogFile != "" {
-		if err := EnsureDirectoryExists(f.LogFile); err != nil {
+		if err := MkdirP(f.LogFile); err != nil {
 			panic("Failed to ensure directory for log file: " + err.Error())
 		}
 	}
 
 	if f.DBPath != "" {
-		if err := EnsureDirectoryExists(f.DBPath); err != nil {
+		if err := MkdirP(f.DBPath); err != nil {
 			panic("Failed to ensure directory for DB path: " + err.Error())
 		}
 	}

pkg/strings.go

@@ -26,6 +26,13 @@ func Truncate(s string, n int) string {
 	return s[:n] + "..."
 }
 
+func LimitString(s string, n int) string {
+	if len(s) <= n {
+		return s
+	}
+	return s[:n]
+}
+
 var (
 	tg   *timegrinder.TimeGrinder
 	once sync.Once

pkg/watcher.go

@@ -12,20 +12,20 @@ import (
 )
 
 type Watcher struct {
-	db              *sql.DB
-	dbName          string // full path
-	filePath        string
-	anomalyKey      string
-	lastLineKey     string
-	lastFileSizeKey string
-	matchPattern    string
-	ignorePattern   string
-	lastLineNum     int
-	lastFileSize    int64
-	anomalizer      *Anomalizer
-	anomaly         bool
-	anomalyWindow   int
-	timestampNow    string
+	db                *sql.DB
+	dbName            string // full path
+	filePath          string
+	anomalyKey        string
+	lastLineKey       string
+	lastFileSizeKey   string
+	matchPattern      string
+	ignorePattern     string
+	lastLineNum       int
+	lastFileSize      int64
+	anomalizer        *Anomalizer
+	anomaly           bool
+	anomalyWindowDays int
+	timestampNow      string
 }
 
 func NewWatcher(
@@ -37,21 +37,23 @@ func NewWatcher(
 	if err != nil {
 		return nil, err
 	}
+	now := time.Now()
 
 	watcher := &Watcher{
-		db:              db,
-		dbName:          dbName,
-		filePath:        filePath,
-		anomaly:         f.Anomaly,
-		anomalizer:      NewAnomalizer(),
-		anomalyWindow:   f.AnomalyWindowDays,
-		matchPattern:    f.Match,
-		ignorePattern:   f.Ignore,
-		anomalyKey:      "anm-" + filePath,
-		lastLineKey:     "llk-" + filePath,
-		lastFileSizeKey: "llks-" + filePath,
-		timestampNow:    time.Now().Format("2006-01-02 15:04:05"),
-	}
+		db:                db,
+		dbName:            dbName,
+		filePath:          filePath,
+		anomaly:           f.Anomaly,
+		anomalizer:        nil,
+		anomalyWindowDays: f.AnomalyWindowDays,
+		matchPattern:      f.Match,
+		ignorePattern:     f.Ignore,
+		anomalyKey:        "anm-" + filePath,
+		lastLineKey:       "llk-" + filePath,
+		lastFileSizeKey:   "llks-" + filePath,
+		timestampNow:      now.Format("2006-01-02 15:04:05"),
+	}
+	watcher.anomalizer = NewAnomalizer(db, f, now, watcher.anomalyKey, f.AnomalyWindowDays)
 	if err := watcher.loadState(); err != nil {
 		return nil, err
 	}
@@ -75,7 +77,7 @@ type ScanResult struct {
 var lines = []string{}
 
 func (w *Watcher) Scan() (*ScanResult, error) {
-	errorCounts := 0
+	matchCounts := 0
 	firstLine := ""
 	lastLine := ""
 	previewLine := ""
@@ -145,7 +147,10 @@ func (w *Watcher) Scan() (*ScanResult, error) {
 			}
 			if exactMatch != "" {
 				slog.Info("Match found", "line", string(line), "match", exactMatch)
-				w.anomalizer.MemSafeCount(exactMatch)
+				if len(exactMatch) > 100 {
+					slog.Warn("Match too long, this impacts DB size, limiting to 100", "match", exactMatch)
+				}
+				w.anomalizer.MemSafeCount(LimitString(exactMatch, 100))
 			}
 		}
 
@@ -158,16 +163,16 @@ func (w *Watcher) Scan() (*ScanResult, error) {
 				previewLine += lineStr + "\n"
 			}
 			lastLine = lineStr
-			errorCounts++
+			matchCounts++
 		}
 	}
 	if w.anomaly {
 		slog.Info("Saving anomalies")
-		if err := w.SaveAnomalies(); err != nil {
+		if err := w.anomalizer.SaveAnomalies(); err != nil {
 			return nil, err
 		}
 		slog.Info("Deleting old anomalies")
-		if err := w.DeleteOldAnomalies(); err != nil {
+		if err := w.anomalizer.DeleteOldAnomalies(); err != nil {
 			return nil, err
 		}
 	}
@@ -178,7 +183,7 @@ func (w *Watcher) Scan() (*ScanResult, error) {
 
 	matchPercentage := 0.0
 	if linesRead > 0 {
-		matchPercentage = float64(errorCounts) * 100 / float64(linesRead)
+		matchPercentage = float64(matchCounts) * 100 / float64(linesRead)
 		if matchPercentage > 100 {
 			matchPercentage = 100
 		}
@@ -197,12 +202,12 @@ func (w *Watcher) Scan() (*ScanResult, error) {
 		return nil, err
 	}
 	return &ScanResult{
-		ErrorCount:   errorCounts,
-		FirstLine:    firstLine,
+		ErrorCount:   matchCounts,
 		FirstDate:    SearchDate(firstLine),
+		LastDate:     SearchDate(lastLine),
+		FirstLine:    firstLine,
 		PreviewLine:  previewLine,
 		LastLine:     lastLine,
-		LastDate:     SearchDate(lastLine),
 		FilePath:     w.filePath,
 		FileInfo:     fileInfo,
 		ErrorPercent: matchPercentage,
@@ -238,26 +243,6 @@ func (w *Watcher) saveState() error {
 	return err
 }
 
-func (w *Watcher) SaveAnomalies() error {
-	for match, value := range w.anomalizer.counter {
-		_, err := w.db.Exec(`INSERT INTO anomalies (key, match, value, created_at) VALUES (?, ?, ?, ?)`, w.anomalyKey, match, value, w.timestampNow)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (w *Watcher) DeleteOldAnomalies() error {
-	now, err := time.Parse("2006-01-02 15:04:05", w.timestampNow)
-	if err != nil {
-		return err
-	}
-	windowAt := now.AddDate(0, 0, -w.anomalyWindow).Format("2006-01-02 15:04:05")
-	_, err = w.db.Exec(`DELETE FROM anomalies WHERE key = ? AND created_at < ?`, w.anomalyKey, windowAt)
-	return err
-}
-
 func (w *Watcher) Close() error {
 	// return w.db.Close()
 	return nil