From c8996fff3ddadcaa99663ff101337173c2471fba Mon Sep 17 00:00:00 2001 From: Messede Degod Date: Sat, 11 Jan 2025 10:54:03 +0000 Subject: [PATCH 1/4] feat: disk logger --- .gitignore | 1 + cmd/certstream-server-go/main.go | 18 +- config.sample.yaml | 6 + go.mod | 4 +- go.sum | 4 +- .../certificatetransparency/ct-watcher.go | 41 +++- internal/config/config.go | 14 ++ internal/disk/filerotate/LICENSE | 21 ++ internal/disk/filerotate/filerotate.go | 217 ++++++++++++++++++ internal/disk/logger.go | 78 +++++++ 10 files changed, 387 insertions(+), 17 deletions(-) create mode 100644 internal/disk/filerotate/LICENSE create mode 100644 internal/disk/filerotate/filerotate.go create mode 100644 internal/disk/logger.go diff --git a/.gitignore b/.gitignore index 9823e0e..0c3829a 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ go.work .cache *.pprof dist/ +logs/* # Ignore actual config files config.yaml diff --git a/cmd/certstream-server-go/main.go b/cmd/certstream-server-go/main.go index e8917b9..1c762f5 100644 --- a/cmd/certstream-server-go/main.go +++ b/cmd/certstream-server-go/main.go @@ -7,6 +7,7 @@ import ( "github.com/d-Rickyy-b/certstream-server-go/internal/certificatetransparency" "github.com/d-Rickyy-b/certstream-server-go/internal/config" + "github.com/d-Rickyy-b/certstream-server-go/internal/disk" "github.com/d-Rickyy-b/certstream-server-go/internal/metrics" "github.com/d-Rickyy-b/certstream-server-go/internal/web" ) @@ -31,12 +32,23 @@ func main() { } webserver := web.NewWebsocketServer(conf.Webserver.ListenAddr, conf.Webserver.ListenPort, conf.Webserver.CertPath, conf.Webserver.CertKeyPath) - setupMetrics(conf, webserver) - go webserver.Start() - watcher := certificatetransparency.Watcher{} + if conf.DiskLogger.Enabled { + disk.StartLogger(conf.DiskLogger.LogDirectory, conf.DiskLogger.Type, conf.DiskLogger.Rotation) + } + + watcherLogChannels := []certificatetransparency.LogChannel{ + certificatetransparency.LOG_CHAN_WEBSOCKET, + } + if conf.DiskLogger.Enabled { + watcherLogChannels = append(watcherLogChannels, certificatetransparency.LOG_CHAN_DISK) + } + + log.Printf("Following log channels are enabled: %v\n", watcherLogChannels) + + watcher := certificatetransparency.Watcher{LogChannels: watcherLogChannels} watcher.Start() } diff --git a/config.sample.yaml b/config.sample.yaml index c6a9866..10b54fa 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -8,6 +8,12 @@ webserver: cert_key_path: "" compression_enabled: false +disklogger: + enabled: true + type: DOMAINS_ONLY # LITE, FULL, DOMAINS_ONLY + log_directory: logs + rotation: DAILY # HOURLY, DAILY + prometheus: enabled: true listen_addr: "0.0.0.0" diff --git a/go.mod b/go.mod index 8f81c69..93762e5 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/d-Rickyy-b/certstream-server-go -go 1.22.7 +go 1.23.1 toolchain go1.23.4 @@ -19,7 +19,7 @@ require ( github.com/valyala/histogram v1.2.0 // indirect golang.org/x/crypto v0.31.0 // indirect golang.org/x/net v0.33.0 // indirect - golang.org/x/sys v0.28.0 // indirect + golang.org/x/sys v0.29.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241223144023-3abc09e42ca8 // indirect google.golang.org/grpc v1.69.2 // indirect google.golang.org/protobuf v1.36.1 // indirect diff --git a/go.sum b/go.sum index 07660d9..15f993e 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,8 @@ golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20241223144023-3abc09e42ca8 h1:TqExAhdPaB60Ux47Cn0oLV07rGnxZzIsaRhQaqS666A= diff --git a/internal/certificatetransparency/ct-watcher.go b/internal/certificatetransparency/ct-watcher.go index 6af73b3..13ffdc1 100644 --- a/internal/certificatetransparency/ct-watcher.go +++ b/internal/certificatetransparency/ct-watcher.go @@ -14,6 +14,7 @@ import ( "github.com/d-Rickyy-b/certstream-server-go/internal/certstream" "github.com/d-Rickyy-b/certstream-server-go/internal/config" + "github.com/d-Rickyy-b/certstream-server-go/internal/disk" "github.com/d-Rickyy-b/certstream-server-go/internal/web" ct "github.com/google/certificate-transparency-go" @@ -29,13 +30,21 @@ var ( userAgent = fmt.Sprintf("Certstream Server v%s (github.com/d-Rickyy-b/certstream-server-go)", config.Version) ) +type LogChannel string + +const ( + LOG_CHAN_WEBSOCKET LogChannel = "WEBSOCKET" + LOG_CHAN_DISK LogChannel = "DISK" +) + // Watcher describes a component that watches for new certificates in a CT log. type Watcher struct { - workers []*worker - wg sync.WaitGroup - context context.Context - certChan chan certstream.Entry - cancelFunc context.CancelFunc + workers []*worker + wg sync.WaitGroup + context context.Context + certChan chan certstream.Entry + cancelFunc context.CancelFunc + LogChannels []LogChannel } // NewWatcher creates a new Watcher. @@ -58,7 +67,7 @@ func (w *Watcher) Start() { w.addNewlyAvailableLogs() log.Println("Started CT watcher") - go certHandler(w.certChan) + go certHandler(w.certChan, w.LogChannels) go w.watchNewLogs() w.wg.Wait() @@ -279,9 +288,20 @@ func (w *worker) foundPrecertCallback(rawEntry *ct.RawLogEntry) { atomic.AddInt64(&processedPrecerts, 1) } -// certHandler takes the entries out of the entryChan channel and broadcasts them to all clients. +// certHandler takes the entries out of the entryChan channel sends them to the appropriate log channels. // Only a single instance of the certHandler runs per certstream server. -func certHandler(entryChan chan certstream.Entry) { +func certHandler(entryChan chan certstream.Entry, logChannels []LogChannel) { + + channels := []chan certstream.Entry{} + for _, logChan := range logChannels { + switch logChan { + case LOG_CHAN_WEBSOCKET: + channels = append(channels, web.ClientHandler.Broadcast) //send entries to web socket + case LOG_CHAN_DISK: + channels = append(channels, disk.CertStreamEntryChan) //send entries to disk logger + } + } + var processed int64 for { @@ -294,8 +314,9 @@ func certHandler(entryChan chan certstream.Entry) { web.SetExampleCert(entry) } - // Run json encoding in the background and send the result to the clients. - web.ClientHandler.Broadcast <- entry + for _, logChan := range channels { + logChan <- entry + } // Update metrics url := entry.Data.Source.NormalizedURL diff --git a/internal/config/config.go b/internal/config/config.go index aeb7388..5496858 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -8,6 +8,7 @@ import ( "regexp" "strings" + "github.com/d-Rickyy-b/certstream-server-go/internal/disk" "gopkg.in/yaml.v3" ) @@ -33,6 +34,12 @@ type Config struct { DomainsOnlyURL string `yaml:"domains_only_url"` CompressionEnabled bool `yaml:"compression_enabled"` } + DiskLogger struct { + Enabled bool `yaml:"enabled"` + Type disk.DiskLog `yaml:"type"` + LogDirectory string `yaml:"log_directory"` + Rotation string `yaml:"rotation"` + } Prometheus struct { ServerConfig `yaml:",inline"` Enabled bool `yaml:"enabled"` @@ -187,5 +194,12 @@ func validateConfig(config Config) bool { } } + if config.DiskLogger.Enabled { + if config.DiskLogger.LogDirectory == "" { + log.Fatalln("Log Directory must be specified for disk logger") + return false + } + } + return true } diff --git a/internal/disk/filerotate/LICENSE b/internal/disk/filerotate/LICENSE new file mode 100644 index 0000000..5b00691 --- /dev/null +++ b/internal/disk/filerotate/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Krzysztof Kowalczyk + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/internal/disk/filerotate/filerotate.go b/internal/disk/filerotate/filerotate.go new file mode 100644 index 0000000..fff5df5 --- /dev/null +++ b/internal/disk/filerotate/filerotate.go @@ -0,0 +1,217 @@ +package filerotate + +import ( + "fmt" + "io" + "os" + "path/filepath" + "sync" + "time" +) + +type Config struct { + DidClose func(path string, didRotate bool) + PathIfShouldRotate func(creationTime time.Time, now time.Time) string +} + +type File struct { + sync.Mutex + + // Path is the path of the current file + Path string + + creationTime time.Time + + //Location *time.Location + + config Config + file *os.File + + // position in the file of last Write or Write2, exposed for tests + lastWritePos int64 +} + +func IsSameDay(t1, t2 time.Time) bool { + return t1.YearDay() == t2.YearDay() && t1.Year() == t2.Year() +} + +func IsSameHour(t1, t2 time.Time) bool { + return t1.YearDay() == t2.YearDay() && t1.Hour() == t2.Hour() +} + +func New(config *Config) (*File, error) { + if nil == config { + return nil, fmt.Errorf("must provide config") + } + if config.PathIfShouldRotate == nil { + return nil, fmt.Errorf("must provide config.ShouldRotate") + } + file := &File{ + config: *config, + } + err := file.reopenIfNeeded() + if err != nil { + return nil, err + } + return file, nil +} + +func MakeDailyRotateInDir(dir string, fileNameSuffix string) func(time.Time, time.Time) string { + return func(creationTime time.Time, now time.Time) string { + if IsSameDay(creationTime, now) { + return "" + } + name := now.Format("2006-01-02") + if fileNameSuffix != "" { + name = name + "-" + fileNameSuffix + } else { + name += ".txt" + } + return filepath.Join(dir, name) + } +} + +func MakeHourlyRotateInDir(dir string, fileNameSuffix string) func(time.Time, time.Time) string { + return func(creationTime time.Time, now time.Time) string { + if IsSameHour(creationTime, now) { + return "" + } + name := now.Format("2006-01-02_15") + if fileNameSuffix != "" { + name = name + "-" + fileNameSuffix + } else { + name += ".txt" + } + return filepath.Join(dir, name) + } +} + +// NewDaily creates a new file, rotating daily in a given directory +func NewDaily(dir string, fileUniqueName string, didClose func(path string, didRotate bool)) (*File, error) { + daily := MakeDailyRotateInDir(dir, fileUniqueName) + config := Config{ + DidClose: didClose, + PathIfShouldRotate: daily, + } + return New(&config) +} + +// NewHourly creates a new file, rotating hourly in a given directory +func NewHourly(dir string, logUniqueName string, didClose func(path string, didRotate bool)) (*File, error) { + hourly := MakeHourlyRotateInDir(dir, logUniqueName) + config := Config{ + DidClose: didClose, + PathIfShouldRotate: hourly, + } + return New(&config) +} + +func (f *File) close(didRotate bool) error { + if f.file == nil { + return nil + } + err := f.file.Close() + f.file = nil + if err == nil && f.config.DidClose != nil { + f.config.DidClose(f.Path, didRotate) + } + return err +} + +/* +func nowInMaybeLocation(loc *time.Location) time.Time { + now := time.Now() + if loc != nil { + now = now.In(loc) + } + return now +} +*/ + +func (f *File) open(path string) error { + f.Path = path + f.creationTime = time.Now() + // we can't assume that the dir for the file already exists + dir := filepath.Dir(f.Path) + err := os.MkdirAll(dir, 0755) + if err != nil { + return err + } + + // would be easier to open with os.O_APPEND but Seek() doesn't work in that case + flag := os.O_CREATE | os.O_WRONLY + f.file, err = os.OpenFile(f.Path, flag, 0644) + if err != nil { + return err + } + _, err = f.file.Seek(0, io.SeekEnd) + return err +} + +func (f *File) reopenIfNeeded() error { + now := time.Now() + newPath := f.config.PathIfShouldRotate(f.creationTime, now) + if newPath == "" { + return nil + } + err := f.close(true) + if err != nil { + return err + } + return f.open(newPath) +} + +func (f *File) write(d []byte, sync bool) (int64, int, error) { + err := f.reopenIfNeeded() + if err != nil { + return 0, 0, err + } + f.lastWritePos, err = f.file.Seek(0, io.SeekCurrent) + if err != nil { + return 0, 0, err + } + n, err := f.file.Write(d) + if err != nil { + return 0, n, err + } + if sync { + err = f.file.Sync() + } + return f.lastWritePos, n, err +} + +// Write writes data to a file +func (f *File) Write(d []byte) (int, error) { + f.Lock() + defer f.Unlock() + + _, n, err := f.write(d, false) + return n, err +} + +// Write2 writes data to a file, optionally syncs to disk. To enable users to later +// seek to where the data was written, it returns offset at which the data was +// written, number of bytes and error. +// You can get path of the file from f.Path +func (f *File) Write2(d []byte, sync bool) (int64, int, error) { + f.Lock() + defer f.Unlock() + + writtenAtPos, n, err := f.write(d, sync) + return writtenAtPos, n, err +} + +func (f *File) Close() error { + f.Lock() + defer f.Unlock() + + return f.close(false) +} + +// Sync commits the current contents of the file to stable storage. +func (f *File) Sync() error { + f.Lock() + defer f.Unlock() + + return f.file.Sync() +} diff --git a/internal/disk/logger.go b/internal/disk/logger.go new file mode 100644 index 0000000..921b3fd --- /dev/null +++ b/internal/disk/logger.go @@ -0,0 +1,78 @@ +package disk + +import ( + "log" + + "github.com/d-Rickyy-b/certstream-server-go/internal/certstream" + "github.com/d-Rickyy-b/certstream-server-go/internal/disk/filerotate" +) + +var ( + CertStreamEntryChan chan certstream.Entry +) + +type DiskLog string + +const ( + DISK_LOG_FULL DiskLog = "FULL" + DISK_LOG_LITE DiskLog = "LOG_LITE" + DISK_LOG_DOMAINS_ONLY DiskLog = "DOMAINS_ONLY" +) + +func StartLogger(logDirectory string, logType DiskLog, rotation string) { + + if CertStreamEntryChan == nil { + CertStreamEntryChan = make(chan certstream.Entry, 10_000) + } + + go logEntries(logDirectory, logType, rotation) +} + +func logEntries(logDirectory string, logType DiskLog, rotation string) { + var logFile *filerotate.File + var err error + + switch rotation { + case "HOURLY": + logFile, err = filerotate.NewHourly(logDirectory, "", onLogClose) + case "DAILY": + fallthrough + default: + logFile, err = filerotate.NewDaily(logDirectory, "", onLogClose) + } + + if err != nil { + log.Panic(err) + } + + for { + entry, ok := <-CertStreamEntryChan + + if !ok { + break + } + + switch logType { + case DISK_LOG_DOMAINS_ONLY: + for _, domain := range entry.Data.LeafCert.AllDomains { + logFile.Write([]byte(domain + "\n")) + } + case DISK_LOG_LITE: + logFile.Write(entry.JSONLite()) + case DISK_LOG_FULL: + fallthrough + default: + logFile.Write(entry.JSON()) + } + } + + logFile.Close() +} + +func onLogClose(path string, didRotate bool) { + if didRotate { + log.Printf("Log file at '%s' was rotated", path) + return + } + log.Printf("Log file at '%s' was closed", path) +} From c79d203ab9259f5c04063aff404abda9e1f8cb95 Mon Sep 17 00:00:00 2001 From: Messede Degod Date: Sun, 12 Jan 2025 09:25:38 +0000 Subject: [PATCH 2/4] update: simplify file rotation mechanism --- config.sample.yaml | 2 +- internal/disk/filerotate/LICENSE | 21 --- internal/disk/filerotate/filerotate.go | 252 ++++++++----------------- internal/disk/logger.go | 14 +- 4 files changed, 85 insertions(+), 204 deletions(-) delete mode 100644 internal/disk/filerotate/LICENSE diff --git a/config.sample.yaml b/config.sample.yaml index 10b54fa..f30b6b2 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -9,7 +9,7 @@ webserver: compression_enabled: false disklogger: - enabled: true + enabled: false type: DOMAINS_ONLY # LITE, FULL, DOMAINS_ONLY log_directory: logs rotation: DAILY # HOURLY, DAILY diff --git a/internal/disk/filerotate/LICENSE b/internal/disk/filerotate/LICENSE deleted file mode 100644 index 5b00691..0000000 --- a/internal/disk/filerotate/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2021 Krzysztof Kowalczyk - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/internal/disk/filerotate/filerotate.go b/internal/disk/filerotate/filerotate.go index fff5df5..f7b4725 100644 --- a/internal/disk/filerotate/filerotate.go +++ b/internal/disk/filerotate/filerotate.go @@ -2,216 +2,126 @@ package filerotate import ( "fmt" - "io" + "log" "os" "path/filepath" "sync" "time" ) -type Config struct { - DidClose func(path string, didRotate bool) - PathIfShouldRotate func(creationTime time.Time, now time.Time) string -} - -type File struct { +type RotatableFile struct { sync.Mutex - // Path is the path of the current file - Path string + Directory string // file dir + Path string // file path creationTime time.Time - //Location *time.Location - - config Config - file *os.File - - // position in the file of last Write or Write2, exposed for tests - lastWritePos int64 + file *os.File + rotateType RotateType } -func IsSameDay(t1, t2 time.Time) bool { - return t1.YearDay() == t2.YearDay() && t1.Year() == t2.Year() -} +type RotateType string -func IsSameHour(t1, t2 time.Time) bool { - return t1.YearDay() == t2.YearDay() && t1.Hour() == t2.Hour() -} +const ( + ROTATE_HOURLY RotateType = "ROTATE_HOURLY" + ROTATE_DAILY RotateType = "ROTATE_DAILY" +) -func New(config *Config) (*File, error) { - if nil == config { - return nil, fmt.Errorf("must provide config") - } - if config.PathIfShouldRotate == nil { - return nil, fmt.Errorf("must provide config.ShouldRotate") - } - file := &File{ - config: *config, +func New(directory string, rotateType RotateType) (*RotatableFile, error) { + file, filePath, ferr := newFile(directory, rotateType) + if ferr != nil { + return &RotatableFile{}, ferr } - err := file.reopenIfNeeded() - if err != nil { - return nil, err - } - return file, nil -} -func MakeDailyRotateInDir(dir string, fileNameSuffix string) func(time.Time, time.Time) string { - return func(creationTime time.Time, now time.Time) string { - if IsSameDay(creationTime, now) { - return "" - } - name := now.Format("2006-01-02") - if fileNameSuffix != "" { - name = name + "-" + fileNameSuffix - } else { - name += ".txt" - } - return filepath.Join(dir, name) + rotatableFile := RotatableFile{ + creationTime: time.Now().UTC(), + Directory: directory, + Path: filePath, + file: file, } -} -func MakeHourlyRotateInDir(dir string, fileNameSuffix string) func(time.Time, time.Time) string { - return func(creationTime time.Time, now time.Time) string { - if IsSameHour(creationTime, now) { - return "" - } - name := now.Format("2006-01-02_15") - if fileNameSuffix != "" { - name = name + "-" + fileNameSuffix - } else { - name += ".txt" - } - return filepath.Join(dir, name) - } -} + go rotatableFile.RotateFile() -// NewDaily creates a new file, rotating daily in a given directory -func NewDaily(dir string, fileUniqueName string, didClose func(path string, didRotate bool)) (*File, error) { - daily := MakeDailyRotateInDir(dir, fileUniqueName) - config := Config{ - DidClose: didClose, - PathIfShouldRotate: daily, - } - return New(&config) + return &rotatableFile, nil } -// NewHourly creates a new file, rotating hourly in a given directory -func NewHourly(dir string, logUniqueName string, didClose func(path string, didRotate bool)) (*File, error) { - hourly := MakeHourlyRotateInDir(dir, logUniqueName) - config := Config{ - DidClose: didClose, - PathIfShouldRotate: hourly, - } - return New(&config) -} +func newFile(directory string, rotateType RotateType) (file *os.File, filePath string, Error error) { + now := time.Now().UTC() + fileName := "" -func (f *File) close(didRotate bool) error { - if f.file == nil { - return nil + switch rotateType { + case ROTATE_HOURLY: + fileName = now.Format(time.RFC3339) + case ROTATE_DAILY: + fallthrough + default: + fileName = now.Format("2006-01-02") } - err := f.file.Close() - f.file = nil - if err == nil && f.config.DidClose != nil { - f.config.DidClose(f.Path, didRotate) - } - return err -} -/* -func nowInMaybeLocation(loc *time.Location) time.Time { - now := time.Now() - if loc != nil { - now = now.In(loc) - } - return now -} -*/ - -func (f *File) open(path string) error { - f.Path = path - f.creationTime = time.Now() - // we can't assume that the dir for the file already exists - dir := filepath.Dir(f.Path) - err := os.MkdirAll(dir, 0755) - if err != nil { - return err - } + filePath = filepath.Join(directory, fmt.Sprintf("%s.txt", fileName)) - // would be easier to open with os.O_APPEND but Seek() doesn't work in that case - flag := os.O_CREATE | os.O_WRONLY - f.file, err = os.OpenFile(f.Path, flag, 0644) - if err != nil { - return err + derr := os.MkdirAll(directory, 0755) + if derr != nil { + return nil, filePath, derr } - _, err = f.file.Seek(0, io.SeekEnd) - return err -} -func (f *File) reopenIfNeeded() error { - now := time.Now() - newPath := f.config.PathIfShouldRotate(f.creationTime, now) - if newPath == "" { - return nil + file, ferr := os.OpenFile(filePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if ferr != nil { + return nil, filePath, ferr } - err := f.close(true) - if err != nil { - return err - } - return f.open(newPath) -} -func (f *File) write(d []byte, sync bool) (int64, int, error) { - err := f.reopenIfNeeded() - if err != nil { - return 0, 0, err - } - f.lastWritePos, err = f.file.Seek(0, io.SeekCurrent) - if err != nil { - return 0, 0, err - } - n, err := f.file.Write(d) - if err != nil { - return 0, n, err - } - if sync { - err = f.file.Sync() - } - return f.lastWritePos, n, err + return file, filePath, nil } -// Write writes data to a file -func (f *File) Write(d []byte) (int, error) { - f.Lock() - defer f.Unlock() +func (rotatableFile *RotatableFile) RotateFile() { + for { + rotateAt := rotatableFile.getNextRotation() - _, n, err := f.write(d, false) - return n, err -} + log.Printf("Next Disk log rotation is in: %f Hours\n", rotateAt.Hours()) + + <-time.After(rotateAt) // wait untill duration + + newFile, _, nfErr := newFile(rotatableFile.Directory, rotatableFile.rotateType) + if nfErr != nil { + log.Println("Error while creation new file for rotation: ", nfErr) + } -// Write2 writes data to a file, optionally syncs to disk. To enable users to later -// seek to where the data was written, it returns offset at which the data was -// written, number of bytes and error. -// You can get path of the file from f.Path -func (f *File) Write2(d []byte, sync bool) (int64, int, error) { - f.Lock() - defer f.Unlock() + rotatableFile.Mutex.Lock() - writtenAtPos, n, err := f.write(d, sync) - return writtenAtPos, n, err + // switch to new file + oldFile := rotatableFile.file + rotatableFile.file = newFile + + rotatableFile.Mutex.Unlock() + + // sync and close old file + oldFile.Sync() + oldFile.Close() + } } -func (f *File) Close() error { - f.Lock() - defer f.Unlock() +func (rotatableFile *RotatableFile) getNextRotation() time.Duration { + now := time.Now().UTC() + switch rotatableFile.rotateType { + case ROTATE_HOURLY: + return now.Add(time.Hour).Sub(now) + case ROTATE_DAILY: + fallthrough + default: + return time.Date(now.Year(), now.Month(), now.Day()+1, 0, 0, 0, 0, time.UTC).Sub(now) // time between now and midnight + } +} - return f.close(false) +func (rotatableFile *RotatableFile) Write(b []byte) (n int, err error) { + rotatableFile.Mutex.Lock() + defer rotatableFile.Mutex.Unlock() + return rotatableFile.file.Write(b) } -// Sync commits the current contents of the file to stable storage. -func (f *File) Sync() error { - f.Lock() - defer f.Unlock() +func (rotatableFile *RotatableFile) Close() error { + rotatableFile.Mutex.Lock() + defer rotatableFile.Mutex.Unlock() - return f.file.Sync() + return rotatableFile.file.Close() } diff --git a/internal/disk/logger.go b/internal/disk/logger.go index 921b3fd..48ec36b 100644 --- a/internal/disk/logger.go +++ b/internal/disk/logger.go @@ -29,16 +29,16 @@ func StartLogger(logDirectory string, logType DiskLog, rotation string) { } func logEntries(logDirectory string, logType DiskLog, rotation string) { - var logFile *filerotate.File + var logFile *filerotate.RotatableFile var err error switch rotation { case "HOURLY": - logFile, err = filerotate.NewHourly(logDirectory, "", onLogClose) + logFile, err = filerotate.New(logDirectory, filerotate.ROTATE_HOURLY) case "DAILY": fallthrough default: - logFile, err = filerotate.NewDaily(logDirectory, "", onLogClose) + logFile, err = filerotate.New(logDirectory, filerotate.ROTATE_DAILY) } if err != nil { @@ -68,11 +68,3 @@ func logEntries(logDirectory string, logType DiskLog, rotation string) { logFile.Close() } - -func onLogClose(path string, didRotate bool) { - if didRotate { - log.Printf("Log file at '%s' was rotated", path) - return - } - log.Printf("Log file at '%s' was closed", path) -} From 5884b21d4040860a97d3d7ff074adeead6fa71d1 Mon Sep 17 00:00:00 2001 From: Messede Degod Date: Tue, 14 Jan 2025 06:47:21 +0000 Subject: [PATCH 3/4] fix: consider rotateType in filerotate.New() --- internal/disk/filerotate/filerotate.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/disk/filerotate/filerotate.go b/internal/disk/filerotate/filerotate.go index f7b4725..0b7a887 100644 --- a/internal/disk/filerotate/filerotate.go +++ b/internal/disk/filerotate/filerotate.go @@ -39,6 +39,7 @@ func New(directory string, rotateType RotateType) (*RotatableFile, error) { Directory: directory, Path: filePath, file: file, + rotateType: rotateType, } go rotatableFile.RotateFile() From 6aaf70d5a91de3e249ede81aaffb9f5e4cf05e62 Mon Sep 17 00:00:00 2001 From: Messede Degod Date: Tue, 14 Jan 2025 06:51:04 +0000 Subject: [PATCH 4/4] fix: panic if new rotation file cannot be created --- internal/disk/filerotate/filerotate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/disk/filerotate/filerotate.go b/internal/disk/filerotate/filerotate.go index 0b7a887..03ee501 100644 --- a/internal/disk/filerotate/filerotate.go +++ b/internal/disk/filerotate/filerotate.go @@ -85,7 +85,7 @@ func (rotatableFile *RotatableFile) RotateFile() { newFile, _, nfErr := newFile(rotatableFile.Directory, rotatableFile.rotateType) if nfErr != nil { - log.Println("Error while creation new file for rotation: ", nfErr) + log.Panicln("Error while creation new file for rotation: ", nfErr) } rotatableFile.Mutex.Lock()