123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199 |
- // Copyright (C) 2023 The Syncthing Authors.
- //
- // This Source Code Form is subject to the terms of the Mozilla Public
- // License, v. 2.0. If a copy of the MPL was not distributed with this file,
- // You can obtain one at https://mozilla.org/MPL/2.0/.
- package main
- import (
- "bytes"
- "compress/gzip"
- "context"
- "io"
- "log"
- "math"
- "os"
- "path/filepath"
- "sort"
- "time"
- )
- type diskStore struct {
- dir string
- inbox chan diskEntry
- maxBytes int64
- maxFiles int
- currentFiles []currentFile
- currentSize int64
- }
- type diskEntry struct {
- path string
- data []byte
- }
- type currentFile struct {
- path string
- size int64
- mtime int64
- }
- func (d *diskStore) Serve(ctx context.Context) {
- if err := os.MkdirAll(d.dir, 0o700); err != nil {
- log.Println("Creating directory:", err)
- return
- }
- if err := d.inventory(); err != nil {
- log.Println("Failed to inventory disk store:", err)
- }
- d.clean()
- cleanTimer := time.NewTicker(time.Minute)
- inventoryTimer := time.NewTicker(24 * time.Hour)
- buf := new(bytes.Buffer)
- gw := gzip.NewWriter(buf)
- for {
- select {
- case entry := <-d.inbox:
- path := d.fullPath(entry.path)
- if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
- log.Println("Creating directory:", err)
- continue
- }
- buf.Reset()
- gw.Reset(buf)
- if _, err := gw.Write(entry.data); err != nil {
- log.Println("Failed to compress crash report:", err)
- continue
- }
- if err := gw.Close(); err != nil {
- log.Println("Failed to compress crash report:", err)
- continue
- }
- if err := os.WriteFile(path, buf.Bytes(), 0o600); err != nil {
- log.Printf("Failed to write %s: %v", entry.path, err)
- _ = os.Remove(path)
- continue
- }
- d.currentSize += int64(buf.Len())
- d.currentFiles = append(d.currentFiles, currentFile{
- size: int64(len(entry.data)),
- path: path,
- })
- case <-cleanTimer.C:
- d.clean()
- case <-inventoryTimer.C:
- if err := d.inventory(); err != nil {
- log.Println("Failed to inventory disk store:", err)
- }
- case <-ctx.Done():
- return
- }
- }
- }
- func (d *diskStore) Put(path string, data []byte) bool {
- select {
- case d.inbox <- diskEntry{
- path: path,
- data: data,
- }:
- return true
- default:
- return false
- }
- }
- func (d *diskStore) Get(path string) ([]byte, error) {
- path = d.fullPath(path)
- bs, err := os.ReadFile(path)
- if err != nil {
- return nil, err
- }
- gr, err := gzip.NewReader(bytes.NewReader(bs))
- if err != nil {
- return nil, err
- }
- defer gr.Close()
- return io.ReadAll(gr)
- }
- func (d *diskStore) Exists(path string) bool {
- path = d.fullPath(path)
- _, err := os.Lstat(path)
- return err == nil
- }
- func (d *diskStore) clean() {
- for len(d.currentFiles) > 0 && (len(d.currentFiles) > d.maxFiles || d.currentSize > d.maxBytes) {
- f := d.currentFiles[0]
- log.Println("Removing", f.path)
- if err := os.Remove(f.path); err != nil {
- log.Println("Failed to remove file:", err)
- }
- d.currentFiles = d.currentFiles[1:]
- d.currentSize -= f.size
- }
- var oldest time.Duration
- if len(d.currentFiles) > 0 {
- oldest = time.Since(time.Unix(d.currentFiles[0].mtime, 0)).Truncate(time.Minute)
- }
- metricDiskstoreFilesTotal.Set(float64(len(d.currentFiles)))
- metricDiskstoreBytesTotal.Set(float64(d.currentSize))
- metricDiskstoreOldestAgeSeconds.Set(math.Round(oldest.Seconds()))
- log.Printf("Clean complete: %d files, %d MB, oldest is %v ago", len(d.currentFiles), d.currentSize>>20, oldest)
- }
- func (d *diskStore) inventory() error {
- d.currentFiles = nil
- d.currentSize = 0
- err := filepath.Walk(d.dir, func(path string, info os.FileInfo, err error) error {
- if err != nil {
- return err
- }
- if info.IsDir() {
- return nil
- }
- if filepath.Ext(path) != ".gz" {
- return nil
- }
- d.currentSize += info.Size()
- d.currentFiles = append(d.currentFiles, currentFile{
- path: path,
- size: info.Size(),
- mtime: info.ModTime().Unix(),
- })
- return nil
- })
- sort.Slice(d.currentFiles, func(i, j int) bool {
- return d.currentFiles[i].mtime < d.currentFiles[j].mtime
- })
- var oldest time.Duration
- if len(d.currentFiles) > 0 {
- oldest = time.Since(time.Unix(d.currentFiles[0].mtime, 0)).Truncate(time.Minute)
- }
- metricDiskstoreFilesTotal.Set(float64(len(d.currentFiles)))
- metricDiskstoreBytesTotal.Set(float64(d.currentSize))
- metricDiskstoreOldestAgeSeconds.Set(math.Round(oldest.Seconds()))
- log.Printf("Inventory complete: %d files, %d MB, oldest is %v ago", len(d.currentFiles), d.currentSize>>20, oldest)
- return err
- }
- func (d *diskStore) fullPath(path string) string {
- return filepath.Join(d.dir, path[0:2], path[2:]) + ".gz"
- }
|