sampler.go 3.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134
  1. // Copyright (c) 2016 Uber Technologies, Inc.
  2. //
  3. // Permission is hereby granted, free of charge, to any person obtaining a copy
  4. // of this software and associated documentation files (the "Software"), to deal
  5. // in the Software without restriction, including without limitation the rights
  6. // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  7. // copies of the Software, and to permit persons to whom the Software is
  8. // furnished to do so, subject to the following conditions:
  9. //
  10. // The above copyright notice and this permission notice shall be included in
  11. // all copies or substantial portions of the Software.
  12. //
  13. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14. // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15. // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  16. // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  17. // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  18. // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  19. // THE SOFTWARE.
  20. package zapcore
  21. import (
  22. "time"
  23. "go.uber.org/atomic"
  24. )
  25. const (
  26. _numLevels = _maxLevel - _minLevel + 1
  27. _countersPerLevel = 4096
  28. )
  29. type counter struct {
  30. resetAt atomic.Int64
  31. counter atomic.Uint64
  32. }
  33. type counters [_numLevels][_countersPerLevel]counter
  34. func newCounters() *counters {
  35. return &counters{}
  36. }
  37. func (cs *counters) get(lvl Level, key string) *counter {
  38. i := lvl - _minLevel
  39. j := fnv32a(key) % _countersPerLevel
  40. return &cs[i][j]
  41. }
  42. // fnv32a, adapted from "hash/fnv", but without a []byte(string) alloc
  43. func fnv32a(s string) uint32 {
  44. const (
  45. offset32 = 2166136261
  46. prime32 = 16777619
  47. )
  48. hash := uint32(offset32)
  49. for i := 0; i < len(s); i++ {
  50. hash ^= uint32(s[i])
  51. hash *= prime32
  52. }
  53. return hash
  54. }
  55. func (c *counter) IncCheckReset(t time.Time, tick time.Duration) uint64 {
  56. tn := t.UnixNano()
  57. resetAfter := c.resetAt.Load()
  58. if resetAfter > tn {
  59. return c.counter.Inc()
  60. }
  61. c.counter.Store(1)
  62. newResetAfter := tn + tick.Nanoseconds()
  63. if !c.resetAt.CAS(resetAfter, newResetAfter) {
  64. // We raced with another goroutine trying to reset, and it also reset
  65. // the counter to 1, so we need to reincrement the counter.
  66. return c.counter.Inc()
  67. }
  68. return 1
  69. }
  70. type sampler struct {
  71. Core
  72. counts *counters
  73. tick time.Duration
  74. first, thereafter uint64
  75. }
  76. // NewSampler creates a Core that samples incoming entries, which caps the CPU
  77. // and I/O load of logging while attempting to preserve a representative subset
  78. // of your logs.
  79. //
  80. // Zap samples by logging the first N entries with a given level and message
  81. // each tick. If more Entries with the same level and message are seen during
  82. // the same interval, every Mth message is logged and the rest are dropped.
  83. //
  84. // Keep in mind that zap's sampling implementation is optimized for speed over
  85. // absolute precision; under load, each tick may be slightly over- or
  86. // under-sampled.
  87. func NewSampler(core Core, tick time.Duration, first, thereafter int) Core {
  88. return &sampler{
  89. Core: core,
  90. tick: tick,
  91. counts: newCounters(),
  92. first: uint64(first),
  93. thereafter: uint64(thereafter),
  94. }
  95. }
  96. func (s *sampler) With(fields []Field) Core {
  97. return &sampler{
  98. Core: s.Core.With(fields),
  99. tick: s.tick,
  100. counts: s.counts,
  101. first: s.first,
  102. thereafter: s.thereafter,
  103. }
  104. }
  105. func (s *sampler) Check(ent Entry, ce *CheckedEntry) *CheckedEntry {
  106. if !s.Enabled(ent.Level) {
  107. return ce
  108. }
  109. counter := s.counts.get(ent.Level, ent.Message)
  110. n := counter.IncCheckReset(ent.Time, s.tick)
  111. if n > s.first && (n-s.first)%s.thereafter != 0 {
  112. return ce
  113. }
  114. return s.Core.Check(ent, ce)
  115. }