2.5.2

CHANGELOG.md

@@ -3,6 +3,14 @@
 The release notes on GitHub are taken from this file: the section whose
 heading is the version number.
 
+## 2.5.2
+
+- The sponsor banner is in the sidebar, above the language and theme; the strip at the top of the page is gone.
+- The version is in the page footer; the collection status is at the foot of the overview.
+- Rooms and racks is called Device locations and is a tab of the inventory page.
+- Deleting one interface's data is removed.
+- Deleting data no longer leaves the last minute, or summaries still being built, to be written back afterwards.
+
 ## 2.5.1
 
 - Long ranges on large installations no longer run out of database memory, and a device or an interface over days is read from summaries of its own (built once in the background for the data already stored).

cmd/traffic66/main.go

@@ -55,7 +55,7 @@ var version = ""
 // CHANGELOG.md. TestDevVersionMatchesChangelog keeps the two in step: the
 // default used to be a hardcoded 0.1.0-dev, and because nothing checked it
 // it stayed that way for two major versions.
-const devVersion = "2.5.1"
+const devVersion = "2.5.2"
 
 func init() {
 	if version == "" {

docs/MANUAL.md

@@ -496,21 +496,14 @@ The data directory holds `raw/` (detail, one file per hour),
 `traffic66.duckdb` (summaries and counters), `password`, `inventory.txt`,
 `license.json`, your logo and databases. Back up by stopping traffic66 and
 copying it; upgrade by replacing the program file. **Data cleanup** deletes
-data older than 7–120 days, or all of it. A registered copy can also delete
-one interface's data on its own — the flow records that went in or out of it
-and its counter samples — for when a link is decommissioned or its data must
-not be kept. A flow record names both the interface it came in on and the
-one it went out by, so those flows leave the other interface's view too;
-they are the same records. The hourly and daily summaries record traffic by
-application, host and dimension, never by interface, so they keep their
-totals.
-
-### Rooms and racks
-
-A picture of where the flows come from: for every device that sends flows,
+data older than 7–120 days, or all of it.
+
+### Device locations
+
+A tab of the inventory page. A picture of where the flows come from: for every device that sends flows,
 the room and rack it stands in, and for each of its ports the carrier and
 circuit ID it goes out to. Devices and ports appear by themselves from the
-flows; click ✎ to type in the room, rack, carrier and circuit. It is filled
+flows; click Edit to type in the device's room and rack and every port's carrier and circuit in one form. It is filled
 in once and kept in `inventory.txt` beside the names
 (`device 192.0.2.1 Core router room=Room_1 rack=A03`,
 `iface 192.0.2.1 3 Uplink isp=China_Telecom circuit=SH-12345`; a space is

internal/api/cleanup.go

@@ -53,6 +53,9 @@ func (s *Server) postCleanup(w http.ResponseWriter, r *http.Request) {
 		fail(w, errors.New("days must be 0 or more"))
 		return
 	}
+	if s.Pipe != nil {
+		s.Pipe.Forget(before)
+	}
 	disk := s.Store.Usage().DiskBytes
 	done, err := s.Store.PurgeBefore(before)
 	if err != nil {

internal/pipeline/pipeline.go

@@ -9,6 +9,7 @@ import (
 	"log"
 	"math"
 	"net/netip"
+	"slices"
 	"strconv"
 	"sync"
 	"sync/atomic"
@@ -218,6 +219,32 @@ func (p *Pipeline) FlushRows() {
 	p.Rows.Add(uint64(len(b)))
 }
 
+// Forget drops what is held in memory from before t (everything for a
+// zero t): the rows not written yet and the summaries of the minute being
+// added up. Deleting data calls it first, or up to a minute of what was
+// deleted would be written back a moment later.
+func (p *Pipeline) Forget(t time.Time) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	old := func(ts time.Time) bool { return t.IsZero() || ts.Before(t) }
+	p.batch = slices.DeleteFunc(p.batch, func(r store.Row) bool { return old(r.TS) })
+	for k := range p.rTS {
+		if old(k.TS) {
+			delete(p.rTS, k)
+		}
+	}
+	for k := range p.rHost {
+		if old(k.TS) {
+			delete(p.rHost, k)
+		}
+	}
+	for k := range p.rDim {
+		if old(k.TS) {
+			delete(p.rDim, k)
+		}
+	}
+}
+
 // FlushRollups writes accumulated rollup rows.
 func (p *Pipeline) FlushRollups() {
 	p.mu.Lock()

internal/pipeline/pipeline_test.go

@@ -608,3 +608,41 @@ func TestRingsAndSegmentCountries(t *testing.T) {
 		t.Error("three-letter country accepted")
 	}
 }
+
+// Deleting data drops what is still held in memory from before the cut,
+// so it is not written back a moment after the delete.
+func TestForget(t *testing.T) {
+	st, err := store.Open(store.Options{Dir: t.TempDir()})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer st.Close()
+	p := New(Config{}, st, enrich.NewInventory(), enrich.NewASNDB(), enrich.NewThreats())
+	now := time.Now().UTC().Truncate(time.Minute)
+	rec := func(at time.Time) flow.Record {
+		return flow.Record{Start: at, End: at, Src: netip.MustParseAddr("10.1.1.1"), Dst: netip.MustParseAddr("198.51.100.9"),
+			SrcPort: 50000, DstPort: 443, Proto: 6, Bytes: 1000, Packets: 1, Mult: 1, SamplingKnown: true,
+			Exporter: netip.MustParseAddr("10.0.0.1"), Source: flow.SrcIPFIX}
+	}
+	p.Ingest([]flow.Record{rec(now.Add(-3 * time.Hour)), rec(now.Add(-time.Minute))})
+	p.Forget(now.Add(-time.Hour))
+	p.FlushRows()
+	p.FlushRollups()
+	for _, q := range []store.Query{
+		{From: now.Add(-4 * time.Hour), To: now.Add(-2 * time.Hour)},                                                              // the records
+		{From: now.Add(-12 * time.Hour), To: now.Add(-2 * time.Hour).Truncate(time.Hour)},                                         // the summaries
+		{From: now.Add(-12 * time.Hour), To: now.Add(time.Minute), Filters: []store.Filter{{Field: "client", Value: "10.1.1.1"}}}, // what is left
+	} {
+		tot, err := st.Totals(q)
+		if err != nil {
+			t.Fatal(err)
+		}
+		want := uint64(0)
+		if len(q.Filters) > 0 {
+			want = 1 // the recent flow stays
+		}
+		if tot.Flows != want {
+			t.Errorf("%v..%v: %d flows, want %d", q.From, q.To, tot.Flows, want)
+		}
+	}
+}

internal/store/purge.go

@@ -72,6 +72,20 @@ func (s *Store) PurgeSizeBefore(before time.Time) PurgeSize {
 func (s *Store) PurgeBefore(before time.Time) (PurgeSize, error) {
 	all := before.IsZero()
 	var done PurgeSize
+	// A build of the summaries still running would write back what is
+	// deleted here. It is stopped first and, where something is left to
+	// summarise, started again afterwards; deleting everything leaves
+	// nothing to summarise.
+	building := s.stopViews()
+	defer func() {
+		switch {
+		case all:
+			s.SetMeta(viewsDone, time.Now().UTC().Format(time.RFC3339))
+			s.viewsReady.Store(true)
+		case building:
+			s.startViews()
+		}
+	}()
 	s.segMu.RLock()
 	var old []segment
 	for _, g := range s.segCache {

internal/store/store.go

@@ -118,8 +118,8 @@ type Store struct {
 	viewsReady atomic.Bool
 	closing    atomic.Bool
 	bg         sync.WaitGroup
-	bgCtx      context.Context
-	bgStop     context.CancelFunc
+	vmu        sync.Mutex
+	vStop      context.CancelFunc // stops the build in progress, nil when none
 
 	dsMu sync.Mutex // guards the cached start of the data
 	dsAt time.Time  // when dsTS was read
@@ -192,7 +192,6 @@ func Open(opt Options) (*Store, error) {
 		return nil, fmt.Errorf("cannot open the database in %s: %w", opt.Dir, err)
 	}
 	s := &Store{opt: opt, conn: c, DB: sql.OpenDB(c)}
-	s.bgCtx, s.bgStop = context.WithCancel(context.Background())
 	for _, q := range append(schema, viewSchema...) {
 		if _, err := s.DB.Exec(q); err != nil {
 			return nil, fmt.Errorf("schema: %w", err)
@@ -222,8 +221,7 @@ func Open(opt Options) (*Store, error) {
 		}
 		s.viewsReady.Store(true)
 	default:
-		s.bg.Add(1)
-		go s.buildViews()
+		s.startViews()
 	}
 	log.Printf("store: %s, DuckDB memory limit %d MB, %d threads", opt.Dir, memMB, opt.Threads)
 	return s, nil
@@ -543,8 +541,7 @@ func (s *Store) Usage() Usage {
 // Close flushes and closes everything.
 func (s *Store) Close() error {
 	s.closing.Store(true)
-	s.bgStop()
-	s.bg.Wait()
+	s.stopViews()
 	s.wmu.Lock()
 	defer s.wmu.Unlock()
 	for _, a := range []*duckdb.Appender{s.appHot, s.appTS, s.appHost, s.appDim, s.appIfc} {

internal/store/views.go

@@ -1,6 +1,7 @@
 package store
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
 	"log"
@@ -245,8 +246,13 @@ func sealViews(tx *sql.Tx, boundary time.Time) error {
 // existed, once per data directory, a file at a time so the memory it
 // needs is a file's worth. Until it is done the pages read those ranges
 // from the flow records as before.
-func (s *Store) buildViews() {
-	defer s.bg.Done()
+func (s *Store) buildViews(ctx context.Context) {
+	defer func() {
+		s.vmu.Lock()
+		s.vStop = nil
+		s.vmu.Unlock()
+		s.bg.Done()
+	}()
 	// Start over: an earlier run may have stopped half way. The files
 	// sealed from here on are summarised by the seal itself, so the list
 	// is taken, and the tables emptied, with sealing held off.
@@ -264,23 +270,23 @@ func (s *Store) buildViews() {
 	s.wmu.Unlock()
 	t0 := time.Now()
 	for i, g := range segs {
-		if s.closing.Load() {
+		if ctx.Err() != nil {
 			return
 		}
 		src := "(SELECT * FROM read_parquet('" + sqlPath(g.path) + "'))"
-		tx, err := s.DB.BeginTx(s.bgCtx, nil)
+		tx, err := s.DB.BeginTx(ctx, nil)
 		if err != nil {
 			log.Printf("store: summaries: %v", err)
 			return
 		}
 		for _, q := range viewInserts(src) {
-			if _, err = tx.ExecContext(s.bgCtx, q); err != nil {
+			if _, err = tx.ExecContext(ctx, q); err != nil {
 				break
 			}
 		}
 		if err != nil {
 			tx.Rollback()
-			if !s.closing.Load() {
+			if ctx.Err() == nil {
 				log.Printf("store: summaries of %s: %v", g.path, err)
 			}
 			return
@@ -302,3 +308,27 @@ func (s *Store) buildViews() {
 		log.Printf("store: summarised %d flow files by device and interface in %s", len(segs), time.Since(t0).Round(time.Second))
 	}
 }
+
+// startViews builds the summaries of the flow files in the background.
+func (s *Store) startViews() {
+	ctx, stop := context.WithCancel(context.Background())
+	s.vmu.Lock()
+	s.vStop = stop
+	s.vmu.Unlock()
+	s.bg.Add(1)
+	go s.buildViews(ctx)
+}
+
+// stopViews stops a build in progress, waits for it, and reports whether
+// there was one.
+func (s *Store) stopViews() bool {
+	s.vmu.Lock()
+	stop := s.vStop
+	s.vmu.Unlock()
+	if stop == nil {
+		return false
+	}
+	stop()
+	s.bg.Wait()
+	return true
+}

internal/web/static/app.css

@@ -54,7 +54,7 @@ button,input,select,textarea{font:inherit;color:inherit}
 [hidden]{display:none!important}
 
 .app{display:grid;grid-template-columns:236px 1fr;min-height:100vh}
-aside{background:var(--side);border-inline-end:1px solid var(--side-line);display:flex;flex-direction:column;position:sticky;top:0;height:calc(100vh - var(--sp-h, 0px))}
+aside{background:var(--side);border-inline-end:1px solid var(--side-line);display:flex;flex-direction:column;position:sticky;top:0;height:100vh}
 .brand{padding:12px 14px 10px;flex-wrap:wrap;position:relative;font-weight:700;font-size:18px;letter-spacing:-.01em;border-bottom:1px solid var(--side-line);display:flex;align-items:center;gap:8px}
 /* the logo fills the sidebar's width (236px less the brand padding = 208px)
    and keeps its aspect ratio; 72px tall at most. Recommended upload:
@@ -75,28 +75,26 @@ nav.dash button[aria-current="page"]::before{content:"";position:absolute;inset-
 @keyframes navbar{from{transform:scaleY(0)}to{transform:scaleY(1)}}
 .ni{width:18px;height:18px;flex:none;fill:none;stroke:currentColor;stroke-width:1.8;stroke-linecap:round;stroke-linejoin:round;opacity:.85}
 nav.dash button[aria-current="page"] .ni{opacity:1}
-.ver{display:flex;justify-content:space-between;gap:8px;padding:6px 16px 8px;font-size:11.5px;color:var(--ink-3);border-bottom:1px solid var(--side-line);font-variant-numeric:tabular-nums}
+/* the version and the clock, in the page footer under the licence line */
+.ver{display:flex;justify-content:center;gap:14px;margin-top:4px;font-size:11.5px;color:var(--ink-3);font-variant-numeric:tabular-nums}
 .badge{min-width:18px;padding:0 5px;border-radius:9px;background:var(--warn);color:#1b2432;font-size:11px;line-height:18px;text-align:center;font-weight:700}
-.self{border-top:1px solid var(--side-line);padding:12px 18px;font-size:12.5px;color:var(--ink-2);display:grid;grid-template-columns:auto 1fr;gap:3px 10px}
-.self .k{color:var(--ink-3)}.self .val{text-align:end;font-weight:600;color:var(--ink)}
-.self .live{grid-column:1/-1;display:flex;align-items:center;gap:7px;font-weight:600;margin-bottom:4px}
+/* how collection is going, at the foot of the overview */
+.self{display:flex;flex-wrap:wrap;align-items:center;gap:10px 28px;font-size:13px;color:var(--ink-2)}
+.self .it{display:flex;flex-direction:column;gap:1px}
+.self .k{color:var(--ink-3);font-size:12px}.self .val{font-weight:600;color:var(--ink);font-size:15px}
+.self .live{display:flex;align-items:center;gap:7px;font-weight:600}
 .self .live i{width:8px;height:8px;border-radius:50%;background:currentColor}
 .self .live.ok{color:var(--good)}.self .live.idle{color:var(--ink-3)}
 .foot{display:flex;flex-direction:column;gap:8px;padding:0 16px 14px}
-/* The sponsor strip of an unregistered installation: the full width of the
-   browser at the top of the page, above the sidebar and the content, and it
-   scrolls away with the page rather than taking a band of every screen.
-
-   One advertiser at a time: a long banner alone, a small 728x90 one three
-   times side by side (sponsor.js decides, and sets the row height). The
-   sidebar's own height is measured against the strip (--sp-h) instead of
-   a fixed 100vh. */
-.sponsor{grid-column:1/-1;background:var(--side);border-bottom:1px solid var(--side-line)}
+/* The sponsor slot of an unregistered installation: in the sidebar, above
+   the language and theme, one small banner at a time at the sidebar's
+   width (sponsor.js decides, and sets the height). */
+.sponsor{border-top:1px solid var(--side-line);padding:8px 12px 4px}
 .sponsor[hidden]{display:none}
-.sponsor .sp-h{font-size:10.5px;letter-spacing:.06em;text-transform:uppercase;color:var(--ink-3);padding:3px 12px 2px}
-.sponsor .sp-text{margin:0 12px 8px;border:1px dashed var(--line-2);border-radius:6px;padding:6px 8px;font-size:11.5px;line-height:1.35;color:var(--ink-3);overflow-wrap:anywhere}
+.sponsor .sp-h{font-size:10.5px;letter-spacing:.06em;text-transform:uppercase;color:var(--ink-3);padding:0 0 3px}
+.sponsor .sp-text{margin:0 0 6px;border:1px dashed var(--line-2);border-radius:6px;padding:6px 8px;font-size:11.5px;line-height:1.35;color:var(--ink-3);overflow-wrap:anywhere}
 .sponsor .sp-text[hidden],.spc[hidden]{display:none}
-.spc-strip{display:grid;grid-auto-flow:column;grid-auto-columns:minmax(0,1fr);gap:1px;background:var(--side-line)}
+.spc-strip{display:grid;border-radius:4px;overflow:hidden}
 .spc-cell{position:relative;overflow:hidden;background:var(--surface)}
 .spc-cell>a,.spc-cell>div{display:block;width:100%;height:100%}
 .spc-cell img{width:100%;height:100%;object-fit:cover;display:block}
@@ -151,7 +149,7 @@ main{padding:16px 20px 32px;min-width:0}
 .ph .sub{font-size:12.5px;color:var(--ink-3)}
 .c12{grid-column:span 12}.c8{grid-column:span 8}.c6{grid-column:span 6}.c4{grid-column:span 4}
 @media (max-width:1180px){.c8,.c4{grid-column:span 12}}
-@media (max-width:820px){.app{grid-template-columns:minmax(0,1fr)}.main{min-width:0}aside{position:static;height:auto}nav.dash{display:flex;overflow-x:auto;padding:6px}nav.dash hr{display:none}nav.dash button{white-space:nowrap;width:auto}.ver{display:none}.self{display:none}.c6{grid-column:span 12}main{padding:12px}.row1,.row2{padding-inline:12px}.range{max-width:100%;overflow-x:auto;margin-inline-start:0}}
+@media (max-width:820px){.app{grid-template-columns:minmax(0,1fr)}.main{min-width:0}aside{position:static;height:auto}nav.dash{display:flex;overflow-x:auto;padding:6px}nav.dash hr{display:none}nav.dash button{white-space:nowrap;width:auto}.c6{grid-column:span 12}main{padding:12px}.row1,.row2{padding-inline:12px}.range{max-width:100%;overflow-x:auto;margin-inline-start:0}}
 
 .kpis{display:grid;grid-template-columns:repeat(5,1fr)}
 .kpi{padding:4px 18px;border-inline-start:1px solid var(--line);min-width:0}
@@ -484,17 +482,6 @@ dialog.aedit h2{margin:0 0 14px;font-size:16px}
 .acctinline input{min-width:0;flex:1 1 140px}
 body.mustchange #nav,body.mustchange #acct,body.mustchange .ifsel{display:none}
 
-/* Licence badge: a block under the logo and above the version, rather
-   than a pill in a toolbar whose contents change from view to view. */
-.licbadge{display:block;margin:8px 14px;text-align:center;font-size:12.5px;font-weight:600;text-decoration:none;
-  padding:3px 9px;border-radius:999px;border:1px solid var(--line);color:var(--muted);
-  background:var(--surface);white-space:nowrap;overflow:hidden;text-overflow:ellipsis}
-/* who the licence is made out to, under the word: a long company name is
-   cut rather than widening the sidebar */
-.licbadge .who{display:block;font-weight:500;font-size:11.5px;opacity:.85;overflow:hidden;text-overflow:ellipsis}
-.licbadge:hover{border-color:var(--accent);color:var(--accent)}
-.licbadge.unregistered,.licbadge.expired{border-color:var(--warn,#b9770e);color:var(--warn,#b9770e)}
-
 /* The evaluation mark: drawn over the view, light enough that nothing
    underneath becomes unreadable, and kept when the page is printed. */
 body.marked .view.on{position:relative}
@@ -587,3 +574,6 @@ body.marked .view.on::after{
 dialog.embdlg{width:min(620px,96vw)}
 .embdlg textarea{font:inherit;font-size:13px;font-family:ui-monospace,monospace;padding:5px 7px;border:1px solid var(--line-2);border-radius:6px;background:var(--surface);color:var(--ink);resize:vertical}
 .afhint{font-size:11.5px;color:var(--ink-3)}
+
+/* the tabs of the inventory page: the inventory and device locations */
+.invtabs{display:inline-flex;margin:0 0 14px}