2.5.0

.github/workflows/ci.yml

@@ -41,11 +41,27 @@ jobs:
     container: ${{ matrix.container || '' }}
     env:
       GOFLAGS: -buildvcs=false
+      PRO_TOKEN: ${{ secrets.PRO_TOKEN }}
+      PRO_SSH_KEY: ${{ secrets.PRO_SSH_KEY }}
     defaults:
       run:
         shell: bash
     steps:
       - uses: actions/checkout@v5
+      - uses: actions/checkout@v5
+        if: ${{ env.PRO_TOKEN != '' || env.PRO_SSH_KEY != '' }}
+        with:
+          repository: githubflyideas/traffic66-pro
+          token: ${{ secrets.PRO_TOKEN || github.token }}
+          ssh-key: ${{ secrets.PRO_SSH_KEY }}
+          path: .pro
+      - name: Registered features
+        if: ${{ env.PRO_TOKEN != '' || env.PRO_SSH_KEY != '' }}
+        run: |
+          cp -R .pro/overlay/tree/. .
+          rm -rf .pro
+          echo "T66_PRO=1" >> "$GITHUB_ENV"
+          echo "GOFLAGS=-buildvcs=false -tags=pro" >> "$GITHUB_ENV"
       - uses: actions/setup-go@v6
         with:
           go-version: "1.26.9"
@@ -96,6 +112,7 @@ jobs:
             # a tag push and a one-click release (a version entered by hand)
             # are both releases; only the builds of every push stay community
             kind=community; [[ "$GITHUB_REF" == refs/tags/* || -n "${{ inputs.version }}" ]] && kind=official
+            if [[ $kind == official && -z "$T66_PRO" ]]; then echo "::error::a release needs the PRO_SSH_KEY or PRO_TOKEN secret"; exit 1; fi
             ldflags="-s -w -X main.version=${ver} -X main.buildKind=${kind}"
             tags=""
             export CGO_ENABLED=1
@@ -114,6 +131,7 @@ jobs:
               macOS)
                 export MACOSX_DEPLOYMENT_TARGET="${MACOSX_DEPLOYMENT_TARGET:-11.0}" ;;
             esac
+            [[ -n "$T66_PRO" ]] && tags="${tags:+$tags,}pro"
             go build -trimpath ${tags:+-tags "$tags"} -ldflags "$ldflags" -o "$out" ./cmd/traffic66
             cp README.md "dist/traffic66-${{ matrix.target }}/"
             cp -r docs "dist/traffic66-${{ matrix.target }}/"

CHANGELOG.md

@@ -3,6 +3,15 @@
 The release notes on GitHub are taken from this file: the section whose
 heading is the version number.
 
+## 2.5.0
+
+- Overview: inbound and outbound follow the interface charted above.
+- Rooms and racks: new look; a device and its ports are edited in one form.
+- Filter chips only on the traffic pages.
+- Black is the default theme.
+- Syslog for findings can be set from the findings page.
+- Links that show one address's bandwidth on another platform (registered installations).
+
 ## 2.4.1
 
 - Overview: Inbound and Outbound now mean which way the bytes went — what arrived at your hosts, what they sent — instead of who opened the connection. On a network whose hosts only connect outwards, Inbound used to be empty (and said the data was still arriving), although most of its traffic is the replies it downloads.

cmd/traffic66/main.go

@@ -40,7 +40,6 @@ import (
 	"github.com/githubflyideas/traffic66/internal/sim"
 	"github.com/githubflyideas/traffic66/internal/snmp"
 	"github.com/githubflyideas/traffic66/internal/store"
-	"github.com/githubflyideas/traffic66/internal/syslogout"
 	"github.com/githubflyideas/traffic66/internal/tlscert"
 	"github.com/githubflyideas/traffic66/internal/tui"
 	"github.com/githubflyideas/traffic66/internal/web"
@@ -56,7 +55,7 @@ var version = ""
 // CHANGELOG.md. TestDevVersionMatchesChangelog keeps the two in step: the
 // default used to be a hardcoded 0.1.0-dev, and because nothing checked it
 // it stayed that way for two major versions.
-const devVersion = "2.4.1"
+const devVersion = "2.5.0"
 
 func init() {
 	if version == "" {
@@ -262,9 +261,19 @@ type serveFlags struct {
 	l2                                                 int
 	hold                                               time.Duration
 	tuiAfter                                           bool
-	syslog, syslogNet, syslogFmt, syslogCA, syslogBase string
-	syslogSev                                          int
-	syslogASCII                                        bool
+}
+
+// Parts compiled in separately add their flags and start with the
+// collector, once the licence and the detector exist.
+var (
+	extFlags []func(fs *flag.FlagSet)
+	extServe []func(x serveExt) (stop func())
+)
+
+type serveExt struct {
+	data string
+	lic  *license.Checker
+	det  *detect.Detector
 }
 
 func serve(args []string, demo bool) {
@@ -280,13 +289,9 @@ func serve(args []string, demo bool) {
 	fs.BoolVar(&f.tls, "tls", false, "serve the web UI over HTTPS with a certificate traffic66 keeps in the data directory (registered installations)")
 	fs.StringVar(&f.tlsCert, "tls-cert", "", "serve the web UI over HTTPS with this certificate (registered installations)")
 	fs.StringVar(&f.tlsKey, "tls-key", "", "the private key for -tls-cert")
-	fs.StringVar(&f.syslog, "syslog", "", "send findings to a syslog collector at host:port (registered installations)")
-	fs.StringVar(&f.syslogNet, "syslog-transport", "udp", "how to reach the collector: udp, tcp or tls")
-	fs.StringVar(&f.syslogFmt, "syslog-format", "cef", "message format: cef (ArcSight, QRadar, Splunk, Graylog, Elastic), leef (QRadar), json (Elastic Common Schema), text")
-	fs.IntVar(&f.syslogSev, "syslog-min-sev", 1, "least severity to send: 1 low, 2 medium, 3 high")
-	fs.BoolVar(&f.syslogASCII, "syslog-ascii", false, "escape everything outside ASCII, for collectors that mangle UTF-8")
-	fs.StringVar(&f.syslogCA, "syslog-ca", "", "certificate authority to trust for -syslog-transport tls")
-	fs.StringVar(&f.syslogBase, "syslog-url", "", "web address of this installation, to build the evidence link with")
+	for _, x := range extFlags {
+		x(fs)
+	}
 	fs.StringVar(&f.listen, "listen", "sflow=:6343,netflow=:2055,ipfix=:4739", "UDP collectors as name=addr, comma separated; every port accepts every protocol; empty disables")
 	fs.StringVar(&f.user, "user", "admin", "login user")
 	fs.StringVar(&f.password, "password", "", "login password for this run instead of the stored one (or set TRAFFIC66_PASSWORD)")
@@ -412,11 +417,11 @@ func serve(args []string, demo bool) {
 	}
 
 	det := detect.New(st, inv, detect.Config{})
-	if sender := startSyslog(f, lic); sender != nil {
-		defer sender.Close()
-		base := f.syslogBase
-		det.OnFinding = func(fi store.Finding) {
-			sender.Send(syslogout.FromFinding(fi, syslogout.PhaseStart, base))
+	if offline == nil {
+		for _, x := range extServe {
+			if stop := x(serveExt{data: f.data, lic: lic, det: det}); stop != nil {
+				defer stop()
+			}
 		}
 	}
 	if demo {
@@ -979,28 +984,3 @@ func setGoMemoryLimit(frac float64) {
 	debug.SetMemoryLimit(lim)
 	log.Printf("memory: database %d MB and program %d MB (soft), from -memory %.2f", lim>>20, lim>>20, frac)
 }
-
-// startSyslog prepares the syslog feed, or returns nil when there is
-// nothing to send to. Unlike HTTPS, a feed that cannot run is not a reason
-// to stop: findings keep being made and keep showing in the web UI, and
-// the one thing lost is that they also go somewhere else.
-func startSyslog(f serveFlags, lic *license.Checker) *syslogout.Sender {
-	if f.syslog == "" {
-		return nil
-	}
-	if lic == nil || !lic.State().Allowed(license.FeatSyslog) {
-		log.Printf("syslog: sending findings to a collector is for registered installations; findings are still made and shown in the web UI")
-		return nil
-	}
-	cfg := syslogout.Config{
-		Enabled: true, Address: f.syslog, Network: f.syslogNet,
-		Format: syslogout.Format(f.syslogFmt), MinSev: f.syslogSev,
-		ASCII: f.syslogASCII, CAFile: f.syslogCA, BaseURL: f.syslogBase,
-	}
-	if err := cfg.Check(); err != nil {
-		fatalf("%v", err)
-	}
-	syslogout.Version = version
-	log.Printf("syslog: sending findings to %s over %s as %s", cfg.Address, cfg.Network, cfg.Format)
-	return syslogout.New(cfg)
-}

internal/api/cleanup.go

@@ -4,11 +4,8 @@ import (
 	"encoding/json"
 	"errors"
 	"net/http"
-	"net/netip"
-	"strconv"
 	"time"
 
-	"github.com/githubflyideas/traffic66/internal/license"
 	"github.com/githubflyideas/traffic66/internal/store"
 )
 
@@ -78,70 +75,3 @@ func (s *Server) allowed(w http.ResponseWriter, feature string) bool {
 	writeJSON(w, http.StatusForbidden, map[string]string{"error": "not_registered", "feature": feature})
 	return false
 }
-
-// ifaceTarget reads the interface and age a per-interface cleanup names.
-// days 0 means the interface's whole history.
-func ifaceTarget(exporter, ifindex, days string) (string, uint32, time.Time, error) {
-	var before time.Time
-	if _, err := netip.ParseAddr(exporter); err != nil {
-		return "", 0, before, errors.New("exporter must be an address")
-	}
-	idx, err := strconv.ParseUint(ifindex, 10, 32)
-	if err != nil || idx == 0 {
-		return "", 0, before, errors.New("ifindex missing")
-	}
-	if days != "" {
-		d, err := strconv.Atoi(days)
-		if err != nil || d < 0 {
-			return "", 0, before, errors.New("days must be 0 or more")
-		}
-		if d > 0 {
-			before = time.Now().AddDate(0, 0, -d)
-		}
-	}
-	return exporter, uint32(idx), before, nil
-}
-
-// getCleanupIface shows what deleting one interface's data would remove.
-func (s *Server) getCleanupIface(w http.ResponseWriter, r *http.Request) {
-	if !s.allowed(w, license.FeatPurgeIface) {
-		return
-	}
-	q := r.URL.Query()
-	exp, idx, before, err := ifaceTarget(q.Get("exporter"), q.Get("ifindex"), q.Get("days"))
-	if err != nil {
-		fail(w, err)
-		return
-	}
-	writeJSON(w, http.StatusOK, s.Store.PurgeSizeIface(exp, idx, before))
-}
-
-// postCleanupIface deletes one interface's flow records and counters.
-func (s *Server) postCleanupIface(w http.ResponseWriter, r *http.Request) {
-	if !s.allowed(w, license.FeatPurgeIface) {
-		return
-	}
-	var in struct {
-		Exporter string `json:"exporter"`
-		IfIndex  uint32 `json:"ifindex"`
-		Days     int    `json:"days"`
-	}
-	if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1024)).Decode(&in); err != nil {
-		fail(w, errors.New("bad request"))
-		return
-	}
-	exp, idx, before, err := ifaceTarget(in.Exporter, strconv.FormatUint(uint64(in.IfIndex), 10), strconv.Itoa(in.Days))
-	if err != nil {
-		fail(w, err)
-		return
-	}
-	disk := s.Store.Usage().DiskBytes
-	done, err := s.Store.PurgeIface(exp, idx, before)
-	if err != nil {
-		fail(w, err)
-		return
-	}
-	// what the files lost, plus what the database file gave back
-	done.Bytes += max(0, disk-s.Store.Usage().DiskBytes)
-	writeJSON(w, http.StatusOK, done)
-}

internal/api/racks.go

@@ -131,8 +131,15 @@ func (s *Server) getRacks(w http.ResponseWriter, r *http.Request) {
 func (s *Server) putRackDevice(w http.ResponseWriter, r *http.Request) {
 	var in struct {
 		Exporter, Name, Room, Rack string
+		// the ports whose name, carrier or circuit changed, so a device
+		// and all its ports are filled in with one form
+		Ifaces []struct {
+			IfIndex      uint32
+			Name         string
+			ISP, Circuit string
+		}
 	}
-	if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4<<10)).Decode(&in); err != nil {
+	if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 512<<10)).Decode(&in); err != nil {
 		fail(w, errors.New("bad request"))
 		return
 	}
@@ -141,10 +148,26 @@ func (s *Server) putRackDevice(w http.ResponseWriter, r *http.Request) {
 		fail(w, errors.New("bad exporter address"))
 		return
 	}
+	if len(in.Ifaces) > 4096 {
+		fail(w, errors.New("too many interfaces"))
+		return
+	}
+	for _, f := range in.Ifaces {
+		if f.IfIndex == 0 {
+			fail(w, errors.New("bad interface"))
+			return
+		}
+	}
 	if err := s.Inv.SetDevice(a, clip(in.Name, 80), clip(in.Room, 60), clip(in.Rack, 60)); err != nil {
 		fail(w, err)
 		return
 	}
+	for _, f := range in.Ifaces {
+		if err := s.Inv.SetCircuit(a, f.IfIndex, clip(f.Name, 80), clip(f.ISP, 60), clip(f.Circuit, 80)); err != nil {
+			fail(w, err)
+			return
+		}
+	}
 	writeJSON(w, http.StatusOK, map[string]any{"ok": true})
 }
 

internal/api/racks_test.go

@@ -40,11 +40,13 @@ func TestRacks(t *testing.T) {
 		}
 		return w.Code
 	}
-	if c := put("device", `{"exporter":"10.0.0.1","name":"Core router","room":"Room 1","rack":"A03"}`); c != 200 {
+	// the device and its ports in one go, as the page's one form sends them
+	if c := put("device", `{"exporter":"10.0.0.1","name":"Core router","room":"Room 1","rack":"A03",
+		"ifaces":[{"ifindex":3,"name":"Uplink","isp":"China Telecom","circuit":"SH-12345"}]}`); c != 200 {
 		t.Fatal(c)
 	}
-	if c := put("iface", `{"exporter":"10.0.0.1","ifindex":3,"name":"Uplink","isp":"China Telecom","circuit":"SH-12345"}`); c != 200 {
-		t.Fatal(c)
+	if c := put("device", `{"exporter":"10.0.0.1","ifaces":[{"ifindex":0}]}`); c != 400 {
+		t.Fatalf("an interface without an index was accepted: %d", c)
 	}
 	if c := put("iface", `{"exporter":"not an address","ifindex":3}`); c != 400 {
 		t.Fatalf("a bad exporter was accepted: %d", c)

internal/api/server.go

@@ -29,6 +29,9 @@ import (
 	"github.com/githubflyideas/traffic66/internal/store"
 )
 
+// extRoutes add routes from parts that are compiled in separately.
+var extRoutes []func(s *Server, mux *http.ServeMux, api, adminAPI func(string, http.HandlerFunc))
+
 // Server wires the API together.
 type Server struct {
 	Store   *store.Store
@@ -171,8 +174,6 @@ func (s *Server) Handler() http.Handler {
 	api("POST /api/findings", s.data((*Server).setFindings))
 	api("GET /api/cleanup", s.getCleanup)
 	adminAPI("POST /api/cleanup", s.postCleanup)
-	api("GET /api/cleanup/iface", s.getCleanupIface)
-	adminAPI("POST /api/cleanup/iface", s.postCleanupIface)
 	api("GET /api/sandbox", s.getSandbox)
 	api("POST /api/sandbox/files", s.putSandboxFile)
 	adminAPI("DELETE /api/sandbox/files", s.deleteSandbox)
@@ -184,6 +185,9 @@ func (s *Server) Handler() http.Handler {
 	adminAPI("POST /api/geo", s.putGeo)
 	adminAPI("DELETE /api/geo", s.deleteGeo)
 	adminAPI("POST /api/geo/dbip", s.updateDBIP)
+	for _, f := range extRoutes {
+		f(s, mux, api, adminAPI)
+	}
 	static := http.FileServer(http.FS(s.Static))
 	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("X-Content-Type-Options", "nosniff")

internal/store/bytesdir_test.go

@@ -43,3 +43,41 @@ func TestBytesDirection(t *testing.T) {
 		t.Error("a bad byte direction was accepted")
 	}
 }
+
+// A data centre whose service addresses are public and not listed as its
+// own networks: every flow is transit by address, so only the interface
+// can say which way it went. In through the uplink is inbound, out
+// through it is outbound, the same split as the interface's own chart.
+func TestInterfaceDirection(t *testing.T) {
+	s, err := Open(Options{Dir: t.TempDir(), Threads: 1})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer s.Close()
+	ts := time.Now().UTC().Add(-10 * time.Minute)
+	user, vip := "198.51.100.7", "203.0.113.10"
+	base := Row{TS: ts, Client: user, Server: vip, Cli4: 0xC6336407, Srv4: 0xCB00710A, Dir: DirTransit, Pkts: 1, Flows: 1, Exporter: "10.0.0.1"}
+	req, resp := base, base
+	req.Src, req.Dst, req.Wire, req.InIf, req.OutIf = user, vip, 1000, 7, 3      // in through the uplink 7
+	resp.Src, resp.Dst, resp.Wire, resp.InIf, resp.OutIf = vip, user, 9000, 3, 7 // back out through it
+	resp.Dup = true                                                              // and seen by a second device too
+	if err := s.AppendRows([]Row{req, resp}); err != nil {
+		t.Fatal(err)
+	}
+	q := func(f ...Filter) uint64 {
+		tot, err := s.Totals(Query{From: ts.Add(-time.Minute), To: ts.Add(time.Minute), Filters: f})
+		if err != nil {
+			t.Fatal(err)
+		}
+		return tot.Wire
+	}
+	if got := q(Filter{Field: "bytes", Value: "in"}); got != 0 {
+		t.Errorf("bytes in = %d: by address nothing is inside", got)
+	}
+	if got := q(Filter{Field: "iface_in", Value: "10.0.0.1/7"}); got != 1000 {
+		t.Errorf("in through the uplink = %d, want 1000", got)
+	}
+	if got := q(Filter{Field: "iface_out", Value: "10.0.0.1/7"}); got != 9000 {
+		t.Errorf("out through the uplink = %d, want 9000 (its own duplicate counts)", got)
+	}
+}