Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/search'
Browse files Browse the repository at this point in the history
  • Loading branch information
@gcla
gcla committed Jun 4, 2022
2 parents f0a80fa b92b83c commit 1e2e0e2
Show file tree
Hide file tree
Showing 17 changed files with 2,911 additions and 117 deletions.
5 changes: 5 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 2,11 @@

## [Unreleased]

### Added

- You can now search for information in packets, in similar fashion to Wireshark's packet search. Hit `ctrl-f`
to open the search bar.

### Changed

- Fixed a bug that caused mouse-clicks within the hex view to not function correctly if the viewport was not
Expand Down
2 changes: 1 addition & 1 deletion assets/statik/statik.go
View file

Large diffs are not rendered by default.

25 changes: 15 additions & 10 deletions cmd/termshark/termshark.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -816,16 816,27 @@ func cmain() int {
}
}

// the app variable is created here so I can bind it in the defer below
var app *gowid.App

// Do this before ui.Build. If ui.Build fails (e.g. bad TERM), then the filter will be left
// running, so we need the defer to be in effect here and not after the processing of ui.Build's
// error
defer func() {
if ui.FilterWidget != nil {
ui.FilterWidget.Close()
}
if ui.SearchWidget != nil {
ui.SearchWidget.Close(app)
}
if ui.CurrentWormholeWidget != nil {
ui.CurrentWormholeWidget.Close()
}
if ui.CurrentColsWidget != nil {
ui.CurrentColsWidget.Close()
}
}()

var app *gowid.App
if app, err = ui.Build(); err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
// Tcell returns ExitError now because if its internal terminfo DB does not have
Expand Down Expand Up @@ -1008,8 1019,8 @@ Loop:
// loads/filters to that now.
ui.Loader.TurnOffPipe()
app.Run(gowid.RunFunction(func(app gowid.IApp) {
ui.ClearProgressWidget(app)
ui.SetProgressDeterminate(app) // always switch back - for pdml (partial) loads of later data.
ui.ClearProgressWidgetFor(app, ui.LoaderOwns)
ui.SetProgressDeterminateFor(app, ui.LoaderOwns) // always switch back - for pdml (partial) loads of later data.
}))

// When the progress bar is enabled, track the previous percentage reached. This is
Expand Down Expand Up @@ -1050,7 1061,7 @@ Loop:
if progCancelTimer == nil {
progCancelTimer = time.AfterFunc(time.Duration(100000)*time.Hour, func() {
app.Run(gowid.RunFunction(func(app gowid.IApp) {
ui.ClearProgressWidget(app)
ui.ClearProgressWidgetFor(app, ui.LoaderOwns)
progCancelTimer = nil
}))
})
Expand Down Expand Up @@ -1235,12 1246,6 @@ Loop:
// will happen next time round because the quitRequested flag is checked.
stopLoaders()
}
if ui.CurrentWormholeWidget != nil {
ui.CurrentWormholeWidget.Close()
}
if ui.CurrentColsWidget != nil {
ui.CurrentColsWidget.Close()
}

case sig := <-sigChan:
if system.IsSigTSTP(sig) {
Expand Down
16 changes: 14 additions & 2 deletions docs/FAQ.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 209,7 @@ Yes, via `~/.config/termshark/termshark.toml`. Here is an example I use:
Termshark uses tshark to provide all the data it displays, and to validate display filter expressions. When you give termshark a pcap file, it will run

````bash
tshark -T psml -r my.pcap -Y '<expression from ui>' -o gui.column.format:\"...\"```
tshark -T psml -r my.pcap -Y '<expression from ui>' -o gui.column.format:"..."```
````
The data provided to the `gui.column.format` tshark argument is stored in termshark's config file under the key `main.column-format`. Let's say the
Expand All @@ -235,7 235,7 @@ tail -f -c 0 tmpfile
The stdout of the `tail` command is connected to the stdin of the PSML reading command, which is adjusted to:

````bash
tshark -T psml -i - -l -Y '<expression from ui>' -o gui.column.format:\"...\"```
tshark -T psml -i - -l -Y '<expression from ui>' -o gui.column.format:"..."```
````
The `-l` switch might push the data to the UI more quickly... The PDML and byte/hex generating commands read directly from `tmpfile`, since they don't need to provide continual updates (they load data in batches as the user moves around).
Expand Down Expand Up @@ -290,6 290,18 @@ The information is displayed in a table by conversation type. If the user has a
tshark -r my.pcap -q -z conv,eth,http -z conv,ip,http -z conv,tcp,http
```
If the user runs a packet search and chooses the Display Filter option, termshark will launch a tshark process to find packets that match the filter.
````bash
tshark -T psml -r my.pcap -Y '<search expression from ui>' -o gui.column.format:"No.","%m"```
````

If the user has an active display filter via the UI, it is combined with the search expression:

````bash
tshark -T psml -r my.pcap -Y '(<display filter from ui>) && (<search expression from ui>)' -o gui.column.format:"No.","%m"```
````
Termshark also uses tshark to generate the possible completions for prefixes of display filter terms. If you type `tcp.` in the filter widget, termshark will show a drop-down menu of possible completions. This is generated once at startup by running
```bash
Expand Down
30 changes: 30 additions & 0 deletions docs/UserGuide.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 18,7 @@ Termshark provides a terminal-based user interface for analyzing packet captures
- [Packet Structure View](#packet-structure-view)
- [Packet Hex View](#packet-hex-view)
- [Marking Packets](#marking-packets)
- [Searching Packets](#searching-packets)
- [Copy Mode](#copy-mode)
- [Packet Capture Information](#packet-capture-information)
- [Stream Reassembly](#stream-reassembly)
Expand Down Expand Up @@ -252,6 253,32 @@ To jump back to that mark, hit `'` followed by the letter you selected. To jump

![marks2](/../gh-pages/images/marks2.png?raw=true)

### Searching Packets

To search within packets, hit `ctrl-f` to open termshark's search bar. The options provided closely mirror those available with Wireshark. The first button displays a menu that lets you choose the type of data searched:

- Packet List - the info shown in the packet list view (by default the top data pane)
- Packet Struct - the info shown in the packet struct view (by default the middle data pane)
- Packet Bytes - the info shown in the packet hex view (by default the bottom data pane)

The second button lets you choose what to search for:

- String (with or without case sensitivity)
- Regex (with or without case sensitivity)
- Hex
- Display Filter

The Hex syntax follows Wireshark and requires a sequence of 2 hex-digits, concatenated. For example, entering "AF054c" would mean to search for the following 3 bytes, consecutively - 175, 5, 76.

Display Filter search is a special case and does not search the packet data directly. Instead, termshark launches a tshark process on the current pcap source with flags to apply the user's search filter. Termshark parses the output and every packet that appears in the PSML data is a match for the search.

If a match is found, termshark will navigate to the match location in the UI. For a Packet List search, the matching row and column are selected. For a Packet Struct search, the matching element in the packet structure view is expanded and the UI centered around it. For a Packet Bytes search, the cursor is moved to the start of the match in the packet hex view.

To terminate the search early, hit `ctrl-c`. To set focus on the search bar's input, hit `ctrl-f` again. To close the search bar, hit `ctrl-f` one more time.

![search1](/../gh-pages/images/search1.png?raw=true)


### Copy Mode

Both the structure and hex view support "copy mode" a feature which lets you copy ranges of data from the currently selected packet. First, move focus to the part of the packet you wish to copy. Now hit the `c` key - a section of the packet will be highlighted in yellow:
Expand Down Expand Up @@ -634,6 661,9 @@ Termshark reads options from a TOML configuration file saved in `$XDG_CONFIG_HOM
- `recent-files` (string list) - the pcap files shown when the user clicks the "recent" button in termshark. Newly viewed files are added to the beginning.
- `recent-filters` (string list) - recently used Wireshark display filters.
- `respect-colorterm` (bool) - if termshark detects you are using base16-shell, it won't map any theme RGB color names (like #90FF32) to 0-21 in the 256-color space to avoid clashes with the active base16 theme. This shouldn't affect color reproduction if the terminal is 24-bit capable, but some terminal emulators (e.g. gnome-terminal) seem to use the 256-color space anyway. Termshark works around this by falling back to 256-color mode, interpolating RGB colors into the 256-color space and avoiding 0-21. If you really want termshark to run in 24-bit color mode anyway, set this to true.
- `search-type` - (string) - how to interpret the user's packet search term; one of `filter`, `hex`, `string` or `regex`.
- `search-target` - (string) - the type of packet data to search (unless `search-type` is `filter`); one of `list`, `details` or `bytes`.
- `search-case-sensitive` - (bool) - true if the user's packet search should be sensitive to the case of the search term.
- `stream-cache-size` (int) - termshark caches the structures and UI used to display reassembled TCP and UDP streams. This allows for quickly redisplaying a stream that's been loaded before. This setting determines how many streams are cached. The default is 100.
- `stream-view` (string - the default view when displaying a reassembled stream. Choose from "hex"/"ascii"/"raw".
- `suppress-tshark-errors` (bool) - if `true`, hide from the UI any errors generated during parsing of tshark-generated XML.
Expand Down
3 changes: 3 additions & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 35,13 @@ require (
github.com/spf13/viper v1.7.1
github.com/stretchr/testify v1.7.0
github.com/tevino/abool v1.2.0
gitlab.com/jonas.jasas/condchan v0.0.0-20190210165812-36637ad2b5bc // indirect
golang.org/x/net v0.0.0-20201224014010-6772e930b67b // indirect
golang.org/x/sys v0.0.0-20220318055525-2edf467146b5
gopkg.in/fsnotify/fsnotify.v1 v1.4.7
gopkg.in/ini.v1 v1.62.0 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
modernc.org/interval v1.0.0 // indirect
modernc.org/mathutil v1.4.1 // indirect
)
6 changes: 6 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 251,7 @@ github.com/psanford/wormhole-william v1.0.6-0.20210402195004-049df45b8d5a/go.mod
github.com/rakyll/statik v0.1.6/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs=
github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ=
github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
Expand Down Expand Up @@ -306,6 307,8 @@ github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d z6vm202qpg1oXVM
github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL OaLUSs=
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax UKWsSmolVDwsd 7N3ZtXu yMGCf907BLYF3GoBXY=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
gitlab.com/jonas.jasas/condchan v0.0.0-20190210165812-36637ad2b5bc h1:zCsu odZEHb2f8U8WWhDgY5N5w3JCLHxuCIqVqCsLcQ=
gitlab.com/jonas.jasas/condchan v0.0.0-20190210165812-36637ad2b5bc/go.mod h1:4JS8TdA7HSdK x43waOdTGodqY/VKsj4w 8pWDL0E88=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL uU=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1: kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
Expand Down Expand Up @@ -466,6 469,9 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
modernc.org/interval v1.0.0 h1:Qn2Wuajm4C459vIf4woVviD0ITEJ1ulMKb2dLm/7qE8=
modernc.org/interval v1.0.0/go.mod h1:/AfW0uaVPVb1/afoQD3MW9Foc/vVUp0YghT6Wm6IM/A=
modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
nhooyr.io/websocket v1.8.6 h1:s C3xAMLwGmlI31Nyn/eAehUlZPwfYZu2JXM621Q5/k=
nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck e2FymRvadv62gMdZztPaShugOCi3I 8D8=
Expand Down
23 changes: 17 additions & 6 deletions pcap/loader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 121,9 @@ type PacketLoader struct {
*ParentLoader
}

// Renew is called when a new pcap is loaded from an open termshark session i.e. termshark
// was started with one packet source, then a new one is selected. This ensures that all
// connected loaders that might still be doing work are cancelled.
func (c *PacketLoader) Renew() {
if c.ParentLoader != nil {
c.ParentLoader.CloseMain()
Expand Down Expand Up @@ -198,9 201,12 @@ type PsmlLoader struct {
packetPsmlData [][]string
packetPsmlColors []PacketColors
packetPsmlHeaders []string
PacketNumberMap map[int]int // map from actual packet row <section>12</section> to pos in unsorted table
PacketNumberMap map[int]int // map from actual packet row <packet>12</packet> to pos in unsorted table
// This would be affected by a display filter e.g. packet 12 might be the 1st packet in the table.
// I need this so that if the user jumps to a mark stored as "packet 12", I can find the right table row.
PacketNumberOrder map[int]int // e.g. {12->44, 44->71, 71->72,...} - the packet numbers, in order, affected by a filter.
// If I use a generic ordered map, I could avoid this separate structure

PacketCache *lru.Cache // i -> [pdml(i * 1000)..pdml(i 1*1000)] - accessed from any goroutine

opt Options
Expand Down Expand Up @@ -324,6 330,7 @@ func (c *ParentLoader) RenewPsmlLoader() {
packetPsmlColors: make([]PacketColors, 0),
packetPsmlHeaders: make([]string, 0, 10),
PacketNumberMap: make(map[int]int),
PacketNumberOrder: make(map[int]int),
startStage2Chan: make(chan struct{}), // do this before signalling start
PsmlFinishedChan: make(chan struct{}),
opt: c.opt,
Expand Down Expand Up @@ -438,7 445,7 @@ func (c *PacketLoader) Reload(filter string, cb interface{}, app gowid.IApp) {

// This is not ideal. I'm clearing the views, but I'm about to
// restart. It's not really a new source, so called the new source
// handler is an untify way of updating the current capture in the
// handler is an untidy way of updating the current capture in the
// title bar again
handleClear(NoneCode, app, cb)

Expand Down Expand Up @@ -468,11 475,11 @@ func (c *PacketLoader) LoadPcap(pcap string, displayFilter string, cb interface{
c.stopLoadIface()

OpsChan <- gowid.RunFunction(func(app gowid.IApp) {
c.Renew()

// This will enable the operation when clear completes
handleClear(NoneCode, app, cb)

c.Renew()

c.psrcs = []IPacketSource{FileSource{Filename: pcap}}
c.ifaceFile = ""

Expand Down Expand Up @@ -521,13 528,13 @@ func (c *PacketLoader) ClearPcap(cb interface{}) {

// When stop is done, launch the clear and restart
OpsChan <- gowid.RunFunction(func(app gowid.IApp) {
handleClear(NoneCode, app, cb)

// Don't CloseMain - that will stop the interface process too
c.loadWasCancelled = false
c.RenewPsmlLoader()
c.RenewPdmlLoader()

handleClear(NoneCode, app, cb)

if !startIfaceAgain {
c.psrcs = c.psrcs[:0]
c.ifaceFile = ""
Expand Down Expand Up @@ -1740,6 1747,8 @@ func (p *PsmlLoader) loadPsmlSync(iloader *InterfaceLoader, e iPsmlLoaderEnv, cb
var fg string
var bg string
var pidx int
ppidx := 0 // the previous packet number read; 0 means no packet. I can use 0 because
// the psml I read will start at packet 1 so - map[0] => 1st packet
ready := false
empty := true
structure := false
Expand Down Expand Up @@ -1778,6 1787,8 @@ func (p *PsmlLoader) loadPsmlSync(iloader *InterfaceLoader, e iPsmlLoaderEnv, cb
log.Fatal(err)
}
p.PacketNumberMap[pidx] = len(p.packetPsmlData)
p.PacketNumberOrder[ppidx] = pidx
ppidx = pidx

p.packetPsmlData = append(p.packetPsmlData, curPsml[1:])

Expand Down
25 changes: 25 additions & 0 deletions ui/prochandlers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 71,9 @@ func MakePacketViewUpdater() updatePacketViews {

func (t updatePacketViews) OnClear(code pcap.HandlerCode, app gowid.IApp) {
clearPacketViews(app)
if packetListView != nil {
updatePacketListWithData(t.Ld, app)
}
}

func (t updatePacketViews) BeforeBegin(code pcap.HandlerCode, app gowid.IApp) {
Expand Down Expand Up @@ -256,6 259,28 @@ func (t checkGlobalJumpAfterPsml) OnClear(code pcap.HandlerCode, app gowid.IApp)

//======================================================================

func clearSearchData(app gowid.IApp) {
if SearchWidget != nil {
SearchWidget.Clear(app)
}
}

type ManageSearchData struct{}

var _ pcap.INewSource = ManageSearchData{}
var _ pcap.IClear = ManageSearchData{}

// Make sure that existing stream widgets are discarded if the user loads a new pcap.
func (t ManageSearchData) OnNewSource(c pcap.HandlerCode, app gowid.IApp) {
clearSearchData(app)
}

func (t ManageSearchData) OnClear(c pcap.HandlerCode, app gowid.IApp) {
clearSearchData(app)
}

//======================================================================

type checkGlobalJumpAfterPsml struct {
Jump termshark.GlobalJumpPos
}
Expand Down
Loading

0 comments on commit 1e2e0e2

Please sign in to comment.