mirror of
https://github.com/mark3labs/kit.git
synced 2026-06-14 03:30:26 +00:00
fix(ui): show pasted image previews in input and transcript (#48)
* fix(ui): show pasted image previews in input and transcript The half-block thumbnail preview added in #47 rendered but was clipped off the bottom of the screen, and submitted images showed only a text badge in the conversation history. - Mark the layout dirty when clipboardImageMsg / thumbnailReadyMsg reach the parent, so distributeHeight re-measures the now-taller input region instead of keeping a stale height that pushed the preview off-screen - Render thumbnail previews in the transcript after a user message, appended as a verbatim ScrollList item (raw ANSI half-blocks would be mangled if folded into the word-wrapped user text block) - Render transcript previews asynchronously via a tea.Cmd so decode + resample never blocks the Bubble Tea event loop - Add regression tests covering the input layout recompute and the transcript preview flow * fix(ui): anchor transcript image preview to its user message - Insert the async thumbnail preview directly after the originating user message (tracked via anchorID) instead of appending, so a streamed assistant reply that lands first no longer pushes the preview out of place - Make the layout regression test deterministic by forcing a truecolor profile, avoiding flakes on low-color CI terminals where the thumbnail would render empty - Add tests for anchored insertion and the unknown-anchor append fallback
This commit is contained in:
@@ -25,6 +25,7 @@ import (
|
||||
"github.com/mark3labs/kit/internal/ui/commands"
|
||||
uicore "github.com/mark3labs/kit/internal/ui/core"
|
||||
"github.com/mark3labs/kit/internal/ui/fileutil"
|
||||
"github.com/mark3labs/kit/internal/ui/imagepreview"
|
||||
"github.com/mark3labs/kit/internal/ui/prefs"
|
||||
"github.com/mark3labs/kit/internal/ui/style"
|
||||
kit "github.com/mark3labs/kit/pkg/kit"
|
||||
@@ -1794,14 +1795,27 @@ func (m *AppModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
// messages stay in chronological order.
|
||||
m.pendingUserPrints = append(m.pendingUserPrints, displayText)
|
||||
m.flushStreamAndPendingUserMessages()
|
||||
// Insert inline thumbnail previews after the user message.
|
||||
cmds = append(cmds, m.transcriptPreviewCmd(msg.Images, m.lastMessageID()))
|
||||
}
|
||||
} else {
|
||||
m.printUserMessage(displayText)
|
||||
// Insert inline thumbnail previews after the user message.
|
||||
cmds = append(cmds, m.transcriptPreviewCmd(msg.Images, m.lastMessageID()))
|
||||
}
|
||||
if m.state != stateWorking {
|
||||
m.state = stateWorking
|
||||
}
|
||||
|
||||
// ── Async transcript image preview ───────────────────────────────────────
|
||||
case imagePreviewReadyMsg:
|
||||
if msg.block != "" {
|
||||
item := NewStyledMessageItem(generateMessageID(), "user", "", msg.block)
|
||||
m.insertMessageAfter(msg.anchorID, item)
|
||||
m.refreshContent()
|
||||
m.layoutDirty = true
|
||||
}
|
||||
|
||||
// ── Shell command (! / !!) ───────────────────────────────────────────────
|
||||
case uicore.ShellCommandMsg:
|
||||
// Show spinner while the shell command runs.
|
||||
@@ -2447,6 +2461,19 @@ func (m *AppModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
m.printSystemMessage(msg.Text)
|
||||
}
|
||||
|
||||
// ── Clipboard image attached / thumbnail rendered ────────────────────────
|
||||
// Both messages change the input region's rendered height (the pill and
|
||||
// the async half-block preview), so forward them to the input and mark the
|
||||
// layout dirty — otherwise distributeHeight keeps a stale, too-short input
|
||||
// height and the preview is clipped off the bottom of the screen.
|
||||
case clipboardImageMsg, thumbnailReadyMsg:
|
||||
if m.input != nil {
|
||||
updated, cmd := m.input.Update(msg)
|
||||
m.input, _ = updated.(inputComponentIface)
|
||||
cmds = append(cmds, cmd)
|
||||
}
|
||||
m.layoutDirty = true
|
||||
|
||||
default:
|
||||
// Pass unrecognised messages to all children.
|
||||
if m.input != nil {
|
||||
@@ -3046,6 +3073,85 @@ func truncateMessageForBlock(msg string, maxLines, width int) string {
|
||||
// Print helpers — add content to ScrollList
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
// imagePreviewReadyMsg carries an asynchronously rendered transcript image
|
||||
// preview block back to the Update loop, where it is inserted into the
|
||||
// ScrollList directly after the originating user message (identified by
|
||||
// anchorID). Inserting by anchor — rather than appending — keeps the preview
|
||||
// next to its message even when the agent's streamed reply has already been
|
||||
// appended while the thumbnail was being decoded off the event loop.
|
||||
type imagePreviewReadyMsg struct {
|
||||
block string
|
||||
anchorID string
|
||||
}
|
||||
|
||||
// transcriptPreviewCmd returns a tea.Cmd that renders half-block thumbnail
|
||||
// previews for the given clipboard images off the Bubble Tea event loop
|
||||
// (decode + resample must not block Update). The rendered block is delivered
|
||||
// via imagePreviewReadyMsg, tagged with anchorID so the consumer can place it
|
||||
// directly after the originating user message. Returns nil when there is
|
||||
// nothing to render or no room for a preview; an empty result (terminal lacks
|
||||
// color support) yields a nil message that Bubble Tea ignores.
|
||||
func (m *AppModel) transcriptPreviewCmd(images []uicore.ImageAttachment, anchorID string) tea.Cmd {
|
||||
if len(images) == 0 {
|
||||
return nil
|
||||
}
|
||||
cols := thumbMaxCols
|
||||
if m.width > 6 && m.width-6 < cols {
|
||||
cols = m.width - 6
|
||||
}
|
||||
if cols < 1 {
|
||||
return nil
|
||||
}
|
||||
bg := style.GetTheme().Background
|
||||
imgs := images
|
||||
return func() tea.Msg {
|
||||
pad := lipgloss.NewStyle().PaddingLeft(2)
|
||||
var blocks []string
|
||||
for _, img := range imgs {
|
||||
thumb, err := imagepreview.Render(img.Data, img.MediaType, cols, thumbMaxRows, bg)
|
||||
if err != nil || thumb == "" {
|
||||
continue
|
||||
}
|
||||
blocks = append(blocks, pad.Render(thumb))
|
||||
}
|
||||
if len(blocks) == 0 {
|
||||
return nil
|
||||
}
|
||||
return imagePreviewReadyMsg{block: strings.Join(blocks, "\n"), anchorID: anchorID}
|
||||
}
|
||||
}
|
||||
|
||||
// lastMessageID returns the ID of the most recently added ScrollList message,
|
||||
// or "" when there are none. Used to anchor an async transcript preview to the
|
||||
// user message that was just printed.
|
||||
func (m *AppModel) lastMessageID() string {
|
||||
if len(m.messages) == 0 {
|
||||
return ""
|
||||
}
|
||||
return m.messages[len(m.messages)-1].ID()
|
||||
}
|
||||
|
||||
// insertMessageAfter inserts item immediately after the message whose ID
|
||||
// matches anchorID. If anchorID is empty or not found, item is appended.
|
||||
func (m *AppModel) insertMessageAfter(anchorID string, item MessageItem) {
|
||||
idx := -1
|
||||
if anchorID != "" {
|
||||
for i, msgItem := range m.messages {
|
||||
if msgItem.ID() == anchorID {
|
||||
idx = i
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if idx < 0 {
|
||||
m.messages = append(m.messages, item)
|
||||
return
|
||||
}
|
||||
m.messages = append(m.messages, nil)
|
||||
copy(m.messages[idx+2:], m.messages[idx+1:])
|
||||
m.messages[idx+1] = item
|
||||
}
|
||||
|
||||
// printUserMessage renders a user message into the ScrollList.
|
||||
func (m *AppModel) printUserMessage(text string) {
|
||||
// Check if this exact message was just added (prevents duplicates)
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
package ui
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
tea "charm.land/bubbletea/v2"
|
||||
uicore "github.com/mark3labs/kit/internal/ui/core"
|
||||
)
|
||||
|
||||
// drainCmds runs a tea.Cmd chain back through m.Update like the BubbleTea
|
||||
// event loop, expanding batches, until no further messages are produced.
|
||||
func drainCmds(t *testing.T, m *AppModel, cmd tea.Cmd) *AppModel {
|
||||
t.Helper()
|
||||
queue := []tea.Cmd{cmd}
|
||||
for i := 0; i < 50 && len(queue) > 0; i++ {
|
||||
c := queue[0]
|
||||
queue = queue[1:]
|
||||
if c == nil {
|
||||
continue
|
||||
}
|
||||
msg := c()
|
||||
if msg == nil {
|
||||
continue
|
||||
}
|
||||
if batch, ok := msg.(tea.BatchMsg); ok {
|
||||
queue = append(queue, batch...)
|
||||
continue
|
||||
}
|
||||
updated, nc := m.Update(msg)
|
||||
m = updated.(*AppModel)
|
||||
_ = m.View()
|
||||
if nc != nil {
|
||||
queue = append(queue, nc)
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func measuredInputHeight(m *AppModel) int {
|
||||
rendered := m.renderInput()
|
||||
if rendered == "" {
|
||||
return 0
|
||||
}
|
||||
return strings.Count(rendered, "\n") + 1
|
||||
}
|
||||
|
||||
// TestPendingThumbnailTriggersLayoutRecompute is a regression test for the bug
|
||||
// where a pasted image's async half-block preview rendered but was clipped off
|
||||
// the bottom of the screen: the thumbnail arrives via thumbnailReadyMsg after
|
||||
// distributeHeight already measured the input region without it. The parent
|
||||
// must mark the layout dirty so the (now taller) input is re-measured.
|
||||
func TestPendingThumbnailTriggersLayoutRecompute(t *testing.T) {
|
||||
// Force a truecolor profile so imagepreview.Render deterministically
|
||||
// produces a thumbnail regardless of the CI terminal's color support.
|
||||
// Without this, a low-color test environment yields an empty preview and
|
||||
// the glyph / height assertions below would flake.
|
||||
t.Setenv("TERM", "xterm-256color")
|
||||
t.Setenv("COLORTERM", "truecolor")
|
||||
t.Setenv("NO_COLOR", "")
|
||||
|
||||
real := NewInputComponent(80, nil)
|
||||
m, _, _ := newTestAppModel(nil)
|
||||
m.input = real
|
||||
m = sendMsg(m, tea.WindowSizeMsg{Width: 80, Height: 24})
|
||||
|
||||
heightBefore := measuredInputHeight(m)
|
||||
|
||||
updated, cmd := m.Update(clipboardImageMsg{image: &uicore.ImageAttachment{
|
||||
Data: makeTestPNG(t, 16, 16),
|
||||
MediaType: "image/png",
|
||||
}})
|
||||
m = updated.(*AppModel)
|
||||
_ = m.View()
|
||||
m = drainCmds(t, m, cmd)
|
||||
|
||||
heightAfter := measuredInputHeight(m)
|
||||
if heightAfter <= heightBefore {
|
||||
t.Errorf("input region should grow to fit the thumbnail (before=%d after=%d)", heightBefore, heightAfter)
|
||||
}
|
||||
|
||||
if !strings.Contains(m.View().Content, "▀") {
|
||||
t.Error("parent View should contain the half-block thumbnail (was clipped or not rendered)")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
package ui
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/png"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
uicore "github.com/mark3labs/kit/internal/ui/core"
|
||||
)
|
||||
|
||||
// makeTestPNG builds a small solid-color PNG for transcript preview tests.
|
||||
func makeTestPNG(t *testing.T, w, h int) []byte {
|
||||
t.Helper()
|
||||
img := image.NewRGBA(image.Rect(0, 0, w, h))
|
||||
for y := range h {
|
||||
for x := range w {
|
||||
img.Set(x, y, color.RGBA{R: 200, G: 40, B: 90, A: 255})
|
||||
}
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := png.Encode(&buf, img); err != nil {
|
||||
t.Fatalf("encode png: %v", err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func TestTranscriptPreviewCmdNoImages(t *testing.T) {
|
||||
m, _, _ := newTestAppModel(nil)
|
||||
if cmd := m.transcriptPreviewCmd(nil, ""); cmd != nil {
|
||||
t.Error("expected nil cmd when there are no images")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscriptPreviewCmdRendersBlock(t *testing.T) {
|
||||
m, _, _ := newTestAppModel(nil)
|
||||
images := []uicore.ImageAttachment{
|
||||
{Data: makeTestPNG(t, 16, 16), MediaType: "image/png"},
|
||||
}
|
||||
cmd := m.transcriptPreviewCmd(images, "anchor-1")
|
||||
if cmd == nil {
|
||||
t.Fatal("expected a non-nil cmd for a valid image")
|
||||
}
|
||||
msg := cmd()
|
||||
// The result depends on the test process color profile. When the
|
||||
// terminal supports color the cmd yields a preview block; otherwise it
|
||||
// yields nil (caller keeps the text badge). Both are valid — assert the
|
||||
// shape only when a block is produced.
|
||||
if msg == nil {
|
||||
t.Skip("color profile below ANSI256 in test env; preview correctly skipped")
|
||||
}
|
||||
ready, ok := msg.(imagePreviewReadyMsg)
|
||||
if !ok {
|
||||
t.Fatalf("expected imagePreviewReadyMsg, got %T", msg)
|
||||
}
|
||||
if !strings.Contains(ready.block, "▀") {
|
||||
t.Errorf("preview block should contain half-block glyphs, got %q", ready.block)
|
||||
}
|
||||
if ready.anchorID != "anchor-1" {
|
||||
t.Errorf("preview should carry the originating anchorID, got %q", ready.anchorID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImagePreviewReadyMsgAppendsItem(t *testing.T) {
|
||||
m, _, _ := newTestAppModel(nil)
|
||||
before := len(m.messages)
|
||||
m = sendMsg(m, imagePreviewReadyMsg{block: "\x1b[38;2;1;2;3;48;2;4;5;6m▀\x1b[0m"})
|
||||
if len(m.messages) != before+1 {
|
||||
t.Fatalf("expected one appended message item, got %d (was %d)", len(m.messages), before)
|
||||
}
|
||||
last, ok := m.messages[len(m.messages)-1].(*TextMessageItem)
|
||||
if !ok {
|
||||
t.Fatalf("expected last item to be *TextMessageItem, got %T", m.messages[len(m.messages)-1])
|
||||
}
|
||||
if !strings.Contains(last.Render(0), "▀") {
|
||||
t.Error("appended preview item should render the half-block block verbatim")
|
||||
}
|
||||
}
|
||||
|
||||
// TestImagePreviewReadyMsgInsertsAfterAnchor verifies the preview is placed
|
||||
// directly after its originating user message even when a later message (e.g.
|
||||
// a streamed assistant reply) was already appended while the thumbnail was
|
||||
// being decoded asynchronously.
|
||||
func TestImagePreviewReadyMsgInsertsAfterAnchor(t *testing.T) {
|
||||
m, _, _ := newTestAppModel(nil)
|
||||
userItem := NewStyledMessageItem("user-anchor", "user", "hi", "hi")
|
||||
assistantItem := NewStyledMessageItem("assistant-1", "assistant", "reply", "reply")
|
||||
m.messages = append(m.messages, userItem, assistantItem)
|
||||
|
||||
m = sendMsg(m, imagePreviewReadyMsg{
|
||||
block: "\x1b[38;2;1;2;3;48;2;4;5;6m▀\x1b[0m",
|
||||
anchorID: "user-anchor",
|
||||
})
|
||||
|
||||
// Expect order: user, preview, assistant.
|
||||
if len(m.messages) != 3 {
|
||||
t.Fatalf("expected 3 messages, got %d", len(m.messages))
|
||||
}
|
||||
if m.messages[0].ID() != "user-anchor" {
|
||||
t.Errorf("messages[0] should be the user message, got %q", m.messages[0].ID())
|
||||
}
|
||||
if m.messages[2].ID() != "assistant-1" {
|
||||
t.Errorf("messages[2] should be the assistant message, got %q", m.messages[2].ID())
|
||||
}
|
||||
if !strings.Contains(m.messages[1].Render(0), "▀") {
|
||||
t.Errorf("messages[1] should be the inserted preview, got %q", m.messages[1].Render(0))
|
||||
}
|
||||
}
|
||||
|
||||
// TestImagePreviewReadyMsgUnknownAnchorAppends verifies that when the anchor
|
||||
// is missing (e.g. the message was cleared), the preview falls back to append.
|
||||
func TestImagePreviewReadyMsgUnknownAnchorAppends(t *testing.T) {
|
||||
m, _, _ := newTestAppModel(nil)
|
||||
m.messages = append(m.messages, NewStyledMessageItem("only", "user", "hi", "hi"))
|
||||
m = sendMsg(m, imagePreviewReadyMsg{
|
||||
block: "\x1b[38;2;1;2;3;48;2;4;5;6m▀\x1b[0m",
|
||||
anchorID: "does-not-exist",
|
||||
})
|
||||
if len(m.messages) != 2 {
|
||||
t.Fatalf("expected 2 messages, got %d", len(m.messages))
|
||||
}
|
||||
if !strings.Contains(m.messages[1].Render(0), "▀") {
|
||||
t.Error("preview should be appended as the last item when anchor is unknown")
|
||||
}
|
||||
}
|
||||
|
||||
func TestImagePreviewReadyMsgEmptyBlockIgnored(t *testing.T) {
|
||||
m, _, _ := newTestAppModel(nil)
|
||||
before := len(m.messages)
|
||||
m = sendMsg(m, imagePreviewReadyMsg{block: ""})
|
||||
if len(m.messages) != before {
|
||||
t.Errorf("empty preview block should not append an item; got %d (was %d)", len(m.messages), before)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user