parser.go

v0.7.0

package logreader

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"regexp"
	"strings"
	"time"
)

// EntryKind classifies a log entry.
type EntryKind int

const (
	KindSlog     EntryKind = iota // time=... level=... structured log line (fallback)
	KindInfo                      // level=INFO
	KindWarn                      // level=WARN
	KindError                     // level=ERROR
	KindDebug                     // level=DEBUG
	KindIncoming                  // INCOMING REQUEST from Claude Code
	KindOutgoing                  // OUTGOING REQUEST to upstream API
	KindResponse                  // UPSTREAM RESPONSE
	KindStream                    // STREAM RESPONSE FROM UPSTREAM
)

func (k EntryKind) String() string {
	switch k {
	case KindSlog:
		return "LOG"
	case KindInfo:
		return "INFO"
	case KindWarn:
		return "WARN"
	case KindError:
		return "ERROR"
	case KindDebug:
		return "DEBUG"
	case KindIncoming:
		return "REQ IN"
	case KindOutgoing:
		return "REQ OUT"
	case KindResponse:
		return "RESPONSE"
	case KindStream:
		return "STREAM"
	default:
		return "UNKNOWN"
	}
}

// IsSlog returns true for any slog-derived entry kind.
func (k EntryKind) IsSlog() bool {
	return k == KindSlog || k == KindInfo || k == KindWarn || k == KindError || k == KindDebug
}

func slogKind(level string) EntryKind {
	switch strings.ToUpper(level) {
	case "INFO":
		return KindInfo
	case "WARN", "WARNING":
		return KindWarn
	case "ERROR":
		return KindError
	case "DEBUG":
		return KindDebug
	default:
		return KindSlog
	}
}

// Header is a parsed HTTP header key-value pair.
type Header struct {
	Key   string
	Value string
}

// StreamData holds parsed SSE stream information.
type StreamData struct {
	Model      string
	Thinking   string // reassembled reasoning content
	Content    string // reassembled output content
	Chunks     int
	Usage      *StreamUsage
	StopReason string
}

// StreamUsage holds token usage from the stream.
type StreamUsage struct {
	PromptTokens     int `json:"prompt_tokens"`
	CompletionTokens int `json:"completion_tokens"`
	TotalTokens      int `json:"total_tokens"`
	CachedTokens     int `json:"cached_tokens,omitempty"`
}

// Entry is a single parsed log entry.
type Entry struct {
	Kind     EntryKind
	Time     time.Time
	Label    string // short summary for the list
	Content  string // full raw text
	SepAfter bool   // render a visual separator after this entry in the list

	// Structured HTTP fields (populated for request/response kinds).
	RequestLine string   // e.g. "POST /v1/messages HTTP/1.1" or "HTTP/2.0 200 OK"
	Headers     []Header // parsed headers
	Body        string   // pretty-printed JSON body (if available), otherwise raw body

	// Structured stream fields (populated for KindStream).
	Stream *StreamData
}

var (
	bannerRe = regexp.MustCompile(`^\[(.+?)\]\s+(.+)$`)
	slogRe   = regexp.MustCompile(`^time=(\S+)\s+level=(\S+)\s+msg=(.*)$`)
	sepLine  = strings.Repeat("=", 80)
	footLine = strings.Repeat("-", 80)
)

// Parse reads a log file and returns parsed entries.
func Parse(r io.Reader) []Entry {
	scanner := bufio.NewScanner(r)
	scanner.Buffer(make([]byte, 0, 1024*1024), 10*1024*1024)

	var entries []Entry

	for scanner.Scan() {
		line := scanner.Text()

		// Check for slog line — each one becomes its own entry.
		if slogRe.MatchString(line) {
			t, lvl, msg := parseSlogLine(line)
			entries = append(entries, Entry{
				Kind:    slogKind(lvl),
				Time:    t,
				Label:   msg,
				Content: line,
			})
			continue
		}

		// Check for banner separator start.
		if line == sepLine {
			entry := parseBannerEntry(scanner)
			if entry != nil {
				entries = append(entries, *entry)
			}
			continue
		}
	}

	// Mark "proxy logging stopped" entries that are followed by more entries
	// as separators between sessions.
	for i := range entries {
		if entries[i].Kind.IsSlog() &&
			strings.Contains(entries[i].Label, "proxy logging stopped") &&
			i < len(entries)-1 {
			entries[i].SepAfter = true
		}
	}

	return entries
}

func parseBannerEntry(scanner *bufio.Scanner) *Entry {
	// Next line should be the label line: [timestamp] LABEL
	if !scanner.Scan() {
		return nil
	}
	labelLine := scanner.Text()

	m := bannerRe.FindStringSubmatch(labelLine)
	if m == nil {
		return nil
	}
	ts, _ := time.Parse(time.RFC3339Nano, m[1])
	label := m[2]

	// Skip the closing separator (another === line).
	scanner.Scan()

	// Determine kind.
	kind := classifyLabel(label)

	// Read content until footer (---) or next banner (===).
	var content strings.Builder
	for scanner.Scan() {
		line := scanner.Text()
		if line == footLine {
			break
		}
		if line == sepLine {
			// We hit the next banner — this entry had no footer.
			// We need to parse this as a new banner, but scanner already consumed
			// the line. We'll just break and let the outer loop handle it on next iteration.
			// Unfortunately we lose this separator. Instead, let's include a note.
			break
		}
		content.WriteString(line)
		content.WriteByte('\n')
	}

	raw := content.String()
	summary := buildSummary(kind, raw)

	entry := &Entry{
		Kind:    kind,
		Time:    ts,
		Label:   fmt.Sprintf("%s — %s", label, summary),
		Content: raw,
	}

	// Parse structured HTTP parts for request/response entries.
	if kind == KindIncoming || kind == KindOutgoing || kind == KindResponse {
		entry.RequestLine, entry.Headers, entry.Body = parseHTTPContent(raw)
	}

	// Parse stream data.
	if kind == KindStream {
		entry.Stream = parseStreamContent(raw)
	}

	return entry
}

func classifyLabel(label string) EntryKind {
	switch {
	case strings.Contains(label, "INCOMING REQUEST"):
		return KindIncoming
	case strings.Contains(label, "OUTGOING REQUEST"):
		return KindOutgoing
	case strings.HasPrefix(label, "STREAM"):
		return KindStream
	case strings.Contains(label, "RESPONSE"):
		return KindResponse
	default:
		return KindSlog
	}
}

func buildSummary(kind EntryKind, content string) string {
	lines := strings.SplitN(content, "\n", 5)
	switch kind {
	case KindIncoming, KindOutgoing:
		// First line is like "POST /v1/messages?beta=true HTTP/1.1"
		if len(lines) > 0 {
			return truncate(lines[0], 60)
		}
	case KindResponse:
		// First line is like "HTTP/2.0 200 OK"
		if len(lines) > 0 {
			return truncate(lines[0], 60)
		}
	case KindStream:
		// Count data: lines.
		count := 0
		for l := range strings.SplitSeq(content, "\n") {
			if strings.HasPrefix(l, "data: ") {
				count++
			}
		}
		return fmt.Sprintf("%d chunks", count)
	}
	return ""
}

const prettyJSONMarker = "--- Pretty JSON ---"

// parseHTTPContent splits raw HTTP dump into request/status line, headers, and body.
// If a "--- Pretty JSON ---" section exists, it is used as the body instead of raw.
func parseHTTPContent(raw string) (requestLine string, headers []Header, body string) {
	lines := strings.Split(raw, "\n")
	if len(lines) == 0 {
		return "", nil, ""
	}

	// First non-empty line is the request/status line.
	i := 0
	for i < len(lines) && strings.TrimSpace(lines[i]) == "" {
		i++
	}
	if i >= len(lines) {
		return "", nil, ""
	}
	requestLine = strings.TrimSpace(lines[i])
	i++

	// Parse headers until empty line.
	for i < len(lines) {
		line := lines[i]
		i++
		if strings.TrimSpace(line) == "" {
			break
		}
		if k, v, ok := strings.Cut(line, ":"); ok {
			headers = append(headers, Header{
				Key:   strings.TrimSpace(k),
				Value: strings.TrimSpace(v),
			})
		}
	}

	// Check if there's a pretty JSON section — use that as body.
	_, after, ok := strings.Cut(raw, prettyJSONMarker)
	if ok {
		body = strings.TrimSpace(after)
		return
	}

	// Otherwise use whatever is after the empty line as body.
	if i < len(lines) {
		body = strings.TrimSpace(strings.Join(lines[i:], "\n"))
	}
	return
}

// sseChunk represents a single SSE data chunk (OpenAI chat completion format).
type sseChunk struct {
	Model   string `json:"model"`
	Choices []struct {
		Delta struct {
			Content          *string `json:"content"`
			Reasoning        *string `json:"reasoning"`
			ReasoningContent *string `json:"reasoning_content"`
		} `json:"delta"`
		FinishReason *string `json:"finish_reason"`
	} `json:"choices"`
	Usage *StreamUsage `json:"usage"`
}

func parseStreamContent(raw string) *StreamData {
	sd := &StreamData{}
	var thinking, content strings.Builder

	for line := range strings.SplitSeq(raw, "\n") {
		if !strings.HasPrefix(line, "data: ") {
			continue
		}
		payload := strings.TrimPrefix(line, "data: ")
		if payload == "[DONE]" {
			continue
		}

		var chunk sseChunk
		if json.Unmarshal([]byte(payload), &chunk) != nil {
			continue
		}

		sd.Chunks++
		if sd.Model == "" && chunk.Model != "" {
			sd.Model = chunk.Model
		}

		if len(chunk.Choices) > 0 {
			c := chunk.Choices[0]
			// Prefer reasoning_content over reasoning.
			if c.Delta.ReasoningContent != nil && *c.Delta.ReasoningContent != "" {
				thinking.WriteString(*c.Delta.ReasoningContent)
			} else if c.Delta.Reasoning != nil && *c.Delta.Reasoning != "" {
				thinking.WriteString(*c.Delta.Reasoning)
			}
			if c.Delta.Content != nil {
				content.WriteString(*c.Delta.Content)
			}
			if c.FinishReason != nil {
				sd.StopReason = *c.FinishReason
			}
		}

		if chunk.Usage != nil {
			sd.Usage = chunk.Usage
		}
	}

	sd.Thinking = thinking.String()
	sd.Content = content.String()
	return sd
}

func truncate(s string, max int) string {
	if len(s) <= max {
		return s
	}
	return s[:max-3] + "..."
}

func parseSlogLine(line string) (time.Time, string, string) {
	m := slogRe.FindStringSubmatch(line)
	if m == nil {
		return time.Time{}, "", line
	}
	t, _ := time.Parse(time.RFC3339Nano, m[1])
	return t, m[2], unquote(m[3])
}

func unquote(s string) string {
	s = strings.TrimSpace(s)
	if len(s) >= 2 && s[0] == '"' && s[len(s)-1] == '"' {
		s = s[1 : len(s)-1]
	}
	return s
}


1	package logreader
2
3	import (
4	"bufio"
5	"encoding/json"
6	"fmt"
7	"io"
8	"regexp"
9	"strings"
10	"time"
11	)
12
13	// EntryKind classifies a log entry.
14	type EntryKind int
15
16	const (
17	KindSlog EntryKind = iota // time=... level=... structured log line (fallback)
18	KindInfo // level=INFO
19	KindWarn // level=WARN
20	KindError // level=ERROR
21	KindDebug // level=DEBUG
22	KindIncoming // INCOMING REQUEST from Claude Code
23	KindOutgoing // OUTGOING REQUEST to upstream API
24	KindResponse // UPSTREAM RESPONSE
25	KindStream // STREAM RESPONSE FROM UPSTREAM
26	)
27
28	func (k EntryKind) String() string {
29	switch k {
30	case KindSlog:
31	return "LOG"
32	case KindInfo:
33	return "INFO"
34	case KindWarn:
35	return "WARN"
36	case KindError:
37	return "ERROR"
38	case KindDebug:
39	return "DEBUG"
40	case KindIncoming:
41	return "REQ IN"
42	case KindOutgoing:
43	return "REQ OUT"
44	case KindResponse:
45	return "RESPONSE"
46	case KindStream:
47	return "STREAM"
48	default:
49	return "UNKNOWN"
50	}
51	}
52
53	// IsSlog returns true for any slog-derived entry kind.
54	func (k EntryKind) IsSlog() bool {
55	return k == KindSlog \|\| k == KindInfo \|\| k == KindWarn \|\| k == KindError \|\| k == KindDebug
56	}
57
58	func slogKind(level string) EntryKind {
59	switch strings.ToUpper(level) {
60	case "INFO":
61	return KindInfo
62	case "WARN", "WARNING":
63	return KindWarn
64	case "ERROR":
65	return KindError
66	case "DEBUG":
67	return KindDebug
68	default:
69	return KindSlog
70	}
71	}
72
73	// Header is a parsed HTTP header key-value pair.
74	type Header struct {
75	Key string
76	Value string
77	}
78
79	// StreamData holds parsed SSE stream information.
80	type StreamData struct {
81	Model string
82	Thinking string // reassembled reasoning content
83	Content string // reassembled output content
84	Chunks int
85	Usage *StreamUsage
86	StopReason string
87	}
88
89	// StreamUsage holds token usage from the stream.
90	type StreamUsage struct {
91	PromptTokens int `json:"prompt_tokens"`
92	CompletionTokens int `json:"completion_tokens"`
93	TotalTokens int `json:"total_tokens"`
94	CachedTokens int `json:"cached_tokens,omitempty"`
95	}
96
97	// Entry is a single parsed log entry.
98	type Entry struct {
99	Kind EntryKind
100	Time time.Time
101	Label string // short summary for the list
102	Content string // full raw text
103	SepAfter bool // render a visual separator after this entry in the list
104
105	// Structured HTTP fields (populated for request/response kinds).
106	RequestLine string // e.g. "POST /v1/messages HTTP/1.1" or "HTTP/2.0 200 OK"
107	Headers []Header // parsed headers
108	Body string // pretty-printed JSON body (if available), otherwise raw body
109
110	// Structured stream fields (populated for KindStream).
111	Stream *StreamData
112	}
113
114	var (
115	bannerRe = regexp.MustCompile(`^\[(.+?)\]\s+(.+)$`)
116	slogRe = regexp.MustCompile(`^time=(\S+)\s+level=(\S+)\s+msg=(.*)$`)
117	sepLine = strings.Repeat("=", 80)
118	footLine = strings.Repeat("-", 80)
119	)
120
121	// Parse reads a log file and returns parsed entries.
122	func Parse(r io.Reader) []Entry {
123	scanner := bufio.NewScanner(r)
124	scanner.Buffer(make([]byte, 0, 10241024), 101024*1024)
125
126	var entries []Entry
127
128	for scanner.Scan() {
129	line := scanner.Text()
130
131	// Check for slog line — each one becomes its own entry.
132	if slogRe.MatchString(line) {
133	t, lvl, msg := parseSlogLine(line)
134	entries = append(entries, Entry{
135	Kind: slogKind(lvl),
136	Time: t,
137	Label: msg,
138	Content: line,
139	})
140	continue
141	}
142
143	// Check for banner separator start.
144	if line == sepLine {
145	entry := parseBannerEntry(scanner)
146	if entry != nil {
147	entries = append(entries, *entry)
148	}
149	continue
150	}
151	}
152
153	// Mark "proxy logging stopped" entries that are followed by more entries
154	// as separators between sessions.
155	for i := range entries {
156	if entries[i].Kind.IsSlog() &&
157	strings.Contains(entries[i].Label, "proxy logging stopped") &&
158	i < len(entries)-1 {
159	entries[i].SepAfter = true
160	}
161	}
162
163	return entries
164	}
165
166	func parseBannerEntry(scanner bufio.Scanner) Entry {
167	// Next line should be the label line: [timestamp] LABEL
168	if !scanner.Scan() {
169	return nil
170	}
171	labelLine := scanner.Text()
172
173	m := bannerRe.FindStringSubmatch(labelLine)
174	if m == nil {
175	return nil
176	}
177	ts, _ := time.Parse(time.RFC3339Nano, m[1])
178	label := m[2]
179
180	// Skip the closing separator (another === line).
181	scanner.Scan()
182
183	// Determine kind.
184	kind := classifyLabel(label)
185
186	// Read content until footer (---) or next banner (===).
187	var content strings.Builder
188	for scanner.Scan() {
189	line := scanner.Text()
190	if line == footLine {
191	break
192	}
193	if line == sepLine {
194	// We hit the next banner — this entry had no footer.
195	// We need to parse this as a new banner, but scanner already consumed
196	// the line. We'll just break and let the outer loop handle it on next iteration.
197	// Unfortunately we lose this separator. Instead, let's include a note.
198	break
199	}
200	content.WriteString(line)
201	content.WriteByte('\n')
202	}
203
204	raw := content.String()
205	summary := buildSummary(kind, raw)
206
207	entry := &Entry{
208	Kind: kind,
209	Time: ts,
210	Label: fmt.Sprintf("%s — %s", label, summary),
211	Content: raw,
212	}
213
214	// Parse structured HTTP parts for request/response entries.
215	if kind == KindIncoming \|\| kind == KindOutgoing \|\| kind == KindResponse {
216	entry.RequestLine, entry.Headers, entry.Body = parseHTTPContent(raw)
217	}
218
219	// Parse stream data.
220	if kind == KindStream {
221	entry.Stream = parseStreamContent(raw)
222	}
223
224	return entry
225	}
226
227	func classifyLabel(label string) EntryKind {
228	switch {
229	case strings.Contains(label, "INCOMING REQUEST"):
230	return KindIncoming
231	case strings.Contains(label, "OUTGOING REQUEST"):
232	return KindOutgoing
233	case strings.HasPrefix(label, "STREAM"):
234	return KindStream
235	case strings.Contains(label, "RESPONSE"):
236	return KindResponse
237	default:
238	return KindSlog
239	}
240	}
241
242	func buildSummary(kind EntryKind, content string) string {
243	lines := strings.SplitN(content, "\n", 5)
244	switch kind {
245	case KindIncoming, KindOutgoing:
246	// First line is like "POST /v1/messages?beta=true HTTP/1.1"
247	if len(lines) > 0 {
248	return truncate(lines[0], 60)
249	}
250	case KindResponse:
251	// First line is like "HTTP/2.0 200 OK"
252	if len(lines) > 0 {
253	return truncate(lines[0], 60)
254	}
255	case KindStream:
256	// Count data: lines.
257	count := 0
258	for l := range strings.SplitSeq(content, "\n") {
259	if strings.HasPrefix(l, "data: ") {
260	count++
261	}
262	}
263	return fmt.Sprintf("%d chunks", count)
264	}
265	return ""
266	}
267
268	const prettyJSONMarker = "--- Pretty JSON ---"
269
270	// parseHTTPContent splits raw HTTP dump into request/status line, headers, and body.
271	// If a "--- Pretty JSON ---" section exists, it is used as the body instead of raw.
272	func parseHTTPContent(raw string) (requestLine string, headers []Header, body string) {
273	lines := strings.Split(raw, "\n")
274	if len(lines) == 0 {
275	return "", nil, ""
276	}
277
278	// First non-empty line is the request/status line.
279	i := 0
280	for i < len(lines) && strings.TrimSpace(lines[i]) == "" {
281	i++
282	}
283	if i >= len(lines) {
284	return "", nil, ""
285	}
286	requestLine = strings.TrimSpace(lines[i])
287	i++
288
289	// Parse headers until empty line.
290	for i < len(lines) {
291	line := lines[i]
292	i++
293	if strings.TrimSpace(line) == "" {
294	break
295	}
296	if k, v, ok := strings.Cut(line, ":"); ok {
297	headers = append(headers, Header{
298	Key: strings.TrimSpace(k),
299	Value: strings.TrimSpace(v),
300	})
301	}
302	}
303
304	// Check if there's a pretty JSON section — use that as body.
305	_, after, ok := strings.Cut(raw, prettyJSONMarker)
306	if ok {
307	body = strings.TrimSpace(after)
308	return
309	}
310
311	// Otherwise use whatever is after the empty line as body.
312	if i < len(lines) {
313	body = strings.TrimSpace(strings.Join(lines[i:], "\n"))
314	}
315	return
316	}
317
318	// sseChunk represents a single SSE data chunk (OpenAI chat completion format).
319	type sseChunk struct {
320	Model string `json:"model"`
321	Choices []struct {
322	Delta struct {
323	Content *string `json:"content"`
324	Reasoning *string `json:"reasoning"`
325	ReasoningContent *string `json:"reasoning_content"`
326	} `json:"delta"`
327	FinishReason *string `json:"finish_reason"`
328	} `json:"choices"`
329	Usage *StreamUsage `json:"usage"`
330	}
331
332	func parseStreamContent(raw string) *StreamData {
333	sd := &StreamData{}
334	var thinking, content strings.Builder
335
336	for line := range strings.SplitSeq(raw, "\n") {
337	if !strings.HasPrefix(line, "data: ") {
338	continue
339	}
340	payload := strings.TrimPrefix(line, "data: ")
341	if payload == "[DONE]" {
342	continue
343	}
344
345	var chunk sseChunk
346	if json.Unmarshal([]byte(payload), &chunk) != nil {
347	continue
348	}
349
350	sd.Chunks++
351	if sd.Model == "" && chunk.Model != "" {
352	sd.Model = chunk.Model
353	}
354
355	if len(chunk.Choices) > 0 {
356	c := chunk.Choices[0]
357	// Prefer reasoning_content over reasoning.
358	if c.Delta.ReasoningContent != nil && *c.Delta.ReasoningContent != "" {
359	thinking.WriteString(*c.Delta.ReasoningContent)
360	} else if c.Delta.Reasoning != nil && *c.Delta.Reasoning != "" {
361	thinking.WriteString(*c.Delta.Reasoning)
362	}
363	if c.Delta.Content != nil {
364	content.WriteString(*c.Delta.Content)
365	}
366	if c.FinishReason != nil {
367	sd.StopReason = *c.FinishReason
368	}
369	}
370
371	if chunk.Usage != nil {
372	sd.Usage = chunk.Usage
373	}
374	}
375
376	sd.Thinking = thinking.String()
377	sd.Content = content.String()
378	return sd
379	}
380
381	func truncate(s string, max int) string {
382	if len(s) <= max {
383	return s
384	}
385	return s[:max-3] + "..."
386	}
387
388	func parseSlogLine(line string) (time.Time, string, string) {
389	m := slogRe.FindStringSubmatch(line)
390	if m == nil {
391	return time.Time{}, "", line
392	}
393	t, _ := time.Parse(time.RFC3339Nano, m[1])
394	return t, m[2], unquote(m[3])
395	}
396
397	func unquote(s string) string {
398	s = strings.TrimSpace(s)
399	if len(s) >= 2 && s[0] == '"' && s[len(s)-1] == '"' {
400	s = s[1 : len(s)-1]
401	}
402	return s
403	}
404

parser.go

Source Files