parser.go

v0.6.0

package logreader

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"regexp"
	"strings"
	"time"
)

// EntryKind classifies a log entry.
type EntryKind int

const (
	KindSlog     EntryKind = iota // time=... level=... structured log line (fallback)
	KindInfo                      // level=INFO
	KindWarn                      // level=WARN
	KindError                     // level=ERROR
	KindDebug                     // level=DEBUG
	KindIncoming                  // INCOMING REQUEST from Claude Code
	KindOutgoing                  // OUTGOING REQUEST to upstream API
	KindResponse                  // UPSTREAM RESPONSE
	KindStream                    // STREAM RESPONSE FROM UPSTREAM
)

func (k EntryKind) String() string {
	switch k {
	case KindSlog:
		return "LOG"
	case KindInfo:
		return "INFO"
	case KindWarn:
		return "WARN"
	case KindError:
		return "ERROR"
	case KindDebug:
		return "DEBUG"
	case KindIncoming:
		return "REQ IN"
	case KindOutgoing:
		return "REQ OUT"
	case KindResponse:
		return "RESPONSE"
	case KindStream:
		return "STREAM"
	default:
		return "UNKNOWN"
	}
}

// IsSlog returns true for any slog-derived entry kind.
func (k EntryKind) IsSlog() bool {
	return k == KindSlog || k == KindInfo || k == KindWarn || k == KindError || k == KindDebug
}

func slogKind(level string) EntryKind {
	switch strings.ToUpper(level) {
	case "INFO":
		return KindInfo
	case "WARN", "WARNING":
		return KindWarn
	case "ERROR":
		return KindError
	case "DEBUG":
		return KindDebug
	default:
		return KindSlog
	}
}

// Header is a parsed HTTP header key-value pair.
type Header struct {
	Key   string
	Value string
}

// StreamData holds parsed SSE stream information.
type StreamData struct {
	Model     string
	Thinking  string // reassembled reasoning content
	Content   string // reassembled output content
	Chunks    int
	Usage     *StreamUsage
	StopReason string
}

// StreamUsage holds token usage from the stream.
type StreamUsage struct {
	PromptTokens     int `json:"prompt_tokens"`
	CompletionTokens int `json:"completion_tokens"`
	TotalTokens      int `json:"total_tokens"`
}

// Entry is a single parsed log entry.
type Entry struct {
	Kind     EntryKind
	Time     time.Time
	Label    string // short summary for the list
	Content  string // full raw text
	SepAfter bool   // render a visual separator after this entry in the list

	// Structured HTTP fields (populated for request/response kinds).
	RequestLine string   // e.g. "POST /v1/messages HTTP/1.1" or "HTTP/2.0 200 OK"
	Headers     []Header // parsed headers
	Body        string   // pretty-printed JSON body (if available), otherwise raw body

	// Structured stream fields (populated for KindStream).
	Stream *StreamData
}

var (
	bannerRe = regexp.MustCompile(`^\[(.+?)\]\s+(.+)$`)
	slogRe   = regexp.MustCompile(`^time=(\S+)\s+level=(\S+)\s+msg=(.*)$`)
	sepLine  = strings.Repeat("=", 80)
	footLine = strings.Repeat("-", 80)
)

// Parse reads a log file and returns parsed entries.
func Parse(r io.Reader) []Entry {
	scanner := bufio.NewScanner(r)
	scanner.Buffer(make([]byte, 0, 1024*1024), 10*1024*1024)

	var entries []Entry

	for scanner.Scan() {
		line := scanner.Text()

		// Check for slog line — each one becomes its own entry.
		if slogRe.MatchString(line) {
			t, lvl, msg := parseSlogLine(line)
			entries = append(entries, Entry{
				Kind:    slogKind(lvl),
				Time:    t,
				Label:   msg,
				Content: line,
			})
			continue
		}

		// Check for banner separator start.
		if line == sepLine {
			entry := parseBannerEntry(scanner)
			if entry != nil {
				entries = append(entries, *entry)
			}
			continue
		}
	}

	// Mark "proxy logging stopped" entries that are followed by more entries
	// as separators between sessions.
	for i := range entries {
		if entries[i].Kind.IsSlog() &&
			strings.Contains(entries[i].Label, "proxy logging stopped") &&
			i < len(entries)-1 {
			entries[i].SepAfter = true
		}
	}

	return entries
}

func parseBannerEntry(scanner *bufio.Scanner) *Entry {
	// Next line should be the label line: [timestamp] LABEL
	if !scanner.Scan() {
		return nil
	}
	labelLine := scanner.Text()

	m := bannerRe.FindStringSubmatch(labelLine)
	if m == nil {
		return nil
	}
	ts, _ := time.Parse(time.RFC3339Nano, m[1])
	label := m[2]

	// Skip the closing separator.
	if scanner.Scan() {
		// Should be another === line, just consume it.
	}

	// Determine kind.
	kind := classifyLabel(label)

	// Read content until footer (---) or next banner (===).
	var content strings.Builder
	for scanner.Scan() {
		line := scanner.Text()
		if line == footLine {
			break
		}
		if line == sepLine {
			// We hit the next banner — this entry had no footer.
			// We need to parse this as a new banner, but scanner already consumed
			// the line. We'll just break and let the outer loop handle it on next iteration.
			// Unfortunately we lose this separator. Instead, let's include a note.
			break
		}
		content.WriteString(line)
		content.WriteByte('\n')
	}

	raw := content.String()
	summary := buildSummary(kind, raw)

	entry := &Entry{
		Kind:    kind,
		Time:    ts,
		Label:   fmt.Sprintf("%s — %s", label, summary),
		Content: raw,
	}

	// Parse structured HTTP parts for request/response entries.
	if kind == KindIncoming || kind == KindOutgoing || kind == KindResponse {
		entry.RequestLine, entry.Headers, entry.Body = parseHTTPContent(raw)
	}

	// Parse stream data.
	if kind == KindStream {
		entry.Stream = parseStreamContent(raw)
	}

	return entry
}

func classifyLabel(label string) EntryKind {
	switch {
	case strings.Contains(label, "INCOMING REQUEST"):
		return KindIncoming
	case strings.Contains(label, "OUTGOING REQUEST"):
		return KindOutgoing
	case strings.HasPrefix(label, "STREAM"):
		return KindStream
	case strings.Contains(label, "RESPONSE"):
		return KindResponse
	default:
		return KindSlog
	}
}

func buildSummary(kind EntryKind, content string) string {
	lines := strings.SplitN(content, "\n", 5)
	switch kind {
	case KindIncoming, KindOutgoing:
		// First line is like "POST /v1/messages?beta=true HTTP/1.1"
		if len(lines) > 0 {
			return truncate(lines[0], 60)
		}
	case KindResponse:
		// First line is like "HTTP/2.0 200 OK"
		if len(lines) > 0 {
			return truncate(lines[0], 60)
		}
	case KindStream:
		// Count data: lines.
		count := 0
		for _, l := range strings.Split(content, "\n") {
			if strings.HasPrefix(l, "data: ") {
				count++
			}
		}
		return fmt.Sprintf("%d chunks", count)
	}
	return ""
}

const prettyJSONMarker = "--- Pretty JSON ---"

// parseHTTPContent splits raw HTTP dump into request/status line, headers, and body.
// If a "--- Pretty JSON ---" section exists, it is used as the body instead of raw.
func parseHTTPContent(raw string) (requestLine string, headers []Header, body string) {
	lines := strings.Split(raw, "\n")
	if len(lines) == 0 {
		return "", nil, ""
	}

	// First non-empty line is the request/status line.
	i := 0
	for i < len(lines) && strings.TrimSpace(lines[i]) == "" {
		i++
	}
	if i >= len(lines) {
		return "", nil, ""
	}
	requestLine = strings.TrimSpace(lines[i])
	i++

	// Parse headers until empty line.
	for i < len(lines) {
		line := lines[i]
		i++
		if strings.TrimSpace(line) == "" {
			break
		}
		if k, v, ok := strings.Cut(line, ":"); ok {
			headers = append(headers, Header{
				Key:   strings.TrimSpace(k),
				Value: strings.TrimSpace(v),
			})
		}
	}

	// Check if there's a pretty JSON section — use that as body.
	prettyIdx := strings.Index(raw, prettyJSONMarker)
	if prettyIdx >= 0 {
		body = strings.TrimSpace(raw[prettyIdx+len(prettyJSONMarker):])
		return
	}

	// Otherwise use whatever is after the empty line as body.
	if i < len(lines) {
		body = strings.TrimSpace(strings.Join(lines[i:], "\n"))
	}
	return
}

// sseChunk represents a single SSE data chunk (OpenAI chat completion format).
type sseChunk struct {
	Model   string `json:"model"`
	Choices []struct {
		Delta struct {
			Content          *string `json:"content"`
			Reasoning        *string `json:"reasoning"`
			ReasoningContent *string `json:"reasoning_content"`
		} `json:"delta"`
		FinishReason *string `json:"finish_reason"`
	} `json:"choices"`
	Usage *StreamUsage `json:"usage"`
}

func parseStreamContent(raw string) *StreamData {
	sd := &StreamData{}
	var thinking, content strings.Builder

	for _, line := range strings.Split(raw, "\n") {
		if !strings.HasPrefix(line, "data: ") {
			continue
		}
		payload := strings.TrimPrefix(line, "data: ")
		if payload == "[DONE]" {
			continue
		}

		var chunk sseChunk
		if json.Unmarshal([]byte(payload), &chunk) != nil {
			continue
		}

		sd.Chunks++
		if sd.Model == "" && chunk.Model != "" {
			sd.Model = chunk.Model
		}

		if len(chunk.Choices) > 0 {
			c := chunk.Choices[0]
			// Prefer reasoning_content over reasoning.
			if c.Delta.ReasoningContent != nil && *c.Delta.ReasoningContent != "" {
				thinking.WriteString(*c.Delta.ReasoningContent)
			} else if c.Delta.Reasoning != nil && *c.Delta.Reasoning != "" {
				thinking.WriteString(*c.Delta.Reasoning)
			}
			if c.Delta.Content != nil {
				content.WriteString(*c.Delta.Content)
			}
			if c.FinishReason != nil {
				sd.StopReason = *c.FinishReason
			}
		}

		if chunk.Usage != nil {
			sd.Usage = chunk.Usage
		}
	}

	sd.Thinking = thinking.String()
	sd.Content = content.String()
	return sd
}

func truncate(s string, max int) string {
	if len(s) <= max {
		return s
	}
	return s[:max-3] + "..."
}

func parseSlogLine(line string) (time.Time, string, string) {
	m := slogRe.FindStringSubmatch(line)
	if m == nil {
		return time.Time{}, "", line
	}
	t, _ := time.Parse(time.RFC3339Nano, m[1])
	return t, m[2], unquote(m[3])
}

func unquote(s string) string {
	s = strings.TrimSpace(s)
	if len(s) >= 2 && s[0] == '"' && s[len(s)-1] == '"' {
		s = s[1 : len(s)-1]
	}
	return s
}


1	package logreader
2
3	import (
4	"bufio"
5	"encoding/json"
6	"fmt"
7	"io"
8	"regexp"
9	"strings"
10	"time"
11	)
12
13	// EntryKind classifies a log entry.
14	type EntryKind int
15
16	const (
17	KindSlog EntryKind = iota // time=... level=... structured log line (fallback)
18	KindInfo // level=INFO
19	KindWarn // level=WARN
20	KindError // level=ERROR
21	KindDebug // level=DEBUG
22	KindIncoming // INCOMING REQUEST from Claude Code
23	KindOutgoing // OUTGOING REQUEST to upstream API
24	KindResponse // UPSTREAM RESPONSE
25	KindStream // STREAM RESPONSE FROM UPSTREAM
26	)
27
28	func (k EntryKind) String() string {
29	switch k {
30	case KindSlog:
31	return "LOG"
32	case KindInfo:
33	return "INFO"
34	case KindWarn:
35	return "WARN"
36	case KindError:
37	return "ERROR"
38	case KindDebug:
39	return "DEBUG"
40	case KindIncoming:
41	return "REQ IN"
42	case KindOutgoing:
43	return "REQ OUT"
44	case KindResponse:
45	return "RESPONSE"
46	case KindStream:
47	return "STREAM"
48	default:
49	return "UNKNOWN"
50	}
51	}
52
53	// IsSlog returns true for any slog-derived entry kind.
54	func (k EntryKind) IsSlog() bool {
55	return k == KindSlog \|\| k == KindInfo \|\| k == KindWarn \|\| k == KindError \|\| k == KindDebug
56	}
57
58	func slogKind(level string) EntryKind {
59	switch strings.ToUpper(level) {
60	case "INFO":
61	return KindInfo
62	case "WARN", "WARNING":
63	return KindWarn
64	case "ERROR":
65	return KindError
66	case "DEBUG":
67	return KindDebug
68	default:
69	return KindSlog
70	}
71	}
72
73	// Header is a parsed HTTP header key-value pair.
74	type Header struct {
75	Key string
76	Value string
77	}
78
79	// StreamData holds parsed SSE stream information.
80	type StreamData struct {
81	Model string
82	Thinking string // reassembled reasoning content
83	Content string // reassembled output content
84	Chunks int
85	Usage *StreamUsage
86	StopReason string
87	}
88
89	// StreamUsage holds token usage from the stream.
90	type StreamUsage struct {
91	PromptTokens int `json:"prompt_tokens"`
92	CompletionTokens int `json:"completion_tokens"`
93	TotalTokens int `json:"total_tokens"`
94	}
95
96	// Entry is a single parsed log entry.
97	type Entry struct {
98	Kind EntryKind
99	Time time.Time
100	Label string // short summary for the list
101	Content string // full raw text
102	SepAfter bool // render a visual separator after this entry in the list
103
104	// Structured HTTP fields (populated for request/response kinds).
105	RequestLine string // e.g. "POST /v1/messages HTTP/1.1" or "HTTP/2.0 200 OK"
106	Headers []Header // parsed headers
107	Body string // pretty-printed JSON body (if available), otherwise raw body
108
109	// Structured stream fields (populated for KindStream).
110	Stream *StreamData
111	}
112
113	var (
114	bannerRe = regexp.MustCompile(`^\[(.+?)\]\s+(.+)$`)
115	slogRe = regexp.MustCompile(`^time=(\S+)\s+level=(\S+)\s+msg=(.*)$`)
116	sepLine = strings.Repeat("=", 80)
117	footLine = strings.Repeat("-", 80)
118	)
119
120	// Parse reads a log file and returns parsed entries.
121	func Parse(r io.Reader) []Entry {
122	scanner := bufio.NewScanner(r)
123	scanner.Buffer(make([]byte, 0, 10241024), 101024*1024)
124
125	var entries []Entry
126
127	for scanner.Scan() {
128	line := scanner.Text()
129
130	// Check for slog line — each one becomes its own entry.
131	if slogRe.MatchString(line) {
132	t, lvl, msg := parseSlogLine(line)
133	entries = append(entries, Entry{
134	Kind: slogKind(lvl),
135	Time: t,
136	Label: msg,
137	Content: line,
138	})
139	continue
140	}
141
142	// Check for banner separator start.
143	if line == sepLine {
144	entry := parseBannerEntry(scanner)
145	if entry != nil {
146	entries = append(entries, *entry)
147	}
148	continue
149	}
150	}
151
152	// Mark "proxy logging stopped" entries that are followed by more entries
153	// as separators between sessions.
154	for i := range entries {
155	if entries[i].Kind.IsSlog() &&
156	strings.Contains(entries[i].Label, "proxy logging stopped") &&
157	i < len(entries)-1 {
158	entries[i].SepAfter = true
159	}
160	}
161
162	return entries
163	}
164
165	func parseBannerEntry(scanner bufio.Scanner) Entry {
166	// Next line should be the label line: [timestamp] LABEL
167	if !scanner.Scan() {
168	return nil
169	}
170	labelLine := scanner.Text()
171
172	m := bannerRe.FindStringSubmatch(labelLine)
173	if m == nil {
174	return nil
175	}
176	ts, _ := time.Parse(time.RFC3339Nano, m[1])
177	label := m[2]
178
179	// Skip the closing separator.
180	if scanner.Scan() {
181	// Should be another === line, just consume it.
182	}
183
184	// Determine kind.
185	kind := classifyLabel(label)
186
187	// Read content until footer (---) or next banner (===).
188	var content strings.Builder
189	for scanner.Scan() {
190	line := scanner.Text()
191	if line == footLine {
192	break
193	}
194	if line == sepLine {
195	// We hit the next banner — this entry had no footer.
196	// We need to parse this as a new banner, but scanner already consumed
197	// the line. We'll just break and let the outer loop handle it on next iteration.
198	// Unfortunately we lose this separator. Instead, let's include a note.
199	break
200	}
201	content.WriteString(line)
202	content.WriteByte('\n')
203	}
204
205	raw := content.String()
206	summary := buildSummary(kind, raw)
207
208	entry := &Entry{
209	Kind: kind,
210	Time: ts,
211	Label: fmt.Sprintf("%s — %s", label, summary),
212	Content: raw,
213	}
214
215	// Parse structured HTTP parts for request/response entries.
216	if kind == KindIncoming \|\| kind == KindOutgoing \|\| kind == KindResponse {
217	entry.RequestLine, entry.Headers, entry.Body = parseHTTPContent(raw)
218	}
219
220	// Parse stream data.
221	if kind == KindStream {
222	entry.Stream = parseStreamContent(raw)
223	}
224
225	return entry
226	}
227
228	func classifyLabel(label string) EntryKind {
229	switch {
230	case strings.Contains(label, "INCOMING REQUEST"):
231	return KindIncoming
232	case strings.Contains(label, "OUTGOING REQUEST"):
233	return KindOutgoing
234	case strings.HasPrefix(label, "STREAM"):
235	return KindStream
236	case strings.Contains(label, "RESPONSE"):
237	return KindResponse
238	default:
239	return KindSlog
240	}
241	}
242
243	func buildSummary(kind EntryKind, content string) string {
244	lines := strings.SplitN(content, "\n", 5)
245	switch kind {
246	case KindIncoming, KindOutgoing:
247	// First line is like "POST /v1/messages?beta=true HTTP/1.1"
248	if len(lines) > 0 {
249	return truncate(lines[0], 60)
250	}
251	case KindResponse:
252	// First line is like "HTTP/2.0 200 OK"
253	if len(lines) > 0 {
254	return truncate(lines[0], 60)
255	}
256	case KindStream:
257	// Count data: lines.
258	count := 0
259	for _, l := range strings.Split(content, "\n") {
260	if strings.HasPrefix(l, "data: ") {
261	count++
262	}
263	}
264	return fmt.Sprintf("%d chunks", count)
265	}
266	return ""
267	}
268
269	const prettyJSONMarker = "--- Pretty JSON ---"
270
271	// parseHTTPContent splits raw HTTP dump into request/status line, headers, and body.
272	// If a "--- Pretty JSON ---" section exists, it is used as the body instead of raw.
273	func parseHTTPContent(raw string) (requestLine string, headers []Header, body string) {
274	lines := strings.Split(raw, "\n")
275	if len(lines) == 0 {
276	return "", nil, ""
277	}
278
279	// First non-empty line is the request/status line.
280	i := 0
281	for i < len(lines) && strings.TrimSpace(lines[i]) == "" {
282	i++
283	}
284	if i >= len(lines) {
285	return "", nil, ""
286	}
287	requestLine = strings.TrimSpace(lines[i])
288	i++
289
290	// Parse headers until empty line.
291	for i < len(lines) {
292	line := lines[i]
293	i++
294	if strings.TrimSpace(line) == "" {
295	break
296	}
297	if k, v, ok := strings.Cut(line, ":"); ok {
298	headers = append(headers, Header{
299	Key: strings.TrimSpace(k),
300	Value: strings.TrimSpace(v),
301	})
302	}
303	}
304
305	// Check if there's a pretty JSON section — use that as body.
306	prettyIdx := strings.Index(raw, prettyJSONMarker)
307	if prettyIdx >= 0 {
308	body = strings.TrimSpace(raw[prettyIdx+len(prettyJSONMarker):])
309	return
310	}
311
312	// Otherwise use whatever is after the empty line as body.
313	if i < len(lines) {
314	body = strings.TrimSpace(strings.Join(lines[i:], "\n"))
315	}
316	return
317	}
318
319	// sseChunk represents a single SSE data chunk (OpenAI chat completion format).
320	type sseChunk struct {
321	Model string `json:"model"`
322	Choices []struct {
323	Delta struct {
324	Content *string `json:"content"`
325	Reasoning *string `json:"reasoning"`
326	ReasoningContent *string `json:"reasoning_content"`
327	} `json:"delta"`
328	FinishReason *string `json:"finish_reason"`
329	} `json:"choices"`
330	Usage *StreamUsage `json:"usage"`
331	}
332
333	func parseStreamContent(raw string) *StreamData {
334	sd := &StreamData{}
335	var thinking, content strings.Builder
336
337	for _, line := range strings.Split(raw, "\n") {
338	if !strings.HasPrefix(line, "data: ") {
339	continue
340	}
341	payload := strings.TrimPrefix(line, "data: ")
342	if payload == "[DONE]" {
343	continue
344	}
345
346	var chunk sseChunk
347	if json.Unmarshal([]byte(payload), &chunk) != nil {
348	continue
349	}
350
351	sd.Chunks++
352	if sd.Model == "" && chunk.Model != "" {
353	sd.Model = chunk.Model
354	}
355
356	if len(chunk.Choices) > 0 {
357	c := chunk.Choices[0]
358	// Prefer reasoning_content over reasoning.
359	if c.Delta.ReasoningContent != nil && *c.Delta.ReasoningContent != "" {
360	thinking.WriteString(*c.Delta.ReasoningContent)
361	} else if c.Delta.Reasoning != nil && *c.Delta.Reasoning != "" {
362	thinking.WriteString(*c.Delta.Reasoning)
363	}
364	if c.Delta.Content != nil {
365	content.WriteString(*c.Delta.Content)
366	}
367	if c.FinishReason != nil {
368	sd.StopReason = *c.FinishReason
369	}
370	}
371
372	if chunk.Usage != nil {
373	sd.Usage = chunk.Usage
374	}
375	}
376
377	sd.Thinking = thinking.String()
378	sd.Content = content.String()
379	return sd
380	}
381
382	func truncate(s string, max int) string {
383	if len(s) <= max {
384	return s
385	}
386	return s[:max-3] + "..."
387	}
388
389	func parseSlogLine(line string) (time.Time, string, string) {
390	m := slogRe.FindStringSubmatch(line)
391	if m == nil {
392	return time.Time{}, "", line
393	}
394	t, _ := time.Parse(time.RFC3339Nano, m[1])
395	return t, m[2], unquote(m[3])
396	}
397
398	func unquote(s string) string {
399	s = strings.TrimSpace(s)
400	if len(s) >= 2 && s[0] == '"' && s[len(s)-1] == '"' {
401	s = s[1 : len(s)-1]
402	}
403	return s
404	}
405

parser.go

Source Files