parser.go

v0.6.1

package logreader

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"regexp"
	"strings"
	"time"
)

// EntryKind classifies a log entry.
type EntryKind int

const (
	KindSlog     EntryKind = iota // time=... level=... structured log line (fallback)
	KindInfo                      // level=INFO
	KindWarn                      // level=WARN
	KindError                     // level=ERROR
	KindDebug                     // level=DEBUG
	KindIncoming                  // INCOMING REQUEST from Claude Code
	KindOutgoing                  // OUTGOING REQUEST to upstream API
	KindResponse                  // UPSTREAM RESPONSE
	KindStream                    // STREAM RESPONSE FROM UPSTREAM
)

func (k EntryKind) String() string {
	switch k {
	case KindSlog:
		return "LOG"
	case KindInfo:
		return "INFO"
	case KindWarn:
		return "WARN"
	case KindError:
		return "ERROR"
	case KindDebug:
		return "DEBUG"
	case KindIncoming:
		return "REQ IN"
	case KindOutgoing:
		return "REQ OUT"
	case KindResponse:
		return "RESPONSE"
	case KindStream:
		return "STREAM"
	default:
		return "UNKNOWN"
	}
}

// IsSlog returns true for any slog-derived entry kind.
func (k EntryKind) IsSlog() bool {
	return k == KindSlog || k == KindInfo || k == KindWarn || k == KindError || k == KindDebug
}

func slogKind(level string) EntryKind {
	switch strings.ToUpper(level) {
	case "INFO":
		return KindInfo
	case "WARN", "WARNING":
		return KindWarn
	case "ERROR":
		return KindError
	case "DEBUG":
		return KindDebug
	default:
		return KindSlog
	}
}

// Header is a parsed HTTP header key-value pair.
type Header struct {
	Key   string
	Value string
}

// StreamData holds parsed SSE stream information.
type StreamData struct {
	Model      string
	Thinking   string // reassembled reasoning content
	Content    string // reassembled output content
	Chunks     int
	Usage      *StreamUsage
	StopReason string
}

// StreamUsage holds token usage from the stream.
type StreamUsage struct {
	PromptTokens     int `json:"prompt_tokens"`
	CompletionTokens int `json:"completion_tokens"`
	TotalTokens      int `json:"total_tokens"`
}

// Entry is a single parsed log entry.
type Entry struct {
	Kind     EntryKind
	Time     time.Time
	Label    string // short summary for the list
	Content  string // full raw text
	SepAfter bool   // render a visual separator after this entry in the list

	// Structured HTTP fields (populated for request/response kinds).
	RequestLine string   // e.g. "POST /v1/messages HTTP/1.1" or "HTTP/2.0 200 OK"
	Headers     []Header // parsed headers
	Body        string   // pretty-printed JSON body (if available), otherwise raw body

	// Structured stream fields (populated for KindStream).
	Stream *StreamData
}

var (
	bannerRe = regexp.MustCompile(`^\[(.+?)\]\s+(.+)$`)
	slogRe   = regexp.MustCompile(`^time=(\S+)\s+level=(\S+)\s+msg=(.*)$`)
	sepLine  = strings.Repeat("=", 80)
	footLine = strings.Repeat("-", 80)
)

// Parse reads a log file and returns parsed entries.
func Parse(r io.Reader) []Entry {
	scanner := bufio.NewScanner(r)
	scanner.Buffer(make([]byte, 0, 1024*1024), 10*1024*1024)

	var entries []Entry

	for scanner.Scan() {
		line := scanner.Text()

		// Check for slog line — each one becomes its own entry.
		if slogRe.MatchString(line) {
			t, lvl, msg := parseSlogLine(line)
			entries = append(entries, Entry{
				Kind:    slogKind(lvl),
				Time:    t,
				Label:   msg,
				Content: line,
			})
			continue
		}

		// Check for banner separator start.
		if line == sepLine {
			entry := parseBannerEntry(scanner)
			if entry != nil {
				entries = append(entries, *entry)
			}
			continue
		}
	}

	// Mark "proxy logging stopped" entries that are followed by more entries
	// as separators between sessions.
	for i := range entries {
		if entries[i].Kind.IsSlog() &&
			strings.Contains(entries[i].Label, "proxy logging stopped") &&
			i < len(entries)-1 {
			entries[i].SepAfter = true
		}
	}

	return entries
}

func parseBannerEntry(scanner *bufio.Scanner) *Entry {
	// Next line should be the label line: [timestamp] LABEL
	if !scanner.Scan() {
		return nil
	}
	labelLine := scanner.Text()

	m := bannerRe.FindStringSubmatch(labelLine)
	if m == nil {
		return nil
	}
	ts, _ := time.Parse(time.RFC3339Nano, m[1])
	label := m[2]

	// Skip the closing separator (another === line).
	scanner.Scan()

	// Determine kind.
	kind := classifyLabel(label)

	// Read content until footer (---) or next banner (===).
	var content strings.Builder
	for scanner.Scan() {
		line := scanner.Text()
		if line == footLine {
			break
		}
		if line == sepLine {
			// We hit the next banner — this entry had no footer.
			// We need to parse this as a new banner, but scanner already consumed
			// the line. We'll just break and let the outer loop handle it on next iteration.
			// Unfortunately we lose this separator. Instead, let's include a note.
			break
		}
		content.WriteString(line)
		content.WriteByte('\n')
	}

	raw := content.String()
	summary := buildSummary(kind, raw)

	entry := &Entry{
		Kind:    kind,
		Time:    ts,
		Label:   fmt.Sprintf("%s — %s", label, summary),
		Content: raw,
	}

	// Parse structured HTTP parts for request/response entries.
	if kind == KindIncoming || kind == KindOutgoing || kind == KindResponse {
		entry.RequestLine, entry.Headers, entry.Body = parseHTTPContent(raw)
	}

	// Parse stream data.
	if kind == KindStream {
		entry.Stream = parseStreamContent(raw)
	}

	return entry
}

func classifyLabel(label string) EntryKind {
	switch {
	case strings.Contains(label, "INCOMING REQUEST"):
		return KindIncoming
	case strings.Contains(label, "OUTGOING REQUEST"):
		return KindOutgoing
	case strings.HasPrefix(label, "STREAM"):
		return KindStream
	case strings.Contains(label, "RESPONSE"):
		return KindResponse
	default:
		return KindSlog
	}
}

func buildSummary(kind EntryKind, content string) string {
	lines := strings.SplitN(content, "\n", 5)
	switch kind {
	case KindIncoming, KindOutgoing:
		// First line is like "POST /v1/messages?beta=true HTTP/1.1"
		if len(lines) > 0 {
			return truncate(lines[0], 60)
		}
	case KindResponse:
		// First line is like "HTTP/2.0 200 OK"
		if len(lines) > 0 {
			return truncate(lines[0], 60)
		}
	case KindStream:
		// Count data: lines.
		count := 0
		for l := range strings.SplitSeq(content, "\n") {
			if strings.HasPrefix(l, "data: ") {
				count++
			}
		}
		return fmt.Sprintf("%d chunks", count)
	}
	return ""
}

const prettyJSONMarker = "--- Pretty JSON ---"

// parseHTTPContent splits raw HTTP dump into request/status line, headers, and body.
// If a "--- Pretty JSON ---" section exists, it is used as the body instead of raw.
func parseHTTPContent(raw string) (requestLine string, headers []Header, body string) {
	lines := strings.Split(raw, "\n")
	if len(lines) == 0 {
		return "", nil, ""
	}

	// First non-empty line is the request/status line.
	i := 0
	for i < len(lines) && strings.TrimSpace(lines[i]) == "" {
		i++
	}
	if i >= len(lines) {
		return "", nil, ""
	}
	requestLine = strings.TrimSpace(lines[i])
	i++

	// Parse headers until empty line.
	for i < len(lines) {
		line := lines[i]
		i++
		if strings.TrimSpace(line) == "" {
			break
		}
		if k, v, ok := strings.Cut(line, ":"); ok {
			headers = append(headers, Header{
				Key:   strings.TrimSpace(k),
				Value: strings.TrimSpace(v),
			})
		}
	}

	// Check if there's a pretty JSON section — use that as body.
	_, after, ok := strings.Cut(raw, prettyJSONMarker)
	if ok {
		body = strings.TrimSpace(after)
		return
	}

	// Otherwise use whatever is after the empty line as body.
	if i < len(lines) {
		body = strings.TrimSpace(strings.Join(lines[i:], "\n"))
	}
	return
}

// sseChunk represents a single SSE data chunk (OpenAI chat completion format).
type sseChunk struct {
	Model   string `json:"model"`
	Choices []struct {
		Delta struct {
			Content          *string `json:"content"`
			Reasoning        *string `json:"reasoning"`
			ReasoningContent *string `json:"reasoning_content"`
		} `json:"delta"`
		FinishReason *string `json:"finish_reason"`
	} `json:"choices"`
	Usage *StreamUsage `json:"usage"`
}

func parseStreamContent(raw string) *StreamData {
	sd := &StreamData{}
	var thinking, content strings.Builder

	for line := range strings.SplitSeq(raw, "\n") {
		if !strings.HasPrefix(line, "data: ") {
			continue
		}
		payload := strings.TrimPrefix(line, "data: ")
		if payload == "[DONE]" {
			continue
		}

		var chunk sseChunk
		if json.Unmarshal([]byte(payload), &chunk) != nil {
			continue
		}

		sd.Chunks++
		if sd.Model == "" && chunk.Model != "" {
			sd.Model = chunk.Model
		}

		if len(chunk.Choices) > 0 {
			c := chunk.Choices[0]
			// Prefer reasoning_content over reasoning.
			if c.Delta.ReasoningContent != nil && *c.Delta.ReasoningContent != "" {
				thinking.WriteString(*c.Delta.ReasoningContent)
			} else if c.Delta.Reasoning != nil && *c.Delta.Reasoning != "" {
				thinking.WriteString(*c.Delta.Reasoning)
			}
			if c.Delta.Content != nil {
				content.WriteString(*c.Delta.Content)
			}
			if c.FinishReason != nil {
				sd.StopReason = *c.FinishReason
			}
		}

		if chunk.Usage != nil {
			sd.Usage = chunk.Usage
		}
	}

	sd.Thinking = thinking.String()
	sd.Content = content.String()
	return sd
}

func truncate(s string, max int) string {
	if len(s) <= max {
		return s
	}
	return s[:max-3] + "..."
}

func parseSlogLine(line string) (time.Time, string, string) {
	m := slogRe.FindStringSubmatch(line)
	if m == nil {
		return time.Time{}, "", line
	}
	t, _ := time.Parse(time.RFC3339Nano, m[1])
	return t, m[2], unquote(m[3])
}

func unquote(s string) string {
	s = strings.TrimSpace(s)
	if len(s) >= 2 && s[0] == '"' && s[len(s)-1] == '"' {
		s = s[1 : len(s)-1]
	}
	return s
}


1	package logreader
2
3	import (
4	"bufio"
5	"encoding/json"
6	"fmt"
7	"io"
8	"regexp"
9	"strings"
10	"time"
11	)
12
13	// EntryKind classifies a log entry.
14	type EntryKind int
15
16	const (
17	KindSlog EntryKind = iota // time=... level=... structured log line (fallback)
18	KindInfo // level=INFO
19	KindWarn // level=WARN
20	KindError // level=ERROR
21	KindDebug // level=DEBUG
22	KindIncoming // INCOMING REQUEST from Claude Code
23	KindOutgoing // OUTGOING REQUEST to upstream API
24	KindResponse // UPSTREAM RESPONSE
25	KindStream // STREAM RESPONSE FROM UPSTREAM
26	)
27
28	func (k EntryKind) String() string {
29	switch k {
30	case KindSlog:
31	return "LOG"
32	case KindInfo:
33	return "INFO"
34	case KindWarn:
35	return "WARN"
36	case KindError:
37	return "ERROR"
38	case KindDebug:
39	return "DEBUG"
40	case KindIncoming:
41	return "REQ IN"
42	case KindOutgoing:
43	return "REQ OUT"
44	case KindResponse:
45	return "RESPONSE"
46	case KindStream:
47	return "STREAM"
48	default:
49	return "UNKNOWN"
50	}
51	}
52
53	// IsSlog returns true for any slog-derived entry kind.
54	func (k EntryKind) IsSlog() bool {
55	return k == KindSlog \|\| k == KindInfo \|\| k == KindWarn \|\| k == KindError \|\| k == KindDebug
56	}
57
58	func slogKind(level string) EntryKind {
59	switch strings.ToUpper(level) {
60	case "INFO":
61	return KindInfo
62	case "WARN", "WARNING":
63	return KindWarn
64	case "ERROR":
65	return KindError
66	case "DEBUG":
67	return KindDebug
68	default:
69	return KindSlog
70	}
71	}
72
73	// Header is a parsed HTTP header key-value pair.
74	type Header struct {
75	Key string
76	Value string
77	}
78
79	// StreamData holds parsed SSE stream information.
80	type StreamData struct {
81	Model string
82	Thinking string // reassembled reasoning content
83	Content string // reassembled output content
84	Chunks int
85	Usage *StreamUsage
86	StopReason string
87	}
88
89	// StreamUsage holds token usage from the stream.
90	type StreamUsage struct {
91	PromptTokens int `json:"prompt_tokens"`
92	CompletionTokens int `json:"completion_tokens"`
93	TotalTokens int `json:"total_tokens"`
94	}
95
96	// Entry is a single parsed log entry.
97	type Entry struct {
98	Kind EntryKind
99	Time time.Time
100	Label string // short summary for the list
101	Content string // full raw text
102	SepAfter bool // render a visual separator after this entry in the list
103
104	// Structured HTTP fields (populated for request/response kinds).
105	RequestLine string // e.g. "POST /v1/messages HTTP/1.1" or "HTTP/2.0 200 OK"
106	Headers []Header // parsed headers
107	Body string // pretty-printed JSON body (if available), otherwise raw body
108
109	// Structured stream fields (populated for KindStream).
110	Stream *StreamData
111	}
112
113	var (
114	bannerRe = regexp.MustCompile(`^\[(.+?)\]\s+(.+)$`)
115	slogRe = regexp.MustCompile(`^time=(\S+)\s+level=(\S+)\s+msg=(.*)$`)
116	sepLine = strings.Repeat("=", 80)
117	footLine = strings.Repeat("-", 80)
118	)
119
120	// Parse reads a log file and returns parsed entries.
121	func Parse(r io.Reader) []Entry {
122	scanner := bufio.NewScanner(r)
123	scanner.Buffer(make([]byte, 0, 10241024), 101024*1024)
124
125	var entries []Entry
126
127	for scanner.Scan() {
128	line := scanner.Text()
129
130	// Check for slog line — each one becomes its own entry.
131	if slogRe.MatchString(line) {
132	t, lvl, msg := parseSlogLine(line)
133	entries = append(entries, Entry{
134	Kind: slogKind(lvl),
135	Time: t,
136	Label: msg,
137	Content: line,
138	})
139	continue
140	}
141
142	// Check for banner separator start.
143	if line == sepLine {
144	entry := parseBannerEntry(scanner)
145	if entry != nil {
146	entries = append(entries, *entry)
147	}
148	continue
149	}
150	}
151
152	// Mark "proxy logging stopped" entries that are followed by more entries
153	// as separators between sessions.
154	for i := range entries {
155	if entries[i].Kind.IsSlog() &&
156	strings.Contains(entries[i].Label, "proxy logging stopped") &&
157	i < len(entries)-1 {
158	entries[i].SepAfter = true
159	}
160	}
161
162	return entries
163	}
164
165	func parseBannerEntry(scanner bufio.Scanner) Entry {
166	// Next line should be the label line: [timestamp] LABEL
167	if !scanner.Scan() {
168	return nil
169	}
170	labelLine := scanner.Text()
171
172	m := bannerRe.FindStringSubmatch(labelLine)
173	if m == nil {
174	return nil
175	}
176	ts, _ := time.Parse(time.RFC3339Nano, m[1])
177	label := m[2]
178
179	// Skip the closing separator (another === line).
180	scanner.Scan()
181
182	// Determine kind.
183	kind := classifyLabel(label)
184
185	// Read content until footer (---) or next banner (===).
186	var content strings.Builder
187	for scanner.Scan() {
188	line := scanner.Text()
189	if line == footLine {
190	break
191	}
192	if line == sepLine {
193	// We hit the next banner — this entry had no footer.
194	// We need to parse this as a new banner, but scanner already consumed
195	// the line. We'll just break and let the outer loop handle it on next iteration.
196	// Unfortunately we lose this separator. Instead, let's include a note.
197	break
198	}
199	content.WriteString(line)
200	content.WriteByte('\n')
201	}
202
203	raw := content.String()
204	summary := buildSummary(kind, raw)
205
206	entry := &Entry{
207	Kind: kind,
208	Time: ts,
209	Label: fmt.Sprintf("%s — %s", label, summary),
210	Content: raw,
211	}
212
213	// Parse structured HTTP parts for request/response entries.
214	if kind == KindIncoming \|\| kind == KindOutgoing \|\| kind == KindResponse {
215	entry.RequestLine, entry.Headers, entry.Body = parseHTTPContent(raw)
216	}
217
218	// Parse stream data.
219	if kind == KindStream {
220	entry.Stream = parseStreamContent(raw)
221	}
222
223	return entry
224	}
225
226	func classifyLabel(label string) EntryKind {
227	switch {
228	case strings.Contains(label, "INCOMING REQUEST"):
229	return KindIncoming
230	case strings.Contains(label, "OUTGOING REQUEST"):
231	return KindOutgoing
232	case strings.HasPrefix(label, "STREAM"):
233	return KindStream
234	case strings.Contains(label, "RESPONSE"):
235	return KindResponse
236	default:
237	return KindSlog
238	}
239	}
240
241	func buildSummary(kind EntryKind, content string) string {
242	lines := strings.SplitN(content, "\n", 5)
243	switch kind {
244	case KindIncoming, KindOutgoing:
245	// First line is like "POST /v1/messages?beta=true HTTP/1.1"
246	if len(lines) > 0 {
247	return truncate(lines[0], 60)
248	}
249	case KindResponse:
250	// First line is like "HTTP/2.0 200 OK"
251	if len(lines) > 0 {
252	return truncate(lines[0], 60)
253	}
254	case KindStream:
255	// Count data: lines.
256	count := 0
257	for l := range strings.SplitSeq(content, "\n") {
258	if strings.HasPrefix(l, "data: ") {
259	count++
260	}
261	}
262	return fmt.Sprintf("%d chunks", count)
263	}
264	return ""
265	}
266
267	const prettyJSONMarker = "--- Pretty JSON ---"
268
269	// parseHTTPContent splits raw HTTP dump into request/status line, headers, and body.
270	// If a "--- Pretty JSON ---" section exists, it is used as the body instead of raw.
271	func parseHTTPContent(raw string) (requestLine string, headers []Header, body string) {
272	lines := strings.Split(raw, "\n")
273	if len(lines) == 0 {
274	return "", nil, ""
275	}
276
277	// First non-empty line is the request/status line.
278	i := 0
279	for i < len(lines) && strings.TrimSpace(lines[i]) == "" {
280	i++
281	}
282	if i >= len(lines) {
283	return "", nil, ""
284	}
285	requestLine = strings.TrimSpace(lines[i])
286	i++
287
288	// Parse headers until empty line.
289	for i < len(lines) {
290	line := lines[i]
291	i++
292	if strings.TrimSpace(line) == "" {
293	break
294	}
295	if k, v, ok := strings.Cut(line, ":"); ok {
296	headers = append(headers, Header{
297	Key: strings.TrimSpace(k),
298	Value: strings.TrimSpace(v),
299	})
300	}
301	}
302
303	// Check if there's a pretty JSON section — use that as body.
304	_, after, ok := strings.Cut(raw, prettyJSONMarker)
305	if ok {
306	body = strings.TrimSpace(after)
307	return
308	}
309
310	// Otherwise use whatever is after the empty line as body.
311	if i < len(lines) {
312	body = strings.TrimSpace(strings.Join(lines[i:], "\n"))
313	}
314	return
315	}
316
317	// sseChunk represents a single SSE data chunk (OpenAI chat completion format).
318	type sseChunk struct {
319	Model string `json:"model"`
320	Choices []struct {
321	Delta struct {
322	Content *string `json:"content"`
323	Reasoning *string `json:"reasoning"`
324	ReasoningContent *string `json:"reasoning_content"`
325	} `json:"delta"`
326	FinishReason *string `json:"finish_reason"`
327	} `json:"choices"`
328	Usage *StreamUsage `json:"usage"`
329	}
330
331	func parseStreamContent(raw string) *StreamData {
332	sd := &StreamData{}
333	var thinking, content strings.Builder
334
335	for line := range strings.SplitSeq(raw, "\n") {
336	if !strings.HasPrefix(line, "data: ") {
337	continue
338	}
339	payload := strings.TrimPrefix(line, "data: ")
340	if payload == "[DONE]" {
341	continue
342	}
343
344	var chunk sseChunk
345	if json.Unmarshal([]byte(payload), &chunk) != nil {
346	continue
347	}
348
349	sd.Chunks++
350	if sd.Model == "" && chunk.Model != "" {
351	sd.Model = chunk.Model
352	}
353
354	if len(chunk.Choices) > 0 {
355	c := chunk.Choices[0]
356	// Prefer reasoning_content over reasoning.
357	if c.Delta.ReasoningContent != nil && *c.Delta.ReasoningContent != "" {
358	thinking.WriteString(*c.Delta.ReasoningContent)
359	} else if c.Delta.Reasoning != nil && *c.Delta.Reasoning != "" {
360	thinking.WriteString(*c.Delta.Reasoning)
361	}
362	if c.Delta.Content != nil {
363	content.WriteString(*c.Delta.Content)
364	}
365	if c.FinishReason != nil {
366	sd.StopReason = *c.FinishReason
367	}
368	}
369
370	if chunk.Usage != nil {
371	sd.Usage = chunk.Usage
372	}
373	}
374
375	sd.Thinking = thinking.String()
376	sd.Content = content.String()
377	return sd
378	}
379
380	func truncate(s string, max int) string {
381	if len(s) <= max {
382	return s
383	}
384	return s[:max-3] + "..."
385	}
386
387	func parseSlogLine(line string) (time.Time, string, string) {
388	m := slogRe.FindStringSubmatch(line)
389	if m == nil {
390	return time.Time{}, "", line
391	}
392	t, _ := time.Parse(time.RFC3339Nano, m[1])
393	return t, m[2], unquote(m[3])
394	}
395
396	func unquote(s string) string {
397	s = strings.TrimSpace(s)
398	if len(s) >= 2 && s[0] == '"' && s[len(s)-1] == '"' {
399	s = s[1 : len(s)-1]
400	}
401	return s
402	}
403

parser.go

Source Files