export_models.go

v0.7.0

package cmd

import (
	"encoding/csv"
	"fmt"
	"os"
	"sort"
	"strings"

	"github.com/spf13/cobra"
	"sourcecraft.dev/bigbes/claudio/config"
	"sourcecraft.dev/bigbes/claudio/provider"
	"sourcecraft.dev/bigbes/claudio/provider/copilot"
	"sourcecraft.dev/bigbes/claudio/provider/deepseek"
	"sourcecraft.dev/bigbes/claudio/provider/kimi"
	"sourcecraft.dev/bigbes/claudio/provider/mistral"
	"sourcecraft.dev/bigbes/claudio/provider/nvidia"
	"sourcecraft.dev/bigbes/claudio/provider/ollama"
	"sourcecraft.dev/bigbes/claudio/provider/openrouter"
)

var exportModelsOutput string

var exportModelsCmd = &cobra.Command{
	Use:   "export-models",
	Short: "Export the full model catalog of every cloud provider to a CSV file",
	Long: `Fetch the full model catalog from every cloud provider that exposes a
live /models endpoint and write the union to a CSV file.

Providers hit live (API key required, pulled from the config file):
  deepseek, mistral, nvidia, kimi, kimi-api-cn, kimi-api-intl, copilot,
  ollama-cloud

Providers hit live without a key:
  openrouter

Providers with a curated static catalog (no /models endpoint):
  zai, zai-coding

Local, passthrough, and config-only providers (ollama, lmstudio, vkproxy,
claude, custom) are skipped — this command is about cloud model catalogs.`,
	RunE: runExportModels,
}

func init() {
	exportModelsCmd.Flags().StringVarP(&exportModelsOutput, "output", "o", "models.csv", "Path to output CSV file")
	rootCmd.AddCommand(exportModelsCmd)
}

type exportedModel struct {
	providerKey   string
	providerName  string
	modelID       string
	displayName   string
	family        string
	vendor        string
	contextWindow int
	source        string // "api" | "static"
}

func runExportModels(cmd *cobra.Command, args []string) error {
	cfg, err := config.Load()
	if err != nil {
		return fmt.Errorf("loading config: %w", err)
	}

	var rows []exportedModel
	for _, key := range provider.Order {
		p, ok := provider.Registry[key]
		if !ok {
			continue
		}
		pc := cfg.Providers[key]
		models, err := fetchCloudModels(key, p, pc)
		if err != nil {
			fmt.Fprintf(os.Stderr, "skip %s: %v\n", key, err)
			continue
		}
		for i := range models {
			models[i].providerKey = key
			models[i].providerName = p.Name
		}
		rows = append(rows, models...)
		fmt.Fprintf(os.Stderr, "ok   %s: %d models\n", key, len(models))
	}

	rows = dedupAndSort(rows)

	f, err := os.Create(exportModelsOutput)
	if err != nil {
		return fmt.Errorf("creating %s: %w", exportModelsOutput, err)
	}
	defer f.Close()

	w := csv.NewWriter(f)
	header := []string{"provider", "provider_name", "model_id", "display_name", "family", "vendor", "context_window", "source"}
	if err := w.Write(header); err != nil {
		return fmt.Errorf("writing header: %w", err)
	}
	for _, r := range rows {
		ctx := ""
		if r.contextWindow > 0 {
			ctx = fmt.Sprintf("%d", r.contextWindow)
		}
		if err := w.Write([]string{
			r.providerKey, r.providerName, r.modelID, r.displayName, r.family, r.vendor, ctx, r.source,
		}); err != nil {
			return fmt.Errorf("writing row: %w", err)
		}
	}
	w.Flush()
	if err := w.Error(); err != nil {
		return fmt.Errorf("flushing CSV: %w", err)
	}

	fmt.Printf("Wrote %d models to %s\n", len(rows), exportModelsOutput)
	return nil
}

// fetchCloudModels returns the full model catalog for a cloud provider.
// Returns an error when the provider is unsupported (local, passthrough) or
// when credentials are missing for one that requires them.
func fetchCloudModels(key string, p provider.Provider, pc config.ProviderConfig) ([]exportedModel, error) {
	baseURL := pc.BaseURL
	if baseURL == "" {
		baseURL = p.BaseURL
	}

	switch key {
	case "deepseek":
		if pc.APIKey == "" {
			return nil, fmt.Errorf("no API key configured")
		}
		ms, err := deepseek.ListModels(pc.APIKey)
		if err != nil {
			return nil, err
		}
		out := make([]exportedModel, 0, len(ms))
		for _, m := range ms {
			out = append(out, exportedModel{modelID: m.ID, displayName: m.DisplayName(), source: "api"})
		}
		return out, nil

	case "mistral":
		if pc.APIKey == "" {
			return nil, fmt.Errorf("no API key configured")
		}
		ms, err := mistral.ListModels(pc.APIKey)
		if err != nil {
			return nil, err
		}
		out := make([]exportedModel, 0, len(ms))
		for _, m := range ms {
			out = append(out, exportedModel{modelID: m.ID, displayName: m.DisplayName(), source: "api"})
		}
		return out, nil

	case "nvidia":
		if pc.APIKey == "" {
			return nil, fmt.Errorf("no API key configured")
		}
		ms, err := nvidia.ListModels(pc.APIKey)
		if err != nil {
			return nil, err
		}
		out := make([]exportedModel, 0, len(ms))
		for _, m := range ms {
			out = append(out, exportedModel{modelID: m.ID, displayName: m.DisplayName(), vendor: m.OwnedBy, source: "api"})
		}
		return out, nil

	case "openrouter":
		ms, err := openrouter.ListModels(pc.APIKey)
		if err != nil {
			return nil, err
		}
		out := make([]exportedModel, 0, len(ms))
		for _, m := range ms {
			out = append(out, exportedModel{
				modelID:       m.ID,
				displayName:   m.DisplayName(),
				contextWindow: m.ContextLength,
				source:        "api",
			})
		}
		return out, nil

	case "kimi-api-cn", "kimi-api-intl":
		if pc.APIKey == "" {
			return nil, fmt.Errorf("no API key configured")
		}
		base := baseURL
		if !strings.HasSuffix(base, "/") {
			base += "/"
		}
		ms, err := kimi.ListModels(base, pc.APIKey)
		if err != nil {
			return nil, err
		}
		out := make([]exportedModel, 0, len(ms))
		for _, m := range ms {
			out = append(out, exportedModel{
				modelID:       m.ID,
				displayName:   m.GetName(),
				contextWindow: m.ContextLength,
				source:        "api",
			})
		}
		return out, nil

	case "copilot":
		if pc.APIKey == "" {
			return nil, fmt.Errorf("no OAuth token — run 'claudio config' to sign in")
		}
		ms, err := copilot.ListModels(pc.APIKey)
		if err != nil {
			return nil, err
		}
		out := make([]exportedModel, 0, len(ms))
		for _, m := range ms {
			out = append(out, exportedModel{
				modelID:       m.ID,
				displayName:   m.DisplayName(),
				vendor:        m.Vendor,
				contextWindow: m.ContextWindow,
				source:        "api",
			})
		}
		return out, nil

	case "ollama-cloud":
		ms, err := ollama.ListModels(baseURL, pc.APIKey)
		if err != nil {
			return nil, err
		}
		out := make([]exportedModel, 0, len(ms))
		for _, m := range ms {
			out = append(out, exportedModel{
				modelID:       m.Name,
				displayName:   m.DisplayName(),
				family:        m.Family(),
				contextWindow: m.ContextLength,
				source:        "api",
			})
		}
		return out, nil

	case "zai", "zai-coding":
		sm := provider.StaticModels[key]
		if len(sm) == 0 {
			return nil, fmt.Errorf("no static catalog")
		}
		out := make([]exportedModel, 0, len(sm))
		for _, m := range sm {
			out = append(out, exportedModel{
				modelID:       m.ID,
				displayName:   m.Name,
				family:        m.Family,
				contextWindow: p.ContextWindow,
				source:        "static",
			})
		}
		return out, nil
	}

	return nil, fmt.Errorf("no cloud catalog endpoint")
}

func dedupAndSort(rows []exportedModel) []exportedModel {
	sort.SliceStable(rows, func(i, j int) bool {
		if rows[i].providerKey != rows[j].providerKey {
			return rows[i].providerKey < rows[j].providerKey
		}
		return rows[i].modelID < rows[j].modelID
	})
	seen := make(map[string]bool)
	out := rows[:0]
	for _, r := range rows {
		k := r.providerKey + "\x00" + r.modelID
		if seen[k] {
			continue
		}
		seen[k] = true
		out = append(out, r)
	}
	return out
}


1	package cmd
2
3	import (
4	"encoding/csv"
5	"fmt"
6	"os"
7	"sort"
8	"strings"
9
10	"github.com/spf13/cobra"
11	"sourcecraft.dev/bigbes/claudio/config"
12	"sourcecraft.dev/bigbes/claudio/provider"
13	"sourcecraft.dev/bigbes/claudio/provider/copilot"
14	"sourcecraft.dev/bigbes/claudio/provider/deepseek"
15	"sourcecraft.dev/bigbes/claudio/provider/kimi"
16	"sourcecraft.dev/bigbes/claudio/provider/mistral"
17	"sourcecraft.dev/bigbes/claudio/provider/nvidia"
18	"sourcecraft.dev/bigbes/claudio/provider/ollama"
19	"sourcecraft.dev/bigbes/claudio/provider/openrouter"
20	)
21
22	var exportModelsOutput string
23
24	var exportModelsCmd = &cobra.Command{
25	Use: "export-models",
26	Short: "Export the full model catalog of every cloud provider to a CSV file",
27	Long: `Fetch the full model catalog from every cloud provider that exposes a
28	live /models endpoint and write the union to a CSV file.
29
30	Providers hit live (API key required, pulled from the config file):
31	deepseek, mistral, nvidia, kimi, kimi-api-cn, kimi-api-intl, copilot,
32	ollama-cloud
33
34	Providers hit live without a key:
35	openrouter
36
37	Providers with a curated static catalog (no /models endpoint):
38	zai, zai-coding
39
40	Local, passthrough, and config-only providers (ollama, lmstudio, vkproxy,
41	claude, custom) are skipped — this command is about cloud model catalogs.`,
42	RunE: runExportModels,
43	}
44
45	func init() {
46	exportModelsCmd.Flags().StringVarP(&exportModelsOutput, "output", "o", "models.csv", "Path to output CSV file")
47	rootCmd.AddCommand(exportModelsCmd)
48	}
49
50	type exportedModel struct {
51	providerKey string
52	providerName string
53	modelID string
54	displayName string
55	family string
56	vendor string
57	contextWindow int
58	source string // "api" \| "static"
59	}
60
61	func runExportModels(cmd *cobra.Command, args []string) error {
62	cfg, err := config.Load()
63	if err != nil {
64	return fmt.Errorf("loading config: %w", err)
65	}
66
67	var rows []exportedModel
68	for _, key := range provider.Order {
69	p, ok := provider.Registry[key]
70	if !ok {
71	continue
72	}
73	pc := cfg.Providers[key]
74	models, err := fetchCloudModels(key, p, pc)
75	if err != nil {
76	fmt.Fprintf(os.Stderr, "skip %s: %v\n", key, err)
77	continue
78	}
79	for i := range models {
80	models[i].providerKey = key
81	models[i].providerName = p.Name
82	}
83	rows = append(rows, models...)
84	fmt.Fprintf(os.Stderr, "ok %s: %d models\n", key, len(models))
85	}
86
87	rows = dedupAndSort(rows)
88
89	f, err := os.Create(exportModelsOutput)
90	if err != nil {
91	return fmt.Errorf("creating %s: %w", exportModelsOutput, err)
92	}
93	defer f.Close()
94
95	w := csv.NewWriter(f)
96	header := []string{"provider", "provider_name", "model_id", "display_name", "family", "vendor", "context_window", "source"}
97	if err := w.Write(header); err != nil {
98	return fmt.Errorf("writing header: %w", err)
99	}
100	for _, r := range rows {
101	ctx := ""
102	if r.contextWindow > 0 {
103	ctx = fmt.Sprintf("%d", r.contextWindow)
104	}
105	if err := w.Write([]string{
106	r.providerKey, r.providerName, r.modelID, r.displayName, r.family, r.vendor, ctx, r.source,
107	}); err != nil {
108	return fmt.Errorf("writing row: %w", err)
109	}
110	}
111	w.Flush()
112	if err := w.Error(); err != nil {
113	return fmt.Errorf("flushing CSV: %w", err)
114	}
115
116	fmt.Printf("Wrote %d models to %s\n", len(rows), exportModelsOutput)
117	return nil
118	}
119
120	// fetchCloudModels returns the full model catalog for a cloud provider.
121	// Returns an error when the provider is unsupported (local, passthrough) or
122	// when credentials are missing for one that requires them.
123	func fetchCloudModels(key string, p provider.Provider, pc config.ProviderConfig) ([]exportedModel, error) {
124	baseURL := pc.BaseURL
125	if baseURL == "" {
126	baseURL = p.BaseURL
127	}
128
129	switch key {
130	case "deepseek":
131	if pc.APIKey == "" {
132	return nil, fmt.Errorf("no API key configured")
133	}
134	ms, err := deepseek.ListModels(pc.APIKey)
135	if err != nil {
136	return nil, err
137	}
138	out := make([]exportedModel, 0, len(ms))
139	for _, m := range ms {
140	out = append(out, exportedModel{modelID: m.ID, displayName: m.DisplayName(), source: "api"})
141	}
142	return out, nil
143
144	case "mistral":
145	if pc.APIKey == "" {
146	return nil, fmt.Errorf("no API key configured")
147	}
148	ms, err := mistral.ListModels(pc.APIKey)
149	if err != nil {
150	return nil, err
151	}
152	out := make([]exportedModel, 0, len(ms))
153	for _, m := range ms {
154	out = append(out, exportedModel{modelID: m.ID, displayName: m.DisplayName(), source: "api"})
155	}
156	return out, nil
157
158	case "nvidia":
159	if pc.APIKey == "" {
160	return nil, fmt.Errorf("no API key configured")
161	}
162	ms, err := nvidia.ListModels(pc.APIKey)
163	if err != nil {
164	return nil, err
165	}
166	out := make([]exportedModel, 0, len(ms))
167	for _, m := range ms {
168	out = append(out, exportedModel{modelID: m.ID, displayName: m.DisplayName(), vendor: m.OwnedBy, source: "api"})
169	}
170	return out, nil
171
172	case "openrouter":
173	ms, err := openrouter.ListModels(pc.APIKey)
174	if err != nil {
175	return nil, err
176	}
177	out := make([]exportedModel, 0, len(ms))
178	for _, m := range ms {
179	out = append(out, exportedModel{
180	modelID: m.ID,
181	displayName: m.DisplayName(),
182	contextWindow: m.ContextLength,
183	source: "api",
184	})
185	}
186	return out, nil
187
188	case "kimi-api-cn", "kimi-api-intl":
189	if pc.APIKey == "" {
190	return nil, fmt.Errorf("no API key configured")
191	}
192	base := baseURL
193	if !strings.HasSuffix(base, "/") {
194	base += "/"
195	}
196	ms, err := kimi.ListModels(base, pc.APIKey)
197	if err != nil {
198	return nil, err
199	}
200	out := make([]exportedModel, 0, len(ms))
201	for _, m := range ms {
202	out = append(out, exportedModel{
203	modelID: m.ID,
204	displayName: m.GetName(),
205	contextWindow: m.ContextLength,
206	source: "api",
207	})
208	}
209	return out, nil
210
211	case "copilot":
212	if pc.APIKey == "" {
213	return nil, fmt.Errorf("no OAuth token — run 'claudio config' to sign in")
214	}
215	ms, err := copilot.ListModels(pc.APIKey)
216	if err != nil {
217	return nil, err
218	}
219	out := make([]exportedModel, 0, len(ms))
220	for _, m := range ms {
221	out = append(out, exportedModel{
222	modelID: m.ID,
223	displayName: m.DisplayName(),
224	vendor: m.Vendor,
225	contextWindow: m.ContextWindow,
226	source: "api",
227	})
228	}
229	return out, nil
230
231	case "ollama-cloud":
232	ms, err := ollama.ListModels(baseURL, pc.APIKey)
233	if err != nil {
234	return nil, err
235	}
236	out := make([]exportedModel, 0, len(ms))
237	for _, m := range ms {
238	out = append(out, exportedModel{
239	modelID: m.Name,
240	displayName: m.DisplayName(),
241	family: m.Family(),
242	contextWindow: m.ContextLength,
243	source: "api",
244	})
245	}
246	return out, nil
247
248	case "zai", "zai-coding":
249	sm := provider.StaticModels[key]
250	if len(sm) == 0 {
251	return nil, fmt.Errorf("no static catalog")
252	}
253	out := make([]exportedModel, 0, len(sm))
254	for _, m := range sm {
255	out = append(out, exportedModel{
256	modelID: m.ID,
257	displayName: m.Name,
258	family: m.Family,
259	contextWindow: p.ContextWindow,
260	source: "static",
261	})
262	}
263	return out, nil
264	}
265
266	return nil, fmt.Errorf("no cloud catalog endpoint")
267	}
268
269	func dedupAndSort(rows []exportedModel) []exportedModel {
270	sort.SliceStable(rows, func(i, j int) bool {
271	if rows[i].providerKey != rows[j].providerKey {
272	return rows[i].providerKey < rows[j].providerKey
273	}
274	return rows[i].modelID < rows[j].modelID
275	})
276	seen := make(map[string]bool)
277	out := rows[:0]
278	for _, r := range rows {
279	k := r.providerKey + "\x00" + r.modelID
280	if seen[k] {
281	continue
282	}
283	seen[k] = true
284	out = append(out, r)
285	}
286	return out
287	}
288

export_models.go

Source Files