…lines 1–288 unchanged (288 lines)
289 js, err = glob(filepath.Join(path, "**/*.json"), "text/plain")
290 if err != nil {
291 return nil, err
292 }
293 files = append(files, js...)
294
295 // only include tokenizer.model is tokenizer.json is not present
296 if !slices.ContainsFunc(files, func(s string) bool {
297 return slices.Contains(strings.Split(s, string(os.PathSeparator)), "tokenizer.json")
298 }) {
299 if tks, _ := glob(filepath.Join(path, "tokenizer.model"), "application/octet-stream"); len(tks) > 0 {
300 // add tokenizer.model if it exists, tokenizer.json is automatically picked up by the previous glob
301 // tokenizer.model might be a unresolved git lfs reference; error if it is
302 files = append(files, tks...)
303 } else if tks, _ := glob(filepath.Join(path, "**/tokenizer.model"), "text/plain"); len(tks) > 0 {
304 // some times tokenizer.model is in a subdirectory (e.g. meta-llama/Meta-Llama-3-8B)
305 files = append(files, tks...)
306 }
307 }
308
309 return files, nil
310}
311
312type Command struct {
313 Name string
…lines 314–655 unchanged (342 lines)