Skip to content

fix: Prevent lm-eval command escape #503

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ruivieira wants to merge 3 commits into
base: main
Choose a base branch
from
Open

fix: Prevent lm-eval command escape #503

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0a62ed6
fix: Prevent lm-eval command escape
ruivieira Jun 16, 2025
35b67f3
fix: Args test expectation
ruivieira Jun 20, 2025
87b2673
fix: Device patching
ruivieira Jun 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 29 additions & 11 deletions api/lmes/v1alpha1/lmevaljob_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,13 +61,16 @@ const (
)

type Arg struct {
Name string `json:"name"`
// +kubebuilder:validation:Pattern=`^[a-zA-Z0-9._-]+$`
Name string `json:"name"`
// +kubebuilder:validation:Pattern=`^[a-zA-Z0-9._/:\- ]*$`
Value string `json:"value,omitempty"`
}

type Card struct {
// Unitxt card's ID
// +optional
// +kubebuilder:validation:Pattern=`^[a-zA-Z0-9._-]+$`
Name string `json:"name,omitempty"`
// A JSON string for a custom unitxt card which contains the custom dataset.
// Use the documentation here: https://www.unitxt.ai/en/latest/docs/adding_dataset.html#adding-to-the-catalog
Expand All @@ -79,20 +82,24 @@ type Card struct {
type Template struct {
// Unitxt template ID
// +optional
// +kubebuilder:validation:Pattern=`^[a-zA-Z0-9._-]+$`
Name string `json:"name,omitempty"`
// The name of the custom template in the custom field. Its value is a JSON string
// for a custom Unitxt template. Use the documentation here: https://www.unitxt.ai/en/latest/docs/adding_template.html
// to compose a custom template, store it as a JSON file by calling the
// add_to_catalog API: https://www.unitxt.ai/en/latest/docs/saving_and_loading_from_catalog.html#adding-assets-to-the-catalog,
// and use the JSON content as the value here.
// +optional
// +kubebuilder:validation:Pattern=`^[a-zA-Z0-9._-]+$`
Ref string `json:"ref,omitempty"`
}

type SystemPrompt struct {
// Unitxt System Prompt id
// +kubebuilder:validation:Pattern=`^[a-zA-Z0-9._-]+$`
Name string `json:"name,omitempty"`
// The name of the custom systemPrompt in the custom field. Its value is a custom system prompt string
// +kubebuilder:validation:Pattern=`^[a-zA-Z0-9._-]+$`
Ref string `json:"ref,omitempty"`
}

Expand All @@ -112,6 +119,7 @@ type Metric struct {
type Task struct {
// Unitxt task id
// +optional
// +kubebuilder:validation:Pattern=`^[a-zA-Z0-9._-]+$`
Name string `json:"name,omitempty"`
// The name of the custom task in the custom field. Its value is a JSON string
// for a custom Unitxt task. Use the documentation here: https://www.unitxt.ai/en/latest/docs/adding_task.html
Expand All @@ -124,6 +132,7 @@ type Task struct {

type CustomArtifact struct {
// Name of the custom artifact
// +kubebuilder:validation:Pattern=`^[a-zA-Z0-9._-]+$`
Name string `json:"name"`
// Value of the custom artifact. It could be a JSON string or plain text
// depending on the artifact type
Expand Down Expand Up @@ -197,6 +206,7 @@ type TaskRecipe struct {
Metrics []Metric `json:"metrics,omitempty"`
// The Unitxt format
// +optional
// +kubebuilder:validation:Pattern=`^[a-zA-Z0-9._-]+$`
Format *string `json:"format,omitempty"`
// A limit number of records to load
// +optional
Expand All @@ -212,6 +222,7 @@ type TaskRecipe struct {
// GitSource specifies the git location of external tasks
type GitSource struct {
// URL specifies the git repository URL
// +kubebuilder:validation:Pattern=`^https://[a-zA-Z0-9._/-]+$`
URL string `json:"url,omitempty"`
// Branch specifies the git branch to use
// +optional
Expand All @@ -221,6 +232,7 @@ type GitSource struct {
Commit *string `json:"commit,omitempty"`
// Path specifies the path to the task file
// +optional
// +kubebuilder:validation:Pattern=`^[a-zA-Z0-9._/-]*$`
Path string `json:"path,omitempty"`
}

Expand All @@ -238,6 +250,7 @@ type CustomTasks struct {

type TaskList struct {
// TaskNames from lm-eval's task list and/or from custom tasks if CustomTasks is defined
// +kubebuilder:validation:items:Pattern=`^[a-zA-Z0-9._-]+$`
TaskNames []string `json:"taskNames,omitempty"`
// Task Recipes specifically for Unitxt
TaskRecipes []TaskRecipe `json:"taskRecipes,omitempty"`
Expand Down Expand Up @@ -446,14 +459,15 @@ func (p *LMEvalPodSpec) GetSideCards() []corev1.Container {
}

type OfflineS3Spec struct {
AccessKeyIdRef corev1.SecretKeySelector `json:"accessKeyId"`
SecretAccessKeyRef corev1.SecretKeySelector `json:"secretAccessKey"`
Bucket corev1.SecretKeySelector `json:"bucket"`
Path string `json:"path"`
Region corev1.SecretKeySelector `json:"region"`
Endpoint corev1.SecretKeySelector `json:"endpoint"`
VerifySSL *bool `json:"verifySSL,omitempty"`
CABundle *corev1.SecretKeySelector `json:"caBundle,omitempty"`
AccessKeyIdRef corev1.SecretKeySelector `json:"accessKeyId"`
SecretAccessKeyRef corev1.SecretKeySelector `json:"secretAccessKey"`
Bucket corev1.SecretKeySelector `json:"bucket"`
// +kubebuilder:validation:Pattern=`^[a-zA-Z0-9._/-]*$`
Path string `json:"path"`
Region corev1.SecretKeySelector `json:"region"`
Endpoint corev1.SecretKeySelector `json:"endpoint"`
VerifySSL *bool `json:"verifySSL,omitempty"`
CABundle *corev1.SecretKeySelector `json:"caBundle,omitempty"`
}

// OfflineStorageSpec defines the storage configuration for LMEvalJob's offline mode
Expand All @@ -469,8 +483,9 @@ type OfflineSpec struct {

// ChatTemplate defines the configuration for the applied chat template during the evaluation.
type ChatTemplate struct {
Enabled bool `json:"enabled"`
Name string `json:"name,omitempty"`
Enabled bool `json:"enabled"`
// +kubebuilder:validation:Pattern=`^[a-zA-Z0-9._-]+$`
Name string `json:"name,omitempty"`
}

func (p *LMEvalPodSpec) GetAffinity() *corev1.Affinity {
Expand All @@ -493,6 +508,7 @@ type LMEvalJobSpec struct {
// Important: Run "make" to regenerate code after modifying this file

// Model name
// +kubebuilder:validation:Enum=hf;openai-completions;openai-chat-completions;local-completions;local-chat-completions;watsonx_llm;textsynth
Model string `json:"model"`
// Args for the model
// +optional
Expand All @@ -506,6 +522,7 @@ type LMEvalJobSpec struct {
// the number of documents to evaluate to the first X documents (if an integer)
// per task or first X% of documents per task
// +optional
// +kubebuilder:validation:Pattern=`^(\d+\.?\d*|\d*\.\d+)$`
Limit string `json:"limit,omitempty"`
// Map to `--gen_kwargs` parameter for the underlying library.
// +optional
Expand Down Expand Up @@ -539,6 +556,7 @@ type LMEvalJobSpec struct {
AllowCodeExecution *bool `json:"allowCodeExecution,omitempty"`
// SystemInstruction will set the system instruction for all prompts passed to the evaluated model
// +optional
// +kubebuilder:validation:Pattern=`^[a-zA-Z0-9\s.,!?;:()'"_/-]*$`
SystemInstruction string `json:"systemInstruction,omitempty"`
// ChatTemplate defines whether to apply the default or specified chat template to prompts. This is required for chat-completions models.
// +optional
Expand Down
30 changes: 30 additions & 0 deletions config/crd/bases/trustyai.opendatahub.io_lmevaljobs.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ spec:
enabled:
type: boolean
name:
pattern: ^[a-zA-Z0-9._-]+$
type: string
required:
- enabled
Expand All @@ -75,8 +76,10 @@ spec:
items:
properties:
name:
pattern: ^[a-zA-Z0-9._-]+$
type: string
value:
pattern: ^[a-zA-Z0-9._/:\- ]*$
type: string
required:
- name
Expand All @@ -87,6 +90,7 @@ spec:
Accepts an integer, or a float between 0.0 and 1.0 . If passed, will limit
the number of documents to evaluate to the first X documents (if an integer)
per task or first X% of documents per task
pattern: ^(\d+\.?\d*|\d*\.\d+)$
type: string
logSamples:
description: |-
Expand All @@ -95,14 +99,24 @@ spec:
type: boolean
model:
description: Model name
enum:
- hf
- openai-completions
- openai-chat-completions
- local-completions
- local-chat-completions
- watsonx_llm
- textsynth
type: string
modelArgs:
description: Args for the model
items:
properties:
name:
pattern: ^[a-zA-Z0-9._-]+$
type: string
value:
pattern: ^[a-zA-Z0-9._/:\- ]*$
type: string
required:
- name
Expand Down Expand Up @@ -204,6 +218,7 @@ spec:
type: object
x-kubernetes-map-type: atomic
path:
pattern: ^[a-zA-Z0-9._/-]*$
type: string
region:
description: SecretKeySelector selects a key of a Secret.
Expand Down Expand Up @@ -4733,6 +4748,7 @@ spec:
systemInstruction:
description: SystemInstruction will set the system instruction for
all prompts passed to the evaluated model
pattern: ^[a-zA-Z0-9\s.,!?;:()'"_/-]*$
type: string
taskList:
description: Evaluation task list
Expand All @@ -4746,6 +4762,7 @@ spec:
properties:
name:
description: Name of the custom artifact
pattern: ^[a-zA-Z0-9._-]+$
type: string
value:
description: |-
Expand All @@ -4763,6 +4780,7 @@ spec:
properties:
name:
description: Name of the custom artifact
pattern: ^[a-zA-Z0-9._-]+$
type: string
value:
description: |-
Expand All @@ -4780,6 +4798,7 @@ spec:
properties:
name:
description: Name of the custom artifact
pattern: ^[a-zA-Z0-9._-]+$
type: string
value:
description: |-
Expand All @@ -4797,6 +4816,7 @@ spec:
properties:
name:
description: Name of the custom artifact
pattern: ^[a-zA-Z0-9._-]+$
type: string
value:
description: |-
Expand Down Expand Up @@ -4828,9 +4848,11 @@ spec:
type: string
path:
description: Path specifies the path to the task file
pattern: ^[a-zA-Z0-9._/-]*$
type: string
url:
description: URL specifies the git repository URL
pattern: ^https://[a-zA-Z0-9._/-]+$
type: string
type: object
type: object
Expand All @@ -4839,6 +4861,7 @@ spec:
description: TaskNames from lm-eval's task list and/or from custom
tasks if CustomTasks is defined
items:
pattern: ^[a-zA-Z0-9._-]+$
type: string
type: array
taskRecipes:
Expand All @@ -4860,13 +4883,15 @@ spec:
type: string
name:
description: Unitxt card's ID
pattern: ^[a-zA-Z0-9._-]+$
type: string
type: object
demosPoolSize:
description: The pool size for the fewshot
type: integer
format:
description: The Unitxt format
pattern: ^[a-zA-Z0-9._-]+$
type: string
loaderLimit:
description: A limit number of records to load
Expand Down Expand Up @@ -4896,18 +4921,21 @@ spec:
properties:
name:
description: Unitxt System Prompt id
pattern: ^[a-zA-Z0-9._-]+$
type: string
ref:
description: The name of the custom systemPrompt in
the custom field. Its value is a custom system prompt
string
pattern: ^[a-zA-Z0-9._-]+$
type: string
type: object
task:
description: The Unitxt Task
properties:
name:
description: Unitxt task id
pattern: ^[a-zA-Z0-9._-]+$
type: string
ref:
description: |-
Expand All @@ -4923,6 +4951,7 @@ spec:
properties:
name:
description: Unitxt template ID
pattern: ^[a-zA-Z0-9._-]+$
type: string
ref:
description: |-
Expand All @@ -4931,6 +4960,7 @@ spec:
to compose a custom template, store it as a JSON file by calling the
add_to_catalog API: https://www.unitxt.ai/en/latest/docs/saving_and_loading_from_catalog.html#adding-assets-to-the-catalog,
and use the JSON content as the value here.
pattern: ^[a-zA-Z0-9._-]+$
type: string
type: object
required:
Expand Down
5 changes: 5 additions & 0 deletions controllers/lmes/driver/driver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -598,6 +598,7 @@ func (d *driverImpl) fetchGitCustomTasks() error {
return err
}

// #nosec G204 -- CustomTaskGitURL is validated by ValidateGitURL() in the controller
cloneCommand := exec.Command("git", "clone", d.Option.CustomTaskGitURL, repositoryDestination)
if output, err := cloneCommand.CombinedOutput(); err != nil {
return fmt.Errorf("failed to clone git repository: %v, output: %s", err, string(output))
Expand All @@ -608,12 +609,14 @@ func (d *driverImpl) fetchGitCustomTasks() error {

// Checkout a specific branch, if specified
if d.Option.CustomTaskGitBranch != "" {
// #nosec G204 -- CustomTaskGitBranch is validated by ValidateGitBranch() in the controller
checkoutCommand := exec.Command("git", clonedDirectory, workTree, "checkout", d.Option.CustomTaskGitBranch)
if output, err := checkoutCommand.CombinedOutput(); err != nil {
return fmt.Errorf("failed to checkout branch %s: %v, output: %s",
d.Option.CustomTaskGitBranch, err, string(output))
}
} else {
// #nosec G204 -- DefaultGitBranch is a constant value, not user input
checkoutCmd := exec.Command("git", clonedDirectory, workTree, "checkout", DefaultGitBranch)
if output, err := checkoutCmd.CombinedOutput(); err != nil {
d.Option.Logger.Info("failed to checkout main branch, using default branch from clone",
Expand All @@ -623,6 +626,7 @@ func (d *driverImpl) fetchGitCustomTasks() error {

// Checkout a specific commit, if specified
if d.Option.CustomTaskGitCommit != "" {
// #nosec G204 -- CustomTaskGitCommit is validated by ValidateGitCommit() in the controller
checkoutCommand := exec.Command("git", clonedDirectory, workTree, "checkout", d.Option.CustomTaskGitCommit)
if output, err := checkoutCommand.CombinedOutput(); err != nil {
return fmt.Errorf("failed to checkout commit %s: %v, output: %s",
Expand All @@ -644,6 +648,7 @@ func (d *driverImpl) fetchGitCustomTasks() error {
return err
}

// #nosec G204 -- taskPath is derived from validated CustomTaskGitPath, TaskRecipesPath is controlled by the application
copyCmd := exec.Command("cp", "-r", taskPath+"/.", d.Option.TaskRecipesPath)
output, err := copyCmd.CombinedOutput()
if err != nil {
Expand Down
Loading
Loading