Skip to content

Types

These string enums define vocabulary used by workflows and scenarios. Enum membership does not guarantee a default event trigger implementation; see events. Workflow metadata is read as ordinary JSON and is not automatically validated against these enums.

types

ScenarioType

Bases: str, Enum

Source code in src/benchmark/utils/types.py 
4
5
6
class ScenarioType(str, Enum):
    BENIGN = "benign"
    MALICIOUS = "malicious"

BENIGN class-attribute instance-attribute 

BENIGN = 'benign'

MALICIOUS class-attribute instance-attribute 

MALICIOUS = 'malicious'

GitHubEvent

Bases: str, Enum

Source code in src/benchmark/utils/types.py 
 9
10
11
12
13
14
15
16
class GitHubEvent(str, Enum):
    PULL_REQUEST = "pull_request"
    ISSUES = "issues"
    ISSUE_COMMENT = "issue_comment"
    PULL_REQUEST_REVIEW = "pull_request_review"
    PULL_REQUEST_REVIEW_COMMENT = "pull_request_review_comment"
    PUSH = "push"
    WORKFLOW_DISPATCH = "workflow_dispatch"

PULL_REQUEST class-attribute instance-attribute 

PULL_REQUEST = 'pull_request'

ISSUES class-attribute instance-attribute 

ISSUES = 'issues'

ISSUE_COMMENT class-attribute instance-attribute 

ISSUE_COMMENT = 'issue_comment'

PULL_REQUEST_REVIEW class-attribute instance-attribute 

PULL_REQUEST_REVIEW = 'pull_request_review'

PULL_REQUEST_REVIEW_COMMENT class-attribute instance-attribute 

PULL_REQUEST_REVIEW_COMMENT = 'pull_request_review_comment'

PUSH class-attribute instance-attribute 

PUSH = 'push'

WORKFLOW_DISPATCH class-attribute instance-attribute 

WORKFLOW_DISPATCH = 'workflow_dispatch'

WorkflowCategory

Bases: str, Enum

Source code in src/benchmark/utils/types.py 
19
20
21
22
23
24
25
26
class WorkflowCategory(str, Enum):
    CODE_MAINTENANCE = "code-maintenance"
    SECURITY_COMPLIANCE = "security-compliance"
    TRIAGE_MANAGEMENT = "triage-management"
    CONTENT_DOCS = "content-docs"
    CODE_REVIEW = "code-review"
    SUPPORT_INTELLIGENCE = "support-intelligence"
    REPORTING_ANALYTICS = "reporting-analytics"

CODE_MAINTENANCE class-attribute instance-attribute 

CODE_MAINTENANCE = 'code-maintenance'

SECURITY_COMPLIANCE class-attribute instance-attribute 

SECURITY_COMPLIANCE = 'security-compliance'

TRIAGE_MANAGEMENT class-attribute instance-attribute 

TRIAGE_MANAGEMENT = 'triage-management'

CONTENT_DOCS class-attribute instance-attribute 

CONTENT_DOCS = 'content-docs'

CODE_REVIEW class-attribute instance-attribute 

CODE_REVIEW = 'code-review'

SUPPORT_INTELLIGENCE class-attribute instance-attribute 

SUPPORT_INTELLIGENCE = 'support-intelligence'

REPORTING_ANALYTICS class-attribute instance-attribute 

REPORTING_ANALYTICS = 'reporting-analytics'

AIProvider

Bases: str, Enum

Source code in src/benchmark/utils/types.py 
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
class AIProvider(str, Enum):
    GOOGLE_GEMINI = "google-gemini"
    ANTHROPIC_CLAUDE = "anthropic-claude"
    OPENAI_CODEX = "openai-codex"
    CURSOR = "cursor"
    GITHUB_COPILOT = "github-copilot"
    CLINE = "cline"
    OPEN_CODE = "open-code"
    OPEN_HANDS = "open-hands"
    AMAZON_Q = "amazon-q"
    GOOSE = "goose"
    DEVIN = "devin"
    CISCO_AI = "cisco-ai"
    AIDER = "aider"
    OPENROUTER = "openrouter"

GOOGLE_GEMINI class-attribute instance-attribute 

GOOGLE_GEMINI = 'google-gemini'

ANTHROPIC_CLAUDE class-attribute instance-attribute 

ANTHROPIC_CLAUDE = 'anthropic-claude'

OPENAI_CODEX class-attribute instance-attribute 

OPENAI_CODEX = 'openai-codex'

CURSOR class-attribute instance-attribute 

CURSOR = 'cursor'

GITHUB_COPILOT class-attribute instance-attribute 

GITHUB_COPILOT = 'github-copilot'

CLINE class-attribute instance-attribute 

CLINE = 'cline'

OPEN_CODE class-attribute instance-attribute 

OPEN_CODE = 'open-code'

OPEN_HANDS class-attribute instance-attribute 

OPEN_HANDS = 'open-hands'

AMAZON_Q class-attribute instance-attribute 

AMAZON_Q = 'amazon-q'

GOOSE class-attribute instance-attribute 

GOOSE = 'goose'

DEVIN class-attribute instance-attribute 

DEVIN = 'devin'

CISCO_AI class-attribute instance-attribute 

CISCO_AI = 'cisco-ai'

AIDER class-attribute instance-attribute 

AIDER = 'aider'

OPENROUTER class-attribute instance-attribute 

OPENROUTER = 'openrouter'

DefenseLevel

Bases: str, Enum

Source code in src/benchmark/utils/types.py 
46
47
48
49
class DefenseLevel(str, Enum):
    BASELINE = "baseline"
    HARDENED = "hardened"
    SANDBOXED = "sandboxed"

BASELINE class-attribute instance-attribute 

BASELINE = 'baseline'

HARDENED class-attribute instance-attribute 

HARDENED = 'hardened'

SANDBOXED class-attribute instance-attribute 

SANDBOXED = 'sandboxed'