Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions .github/workflows/license-header-check.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
name: License Header Check

on:
push:
pull_request:

workflow_dispatch:

jobs:
License-Check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4

- name: Check License Header
uses: apache/skywalking-eyes/header@v0.4.0
23 changes: 23 additions & 0 deletions .licenserc.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
header:
- license:
spdx-id: Apache-2.0
copyright-owner: Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates
content: |
Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

paths:
- '**/*.py'

comment: on-failure
14 changes: 14 additions & 0 deletions veadk/a2a/remote_ve_agent.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,17 @@
# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json

import requests
Expand Down
14 changes: 14 additions & 0 deletions veadk/cli/studio/fast_api.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,17 @@
# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json
import os
import uuid
Expand Down
14 changes: 14 additions & 0 deletions veadk/cli/studio/models.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,17 @@
# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from pydantic import BaseModel


Expand Down
14 changes: 14 additions & 0 deletions veadk/cli/studio/studio_processor.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,17 @@
# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os

from deepeval.metrics import GEval
Expand Down
32 changes: 19 additions & 13 deletions veadk/evaluation/deepeval_evaluator/deepeval_evaluator.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,15 @@
from deepeval.models import LocalModel
from deepeval.test_case import LLMTestCase
from deepeval.test_case.llm_test_case import ToolCall
from pydantic import Field
from typing_extensions import override

from veadk.config import getenv
from veadk.evaluation.types import EvalResultCaseData, EvalResultMetadata
from veadk.utils.logger import get_logger

from ..base_evaluator import BaseEvaluator, EvalResultData, MetricResult
from ..utils.prometheus import (
EvalResultCaseData,
EvalResultMetadata,
PrometheusPushgatewayConfig,
push_to_prometheus,
)
from ..utils.prometheus import PrometheusPushgatewayConfig, push_to_prometheus

logger = get_logger(__name__)

Expand All @@ -46,14 +43,23 @@ class DeepevalEvaluator(BaseEvaluator):
def __init__(
self,
agent,
judge_model_api_key: str = getenv("MODEL_JUDGE_API_KEY"),
judge_model_name: str = getenv(
"MODEL_JUDGE_NAME",
"doubao-seed-1-6-250615",
judge_model_api_key: str = Field(
...,
default_factory=lambda: getenv("MODEL_JUDGE_API_KEY"),
),
judge_model_api_base: str = getenv(
"MODEL_JUDGE_API_BASE",
"https://ark.cn-beijing.volces.com/api/v3/",
judge_model_name: str = Field(
...,
default_factory=lambda: getenv(
"MODEL_JUDGE_NAME",
"doubao-seed-1-6-250615",
),
),
judge_model_api_base: str = Field(
...,
default_factory=lambda: getenv(
"MODEL_JUDGE_API_BASE",
"https://ark.cn-beijing.volces.com/api/v3/",
),
),
name: str = "veadk_deepeval_evaluator",
prometheus_config: PrometheusPushgatewayConfig = None,
Expand Down
33 changes: 33 additions & 0 deletions veadk/evaluation/types.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from dataclasses import dataclass


@dataclass
class EvalResultCaseData:
id: str
input: str
actual_output: str
expected_output: str
score: str
reason: str
status: str # `PASSED` or `FAILURE`
latency: str


@dataclass
class EvalResultMetadata:
tested_model: str
judge_model: str
42 changes: 19 additions & 23 deletions veadk/evaluation/utils/prometheus.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,38 +12,34 @@
# See the License for the specific language governing permissions and
# limitations under the License.

from dataclasses import dataclass

from prometheus_client import CollectorRegistry, Gauge, push_to_gateway
from prometheus_client.exposition import basic_auth_handler
from pydantic import Field

from veadk.config import getenv


@dataclass
class EvalResultCaseData:
id: str
input: str
actual_output: str
expected_output: str
score: str
reason: str
status: str # `PASSED` or `FAILURE`
latency: str


@dataclass
class EvalResultMetadata:
tested_model: str
judge_model: str
from veadk.evaluation.types import EvalResultCaseData, EvalResultMetadata


class PrometheusPushgatewayConfig:
url: str = getenv(
"OBSERVABILITY_PROMETHEUS_PUSHGATEWAY_URL",
url: str = Field(
...,
default_factory=lambda: getenv(
"OBSERVABILITY_PROMETHEUS_PUSHGATEWAY_URL",
),
)
username: str = Field(
...,
default_factory=lambda: getenv(
"OBSERVABILITY_PROMETHEUS_USERNAME",
),
)
password: str = Field(
...,
default_factory=lambda: getenv(
"OBSERVABILITY_PROMETHEUS_PASSWORD",
),
)
username: str = getenv("OBSERVABILITY_PROMETHEUS_USERNAME")
password: str = getenv("OBSERVABILITY_PROMETHEUS_PASSWORD")


registry = CollectorRegistry()
Expand Down