From fbf044562d96108e361f5980afb56acb694040fd Mon Sep 17 00:00:00 2001 From: "fangyaozheng@bytedance.com" Date: Wed, 6 Aug 2025 08:54:08 +0800 Subject: [PATCH 1/4] chore:add license checker --- .github/workflows/license-header-check.yaml | 16 +++++++++++++ .../{secrets_scan.yaml => secrets-scan.yaml} | 0 .licenserc.yaml | 23 +++++++++++++++++++ veadk/cli/studio/fast_api.py | 14 +++++++++++ veadk/cli/studio/models.py | 14 +++++++++++ veadk/cli/studio/studio_processor.py | 14 +++++++++++ 6 files changed, 81 insertions(+) create mode 100644 .github/workflows/license-header-check.yaml rename .github/workflows/{secrets_scan.yaml => secrets-scan.yaml} (100%) create mode 100644 .licenserc.yaml diff --git a/.github/workflows/license-header-check.yaml b/.github/workflows/license-header-check.yaml new file mode 100644 index 00000000..1f4a4f45 --- /dev/null +++ b/.github/workflows/license-header-check.yaml @@ -0,0 +1,16 @@ +name: License Header Check + +on: + push: + pull_request: + + workflow_dispatch: + +jobs: + License-Check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Check License Header + uses: apache/skywalking-eyes/header@v0.4.0 diff --git a/.github/workflows/secrets_scan.yaml b/.github/workflows/secrets-scan.yaml similarity index 100% rename from .github/workflows/secrets_scan.yaml rename to .github/workflows/secrets-scan.yaml diff --git a/.licenserc.yaml b/.licenserc.yaml new file mode 100644 index 00000000..98380017 --- /dev/null +++ b/.licenserc.yaml @@ -0,0 +1,23 @@ +header: + - license: + spdx-id: Apache-2.0 + copyright-owner: Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates + content: | + Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + paths: + - '**/*.py' + + comment: on-failure \ No newline at end of file diff --git a/veadk/cli/studio/fast_api.py b/veadk/cli/studio/fast_api.py index d782a108..85af90a0 100644 --- a/veadk/cli/studio/fast_api.py +++ b/veadk/cli/studio/fast_api.py @@ -1,3 +1,17 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import json import os import uuid diff --git a/veadk/cli/studio/models.py b/veadk/cli/studio/models.py index 32c0d0ad..ced00533 100644 --- a/veadk/cli/studio/models.py +++ b/veadk/cli/studio/models.py @@ -1,3 +1,17 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from pydantic import BaseModel diff --git a/veadk/cli/studio/studio_processor.py b/veadk/cli/studio/studio_processor.py index 4f7a91a6..bfc5728b 100644 --- a/veadk/cli/studio/studio_processor.py +++ b/veadk/cli/studio/studio_processor.py @@ -1,3 +1,17 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os from deepeval.metrics import GEval From f1d7b2847e349868c5637c73d90c75816454ee88 Mon Sep 17 00:00:00 2001 From: "fangyaozheng@bytedance.com" Date: Wed, 6 Aug 2025 08:56:15 +0800 Subject: [PATCH 2/4] add file header for remote ve agent --- veadk/a2a/remote_ve_agent.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/veadk/a2a/remote_ve_agent.py b/veadk/a2a/remote_ve_agent.py index 9f7a209e..80e0c2c8 100644 --- a/veadk/a2a/remote_ve_agent.py +++ b/veadk/a2a/remote_ve_agent.py @@ -1,3 +1,17 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import json import requests From 07b1518cca5badfd24a3252e50075fc0b550d9f3 Mon Sep 17 00:00:00 2001 From: "fangyaozheng@bytedance.com" Date: Wed, 6 Aug 2025 11:20:59 +0800 Subject: [PATCH 3/4] fix envs get --- .../deepeval_evaluator/deepeval_evaluator.py | 32 ++++++++------ veadk/evaluation/types.py | 19 +++++++++ veadk/evaluation/utils/prometheus.py | 42 +++++++++---------- 3 files changed, 57 insertions(+), 36 deletions(-) create mode 100644 veadk/evaluation/types.py diff --git a/veadk/evaluation/deepeval_evaluator/deepeval_evaluator.py b/veadk/evaluation/deepeval_evaluator/deepeval_evaluator.py index 780b6159..294963a3 100644 --- a/veadk/evaluation/deepeval_evaluator/deepeval_evaluator.py +++ b/veadk/evaluation/deepeval_evaluator/deepeval_evaluator.py @@ -21,18 +21,15 @@ from deepeval.models import LocalModel from deepeval.test_case import LLMTestCase from deepeval.test_case.llm_test_case import ToolCall +from pydantic import Field from typing_extensions import override from veadk.config import getenv +from veadk.evaluation.types import EvalResultCaseData, EvalResultMetadata from veadk.utils.logger import get_logger from ..base_evaluator import BaseEvaluator, EvalResultData, MetricResult -from ..utils.prometheus import ( - EvalResultCaseData, - EvalResultMetadata, - PrometheusPushgatewayConfig, - push_to_prometheus, -) +from ..utils.prometheus import PrometheusPushgatewayConfig, push_to_prometheus logger = get_logger(__name__) @@ -46,14 +43,23 @@ class DeepevalEvaluator(BaseEvaluator): def __init__( self, agent, - judge_model_api_key: str = getenv("MODEL_JUDGE_API_KEY"), - judge_model_name: str = getenv( - "MODEL_JUDGE_NAME", - "doubao-seed-1-6-250615", + judge_model_api_key: str = Field( + ..., + default_factory=lambda: getenv("MODEL_JUDGE_API_KEY"), ), - judge_model_api_base: str = getenv( - "MODEL_JUDGE_API_BASE", - "https://ark.cn-beijing.volces.com/api/v3/", + judge_model_name: str = Field( + ..., + default_factory=lambda: getenv( + "MODEL_JUDGE_NAME", + "doubao-seed-1-6-250615", + ), + ), + judge_model_api_base: str = Field( + ..., + default_factory=lambda: getenv( + "MODEL_JUDGE_API_BASE", + "https://ark.cn-beijing.volces.com/api/v3/", + ), ), name: str = "veadk_deepeval_evaluator", prometheus_config: PrometheusPushgatewayConfig = None, diff --git a/veadk/evaluation/types.py b/veadk/evaluation/types.py new file mode 100644 index 00000000..bdb75d09 --- /dev/null +++ b/veadk/evaluation/types.py @@ -0,0 +1,19 @@ +from dataclasses import dataclass + + +@dataclass +class EvalResultCaseData: + id: str + input: str + actual_output: str + expected_output: str + score: str + reason: str + status: str # `PASSED` or `FAILURE` + latency: str + + +@dataclass +class EvalResultMetadata: + tested_model: str + judge_model: str diff --git a/veadk/evaluation/utils/prometheus.py b/veadk/evaluation/utils/prometheus.py index b33d63ad..861db30f 100644 --- a/veadk/evaluation/utils/prometheus.py +++ b/veadk/evaluation/utils/prometheus.py @@ -12,38 +12,34 @@ # See the License for the specific language governing permissions and # limitations under the License. -from dataclasses import dataclass from prometheus_client import CollectorRegistry, Gauge, push_to_gateway from prometheus_client.exposition import basic_auth_handler +from pydantic import Field from veadk.config import getenv - - -@dataclass -class EvalResultCaseData: - id: str - input: str - actual_output: str - expected_output: str - score: str - reason: str - status: str # `PASSED` or `FAILURE` - latency: str - - -@dataclass -class EvalResultMetadata: - tested_model: str - judge_model: str +from veadk.evaluation.types import EvalResultCaseData, EvalResultMetadata class PrometheusPushgatewayConfig: - url: str = getenv( - "OBSERVABILITY_PROMETHEUS_PUSHGATEWAY_URL", + url: str = Field( + ..., + default_factory=lambda: getenv( + "OBSERVABILITY_PROMETHEUS_PUSHGATEWAY_URL", + ), + ) + username: str = Field( + ..., + default_factory=lambda: getenv( + "OBSERVABILITY_PROMETHEUS_USERNAME", + ), + ) + password: str = Field( + ..., + default_factory=lambda: getenv( + "OBSERVABILITY_PROMETHEUS_PASSWORD", + ), ) - username: str = getenv("OBSERVABILITY_PROMETHEUS_USERNAME") - password: str = getenv("OBSERVABILITY_PROMETHEUS_PASSWORD") registry = CollectorRegistry() From 256129072e7da267e60eb5ab228510fbdf8edf43 Mon Sep 17 00:00:00 2001 From: "fangyaozheng@bytedance.com" Date: Wed, 6 Aug 2025 11:22:15 +0800 Subject: [PATCH 4/4] add file header --- veadk/evaluation/types.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/veadk/evaluation/types.py b/veadk/evaluation/types.py index bdb75d09..5443e899 100644 --- a/veadk/evaluation/types.py +++ b/veadk/evaluation/types.py @@ -1,3 +1,17 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from dataclasses import dataclass