diff --git a/.github/workflows/license-header-check.yaml b/.github/workflows/license-header-check.yaml new file mode 100644 index 00000000..1f4a4f45 --- /dev/null +++ b/.github/workflows/license-header-check.yaml @@ -0,0 +1,16 @@ +name: License Header Check + +on: + push: + pull_request: + + workflow_dispatch: + +jobs: + License-Check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Check License Header + uses: apache/skywalking-eyes/header@v0.4.0 diff --git a/.github/workflows/secrets_scan.yaml b/.github/workflows/secrets-scan.yaml similarity index 100% rename from .github/workflows/secrets_scan.yaml rename to .github/workflows/secrets-scan.yaml diff --git a/.licenserc.yaml b/.licenserc.yaml new file mode 100644 index 00000000..98380017 --- /dev/null +++ b/.licenserc.yaml @@ -0,0 +1,23 @@ +header: + - license: + spdx-id: Apache-2.0 + copyright-owner: Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates + content: | + Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + paths: + - '**/*.py' + + comment: on-failure \ No newline at end of file diff --git a/veadk/a2a/remote_ve_agent.py b/veadk/a2a/remote_ve_agent.py index 9f7a209e..80e0c2c8 100644 --- a/veadk/a2a/remote_ve_agent.py +++ b/veadk/a2a/remote_ve_agent.py @@ -1,3 +1,17 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import json import requests diff --git a/veadk/cli/studio/fast_api.py b/veadk/cli/studio/fast_api.py index d782a108..85af90a0 100644 --- a/veadk/cli/studio/fast_api.py +++ b/veadk/cli/studio/fast_api.py @@ -1,3 +1,17 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import json import os import uuid diff --git a/veadk/cli/studio/models.py b/veadk/cli/studio/models.py index 32c0d0ad..ced00533 100644 --- a/veadk/cli/studio/models.py +++ b/veadk/cli/studio/models.py @@ -1,3 +1,17 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from pydantic import BaseModel diff --git a/veadk/cli/studio/studio_processor.py b/veadk/cli/studio/studio_processor.py index 4f7a91a6..bfc5728b 100644 --- a/veadk/cli/studio/studio_processor.py +++ b/veadk/cli/studio/studio_processor.py @@ -1,3 +1,17 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os from deepeval.metrics import GEval diff --git a/veadk/evaluation/deepeval_evaluator/deepeval_evaluator.py b/veadk/evaluation/deepeval_evaluator/deepeval_evaluator.py index 780b6159..294963a3 100644 --- a/veadk/evaluation/deepeval_evaluator/deepeval_evaluator.py +++ b/veadk/evaluation/deepeval_evaluator/deepeval_evaluator.py @@ -21,18 +21,15 @@ from deepeval.models import LocalModel from deepeval.test_case import LLMTestCase from deepeval.test_case.llm_test_case import ToolCall +from pydantic import Field from typing_extensions import override from veadk.config import getenv +from veadk.evaluation.types import EvalResultCaseData, EvalResultMetadata from veadk.utils.logger import get_logger from ..base_evaluator import BaseEvaluator, EvalResultData, MetricResult -from ..utils.prometheus import ( - EvalResultCaseData, - EvalResultMetadata, - PrometheusPushgatewayConfig, - push_to_prometheus, -) +from ..utils.prometheus import PrometheusPushgatewayConfig, push_to_prometheus logger = get_logger(__name__) @@ -46,14 +43,23 @@ class DeepevalEvaluator(BaseEvaluator): def __init__( self, agent, - judge_model_api_key: str = getenv("MODEL_JUDGE_API_KEY"), - judge_model_name: str = getenv( - "MODEL_JUDGE_NAME", - "doubao-seed-1-6-250615", + judge_model_api_key: str = Field( + ..., + default_factory=lambda: getenv("MODEL_JUDGE_API_KEY"), ), - judge_model_api_base: str = getenv( - "MODEL_JUDGE_API_BASE", - "https://ark.cn-beijing.volces.com/api/v3/", + judge_model_name: str = Field( + ..., + default_factory=lambda: getenv( + "MODEL_JUDGE_NAME", + "doubao-seed-1-6-250615", + ), + ), + judge_model_api_base: str = Field( + ..., + default_factory=lambda: getenv( + "MODEL_JUDGE_API_BASE", + "https://ark.cn-beijing.volces.com/api/v3/", + ), ), name: str = "veadk_deepeval_evaluator", prometheus_config: PrometheusPushgatewayConfig = None, diff --git a/veadk/evaluation/types.py b/veadk/evaluation/types.py new file mode 100644 index 00000000..5443e899 --- /dev/null +++ b/veadk/evaluation/types.py @@ -0,0 +1,33 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass + + +@dataclass +class EvalResultCaseData: + id: str + input: str + actual_output: str + expected_output: str + score: str + reason: str + status: str # `PASSED` or `FAILURE` + latency: str + + +@dataclass +class EvalResultMetadata: + tested_model: str + judge_model: str diff --git a/veadk/evaluation/utils/prometheus.py b/veadk/evaluation/utils/prometheus.py index b33d63ad..861db30f 100644 --- a/veadk/evaluation/utils/prometheus.py +++ b/veadk/evaluation/utils/prometheus.py @@ -12,38 +12,34 @@ # See the License for the specific language governing permissions and # limitations under the License. -from dataclasses import dataclass from prometheus_client import CollectorRegistry, Gauge, push_to_gateway from prometheus_client.exposition import basic_auth_handler +from pydantic import Field from veadk.config import getenv - - -@dataclass -class EvalResultCaseData: - id: str - input: str - actual_output: str - expected_output: str - score: str - reason: str - status: str # `PASSED` or `FAILURE` - latency: str - - -@dataclass -class EvalResultMetadata: - tested_model: str - judge_model: str +from veadk.evaluation.types import EvalResultCaseData, EvalResultMetadata class PrometheusPushgatewayConfig: - url: str = getenv( - "OBSERVABILITY_PROMETHEUS_PUSHGATEWAY_URL", + url: str = Field( + ..., + default_factory=lambda: getenv( + "OBSERVABILITY_PROMETHEUS_PUSHGATEWAY_URL", + ), + ) + username: str = Field( + ..., + default_factory=lambda: getenv( + "OBSERVABILITY_PROMETHEUS_USERNAME", + ), + ) + password: str = Field( + ..., + default_factory=lambda: getenv( + "OBSERVABILITY_PROMETHEUS_PASSWORD", + ), ) - username: str = getenv("OBSERVABILITY_PROMETHEUS_USERNAME") - password: str = getenv("OBSERVABILITY_PROMETHEUS_PASSWORD") registry = CollectorRegistry()