Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,24 @@
# Changelog

## [v0.6.6] - 2025-07-24

### Added
- Added a new method `verify_content_type` in the `Stuff` class to verify and convert content to the expected type.
- Added `cocode==0.0.6` to the development dependencies in `pyproject.toml`.

### Changed
- Updated `Stuff` class methods to use the new `verify_content_type` method for content verification.
- Updated `vertexai.toml` to change LLM IDs from preview models to released models: `gemini-2.5-pro` and `gemini-2.5-flash`.

### Removed
- Removed `reinitlibraries`, `rl`, `v`, and `init` targets from the Makefile.

## [v0.6.5] - 2025-07-21

### Fixed

- In the documentation, fixed the use of `execute_pipeline`.

## [v0.6.4] - 2025-07-19

- Fixed the `README.md` link to the documentation
Expand Down
14 changes: 0 additions & 14 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -56,16 +56,13 @@ make cleanenv - Remove virtual env and lock files
make cleanderived - Remove extraneous compiled files, caches, logs, etc.
make cleanlibraries - Remove pipelex_libraries
make cleanall - Remove all -> cleanenv + cleanderived + cleanlibraries
make reinitlibraries - Remove pipelex_libraries and init libraries again

make merge-check-ruff-lint - Run ruff merge check without updating files
make merge-check-ruff-format - Run ruff merge check without updating files
make merge-check-mypy - Run mypy merge check without updating files
make merge-check-pyright - Run pyright merge check without updating files

make rl - Shorthand -> reinitlibraries
make v - Shorthand -> validate
make init - Run `pipelex init-libraries` and `pipelex init-config`
make codex-tests - Run tests for Codex (exit on first failure) (no inference, no codex_disabled)
make gha-tests - Run tests for github actions (exit on first failure) (no inference, no gha_disabled)
make test - Run unit tests (no inference)
Expand Down Expand Up @@ -154,11 +151,6 @@ update: env
uv sync --all-extras && \
echo "Updated dependencies in ${VIRTUAL_ENV}";

init: env
$(call PRINT_TITLE,"Running pipelex init-libraries and init-config")
$(VENV_PIPELEX) init-libraries
$(VENV_PIPELEX) init-config

validate: env
$(call PRINT_TITLE,"Running setup sequence")
$(VENV_PIPELEX) validate -c pipelex/libraries
Expand Down Expand Up @@ -196,12 +188,6 @@ cleanlibraries:
@find . -type d -wholename './pipelex_libraries' -exec rm -rf {} + && \
echo "Cleaned up pipelex_libraries";

reinitlibraries: cleanlibraries init
@echo "Reinitialized pipelex_libraries";

rl: reinitlibraries
@echo "> done: rl = reinitlibraries"

cleanall: cleanderived cleanenv cleanlibraries
@echo "Cleaned up all derived files and directories";

Expand Down
24 changes: 14 additions & 10 deletions pipelex/core/stuff.py
Original file line number Diff line number Diff line change
Expand Up @@ -106,25 +106,30 @@ def is_number(self) -> bool:

def content_as(self, content_type: Type[StuffContentType]) -> StuffContentType:
"""Get content with proper typing if it's of the expected type."""
return self.verify_content_type(self.content, content_type)

@classmethod
def verify_content_type(cls, content: StuffContent, content_type: Type[StuffContentType]) -> StuffContentType:
"""Verify and convert content to the expected type."""
# First try the direct isinstance check for performance
if isinstance(self.content, content_type):
return self.content
if isinstance(content, content_type):
return content

# If isinstance failed, try model validation approach
try:
# Check if class names match (quick filter before attempting validation)
if type(self.content).__name__ == content_type.__name__:
content_dict = self.content.smart_dump()
if type(content).__name__ == content_type.__name__:
content_dict = content.smart_dump()
validated_content = content_type.model_validate(content_dict)
log.debug(f"Model validation passed: converted {type(self.content).__name__} to {content_type.__name__}")
log.debug(f"Model validation passed: converted {type(content).__name__} to {content_type.__name__}")
return validated_content
except ValidationError as exc:
formatted_error = format_pydantic_validation_error(exc)
raise StuffContentValidationError(
original_type=type(self.content).__name__, target_type=content_type.__name__, validation_error=formatted_error
original_type=type(content).__name__, target_type=content_type.__name__, validation_error=formatted_error
) from exc

raise TypeError(f"Content is of type '{type(self.content)}', instead of the expected '{content_type}'")
raise TypeError(f"Content is of type '{type(content)}', instead of the expected '{content_type}'")

def as_list_content(self) -> ListContent: # pyright: ignore[reportMissingTypeArgument, reportUnknownParameterType]
"""Get content as ListContent with items of any type."""
Expand All @@ -146,9 +151,8 @@ def as_list_of_fixed_content_type(self, item_type: Type[StuffContentType]) -> Li
list_content: ListContent[StuffContentType] = self.content_as(content_type=ListContent)

# Validate all items are of the expected type
for i, item in enumerate(list_content.items):
if not isinstance(item, item_type):
raise TypeError(f"Item {i} in list is of type {type(item)}, not {item_type}, in {self.stuff_name=} and {self.concept_code=}")
for item in list_content.items:
self.verify_content_type(item, item_type)

return list_content

Expand Down
27 changes: 2 additions & 25 deletions pipelex/libraries/llm_integrations/vertexai.toml
Original file line number Diff line number Diff line change
Expand Up @@ -10,35 +10,12 @@ is_gen_object_supported = true
is_vision_supported = true
max_prompt_images = 3000
cost_per_million_tokens_usd = { input = 0.0, output = 0.0 }
platform_llm_id = { vertexai = "google/gemini-2.5-pro-preview-05-06" }

# Update commented because the latest version is not yet on VertexAI

# [gemini."gemini-2.5-pro"."2025-05-06"]
# is_gen_object_supported = true
# is_vision_supported = true
# max_prompt_images = 3000
# cost_per_million_tokens_usd = { input = 0.0, output = 0.0 }
# platform_llm_id = { vertexai = "google/gemini-2.5-pro-preview-05-06" }

# [gemini."gemini-2.5-pro".latest]
# is_gen_object_supported = true
# is_vision_supported = true
# max_prompt_images = 3000
# cost_per_million_tokens_usd = { input = 0.0, output = 0.0 }
# platform_llm_id = { vertexai = "google/gemini-2.5-pro-preview-06-05" }

[gemini."gemini-2.5-flash"."2025-04-17"]
is_gen_object_supported = true
is_vision_supported = true
max_prompt_images = 3000
cost_per_million_tokens_usd = { input = 0.15, output = 0.6 }
platform_llm_id = { vertexai = "google/gemini-2.5-flash-preview-04-17" }
platform_llm_id = { vertexai = "google/gemini-2.5-pro" }

[gemini."gemini-2.5-flash".latest]
is_gen_object_supported = true
is_vision_supported = true
max_prompt_images = 3000
cost_per_million_tokens_usd = { input = 0.15, output = 0.6 }
platform_llm_id = { vertexai = "google/gemini-2.5-flash-preview-05-20" }
platform_llm_id = { vertexai = "google/gemini-2.5-flash" }

17 changes: 0 additions & 17 deletions pipelex/test_extras/shared_pytest_plugins.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
from pytest import FixtureRequest, Parser

from pipelex.core.pipe_run_params import PipeRunMode
from pipelex.libraries.library_config import LibraryConfig
from pipelex.tools.runtime_manager import RunMode, runtime_manager


Expand All @@ -11,22 +10,6 @@ def set_run_mode():
runtime_manager.set_run_mode(run_mode=RunMode.UNIT_TEST)


@pytest.fixture(scope="session")
def manage_pipelex_libraries():
LibraryConfig(config_folder_path="./pipelex_libraries").export_libraries(overwrite=False)
yield
# TODO: make it safe to erase/replace standard libraries in client projects without touching custom stuff
# LibraryConfig.remove_libraries()


@pytest.fixture(scope="session")
def manage_pipelex_libraries_with_overwrite():
LibraryConfig(config_folder_path="./pipelex_libraries").export_libraries(overwrite=True)
yield
# TODO: make it safe to erase/replace standard libraries in client projects without touching custom stuff
# LibraryConfig.remove_libraries()


def pytest_addoption(parser: Parser):
parser.addoption(
"--pipe-run-mode",
Expand Down
3 changes: 2 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "pipelex"
version = "0.6.5"
version = "0.6.6"
description = "Pipelex is an open-source dev tool based on a simple declarative language that lets you define replicable, structured, composable LLM pipelines."
authors = [{ name = "Evotis S.A.S.", email = "evotis@pipelex.com" }]
maintainers = [{ name = "Pipelex staff", email = "oss@pipelex.com" }]
Expand Down Expand Up @@ -65,6 +65,7 @@ docs = [
]
dev = [
"boto3-stubs>=1.35.24",
"cocode==0.0.6",
"mypy>=1.11.2",
"pyright==1.1.398",
"pandas-stubs>=2.2.3.241126",
Expand Down
7 changes: 0 additions & 7 deletions tests/conftest.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
from typing import Any

import pytest
from rich import print
from rich.console import Console
Expand Down Expand Up @@ -46,11 +44,6 @@ def pretty():
# Code to run after each test


@pytest.fixture(scope="session", autouse=True) # pyright: ignore[reportUntypedFunctionDecorator, reportUnknownMemberType]
def apply_manage_pipelex_libraries(manage_pipelex_libraries_with_overwrite: Any): # pyright: ignore[reportUnknownParameterType]
return


# Test data fixtures
@pytest.fixture(scope="session")
def apple() -> Fruit:
Expand Down
2 changes: 2 additions & 0 deletions tests/pipelex_unit_test.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,5 @@
[cogt.imgg_config]
default_imgg_handle = "fal-ai/fast-lightning-sdxl"

[pipelex.feature_config]
is_reporting_enabled = false
Loading