Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@ dmypy.json
# Cython debug symbols
cython_debug/
openviking/bin/
third_party/agfs/bin/
test_scripts/
test_large_scale_collection/

Expand Down
17 changes: 0 additions & 17 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -349,23 +349,6 @@ set "OPENVIKING_CONFIG_FILE=%USERPROFILE%\.openviking\ov.conf"

> 💡 **Tip**: You can also place the configuration file in other locations, just specify the correct path in the environment variable.

#### AGFS Binding Mode (High Performance)

For better performance, you can use the Python Binding mode to bypass HTTP overhead by directly calling the AGFS core.

**Update your `ov.conf`**:
Add `mode` to the `agfs` section:
```json
{
"storage": {
"agfs": {
"mode": "binding-client",
"backend": "local"
}
}
}
```

### 4. Run Your First Example

> 📝 **Prerequisite**: Ensure you have completed the environment configuration in the previous step.
Expand Down
24 changes: 20 additions & 4 deletions openviking/utils/agfs_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
"""

import os
from pathlib import Path
from typing import Any

from openviking_cli.utils.logger import get_logger
Expand All @@ -28,15 +29,30 @@ def create_agfs_client(agfs_config: Any) -> Any:
mode = getattr(agfs_config, "mode", "http-client")

if mode == "binding-client":
# Setup library path if needed
from pyagfs import AGFSBindingClient
# Import binding client if mode is binding-client
try:
from pyagfs import AGFSBindingClient
except ImportError:
raise ImportError(
"AGFS binding client not found. Please run: cd third_party/agfs/agfs-sdk/python && uv pip install -e ."
)

lib_path = getattr(agfs_config, "lib_path", None)
if lib_path and lib_path not in ["1", "default"]:
os.environ["AGFS_LIB_PATH"] = lib_path
else:
os.environ["AGFS_LIB_PATH"] = str(Path(__file__).parent.parent / "lib")

# Check if binding library exists
try:
from pyagfs.binding_client import _find_library

actual_lib_path = _find_library()
except Exception:
raise ImportError("AGFS binding library not found. Please run: uv pip install -e .")

client = AGFSBindingClient()
logger.info(f"[AGFSUtils] Created AGFSBindingClient (lib_path={lib_path})")
logger.debug(f"[AGFSUtils] Created AGFSBindingClient (lib_path={actual_lib_path})")

# Automatically mount backend for binding client
mount_agfs_backend(client, agfs_config)
Expand Down Expand Up @@ -126,7 +142,7 @@ def mount_agfs_backend(agfs: Any, agfs_config: Any) -> None:

try:
agfs.mount(fstype, mount_path, config)
logger.info(f"[AGFSUtils] Mounted {fstype} at {mount_path} with config={config}")
logger.info(f"[AGFSUtils] Mounted {fstype} at {mount_path}")
except Exception as e:
logger.error(f"[AGFSUtils] Failed to mount {fstype} at {mount_path}: {e}")
raise e
23 changes: 8 additions & 15 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,9 @@ def build_agfs(self):

# Target in source tree (for development/install)
agfs_bin_dir = Path("openviking/bin").resolve()
agfs_lib_dir = Path("openviking/lib").resolve()
agfs_target_binary = agfs_bin_dir / binary_name
agfs_target_lib = agfs_bin_dir / lib_name
agfs_target_lib = agfs_lib_dir / lib_name

# 1. Try to build from source
if agfs_server_dir.exists() and shutil.which("go"):
Expand Down Expand Up @@ -100,26 +101,18 @@ def build_agfs(self):
env = os.environ.copy()
env["CGO_ENABLED"] = "1"

pybinding_dir = agfs_server_dir / "cmd/pybinding"
lib_build_args = [
"go",
"build",
"-buildmode=c-shared",
"-o",
f"build/{lib_name}",
".",
]
lib_build_args = ["make", "build-lib"]

subprocess.run(
lib_build_args,
cwd=str(pybinding_dir),
cwd=str(agfs_server_dir),
env=env,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)

agfs_built_lib = pybinding_dir / "build" / lib_name
agfs_built_lib = agfs_server_dir / "build" / lib_name
if agfs_built_lib.exists():
self._copy_binary(agfs_built_lib, agfs_target_lib)
print("[OK] AGFS binding library built successfully")
Expand All @@ -141,11 +134,11 @@ def build_agfs(self):

# 2. Ensure binaries are copied to the build directory (where wheel is packaged from)
if self.build_lib:
agfs_bin_dir_build = Path(self.build_lib) / "openviking/bin"
agfs_bin_dir_build = Path(self.build_lib) / "openviking"
if agfs_target_binary.exists():
self._copy_binary(agfs_target_binary, agfs_bin_dir_build / binary_name)
self._copy_binary(agfs_target_binary, agfs_bin_dir_build / "bin" / binary_name)
if agfs_target_lib.exists():
self._copy_binary(agfs_target_lib, agfs_bin_dir_build / lib_name)
self._copy_binary(agfs_target_lib, agfs_bin_dir_build / "lib" / lib_name)

def build_extension(self, ext):
"""Build a single C++ extension module using CMake."""
Expand Down
2 changes: 1 addition & 1 deletion third_party/agfs/agfs-sdk/python/pyagfs/binding_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,12 @@ def _find_library() -> str:
search_paths = [
Path(__file__).parent / "lib" / lib_name,
Path(__file__).parent.parent / "lib" / lib_name,
Path(__file__).parent.parent.parent / "lib" / lib_name,
Path("/usr/local/lib") / lib_name,
Path("/usr/lib") / lib_name,
Path(os.environ.get("AGFS_LIB_PATH", "")) / lib_name
if os.environ.get("AGFS_LIB_PATH")
else None,
Path("/tmp") / lib_name,
]

for path in search_paths:
Expand Down
22 changes: 22 additions & 0 deletions third_party/agfs/agfs-server/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,28 @@ build: ## Build the server binary
$(GO) build $(GOFLAGS) $(LDFLAGS) -o $(BUILD_DIR)/$(BINARY_NAME) $(CMD_DIR)/main.go
@echo "Build complete: $(BUILD_DIR)/$(BINARY_NAME)"

build-lib: ## Build AGFS binding library
@echo "Detecting OS for building binding library..."
@OS=$$(uname -s | tr '[:upper:]' '[:lower:]'); \
LIB_NAME=libagfsbinding; \
CMD_PYBINDING_DIR=cmd/pybinding; \
case "$$OS" in \
darwin*) \
echo "Building for macOS..."; \
CGO_ENABLED=1 $(GO) build -buildmode=c-shared -o $(BUILD_DIR)/$$LIB_NAME.dylib $$CMD_PYBINDING_DIR/main.go; \
;; \
linux*) \
echo "Building for Linux..."; \
CGO_ENABLED=1 $(GO) build -buildmode=c-shared -o $(BUILD_DIR)/$$LIB_NAME.so $$CMD_PYBINDING_DIR/main.go; \
;; \
msys*|cygwin*|mingw*) \
echo "Building for Windows..."; \
CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ AR=x86_64-w64-mingw32-ar GOOS=windows GOARCH=amd64 CGO_ENABLED=1 $(GO) build -buildmode=c-shared -o $(BUILD_DIR)/$$LIB_NAME.dll $$CMD_PYBINDING_DIR/main.go; \
;; \
*) echo "Unsupported OS: $$OS"; exit 1 ;; \
esac; \
echo "Build complete in $(BUILD_DIR)"

run: build
@echo "Starting $(BINARY_NAME) on $(ADDR)..."
./$(BUILD_DIR)/$(BINARY_NAME) -addr $(ADDR)
Expand Down
47 changes: 0 additions & 47 deletions third_party/agfs/agfs-server/cmd/pybinding/Makefile

This file was deleted.

Binary file removed third_party/agfs/bin/libagfsbinding.dll
Binary file not shown.
Binary file removed third_party/agfs/bin/libagfsbinding.dylib
Binary file not shown.
134 changes: 0 additions & 134 deletions third_party/agfs/bin/libagfsbinding.h

This file was deleted.

Binary file removed third_party/agfs/bin/libagfsbinding.so
Binary file not shown.
Loading