diff --git a/.gitignore b/.gitignore index 90296c16..1e28962d 100644 --- a/.gitignore +++ b/.gitignore @@ -156,6 +156,7 @@ dmypy.json # Cython debug symbols cython_debug/ openviking/bin/ +third_party/agfs/bin/ test_scripts/ test_large_scale_collection/ diff --git a/README.md b/README.md index bdb3349b..862d23bf 100644 --- a/README.md +++ b/README.md @@ -349,23 +349,6 @@ set "OPENVIKING_CONFIG_FILE=%USERPROFILE%\.openviking\ov.conf" > 💡 **Tip**: You can also place the configuration file in other locations, just specify the correct path in the environment variable. -#### AGFS Binding Mode (High Performance) - -For better performance, you can use the Python Binding mode to bypass HTTP overhead by directly calling the AGFS core. - -**Update your `ov.conf`**: - Add `mode` to the `agfs` section: - ```json - { - "storage": { - "agfs": { - "mode": "binding-client", - "backend": "local" - } - } - } - ``` - ### 4. Run Your First Example > 📝 **Prerequisite**: Ensure you have completed the environment configuration in the previous step. diff --git a/openviking/utils/agfs_utils.py b/openviking/utils/agfs_utils.py index 25a24db6..ba7977a8 100644 --- a/openviking/utils/agfs_utils.py +++ b/openviking/utils/agfs_utils.py @@ -5,6 +5,7 @@ """ import os +from pathlib import Path from typing import Any from openviking_cli.utils.logger import get_logger @@ -28,15 +29,30 @@ def create_agfs_client(agfs_config: Any) -> Any: mode = getattr(agfs_config, "mode", "http-client") if mode == "binding-client": - # Setup library path if needed - from pyagfs import AGFSBindingClient + # Import binding client if mode is binding-client + try: + from pyagfs import AGFSBindingClient + except ImportError: + raise ImportError( + "AGFS binding client not found. Please run: cd third_party/agfs/agfs-sdk/python && uv pip install -e ." + ) lib_path = getattr(agfs_config, "lib_path", None) if lib_path and lib_path not in ["1", "default"]: os.environ["AGFS_LIB_PATH"] = lib_path + else: + os.environ["AGFS_LIB_PATH"] = str(Path(__file__).parent.parent / "lib") + + # Check if binding library exists + try: + from pyagfs.binding_client import _find_library + + actual_lib_path = _find_library() + except Exception: + raise ImportError("AGFS binding library not found. Please run: uv pip install -e .") client = AGFSBindingClient() - logger.info(f"[AGFSUtils] Created AGFSBindingClient (lib_path={lib_path})") + logger.debug(f"[AGFSUtils] Created AGFSBindingClient (lib_path={actual_lib_path})") # Automatically mount backend for binding client mount_agfs_backend(client, agfs_config) @@ -126,7 +142,7 @@ def mount_agfs_backend(agfs: Any, agfs_config: Any) -> None: try: agfs.mount(fstype, mount_path, config) - logger.info(f"[AGFSUtils] Mounted {fstype} at {mount_path} with config={config}") + logger.info(f"[AGFSUtils] Mounted {fstype} at {mount_path}") except Exception as e: logger.error(f"[AGFSUtils] Failed to mount {fstype} at {mount_path}: {e}") raise e diff --git a/setup.py b/setup.py index 8a8089c4..a5b8f5ce 100644 --- a/setup.py +++ b/setup.py @@ -47,8 +47,9 @@ def build_agfs(self): # Target in source tree (for development/install) agfs_bin_dir = Path("openviking/bin").resolve() + agfs_lib_dir = Path("openviking/lib").resolve() agfs_target_binary = agfs_bin_dir / binary_name - agfs_target_lib = agfs_bin_dir / lib_name + agfs_target_lib = agfs_lib_dir / lib_name # 1. Try to build from source if agfs_server_dir.exists() and shutil.which("go"): @@ -100,26 +101,18 @@ def build_agfs(self): env = os.environ.copy() env["CGO_ENABLED"] = "1" - pybinding_dir = agfs_server_dir / "cmd/pybinding" - lib_build_args = [ - "go", - "build", - "-buildmode=c-shared", - "-o", - f"build/{lib_name}", - ".", - ] + lib_build_args = ["make", "build-lib"] subprocess.run( lib_build_args, - cwd=str(pybinding_dir), + cwd=str(agfs_server_dir), env=env, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) - agfs_built_lib = pybinding_dir / "build" / lib_name + agfs_built_lib = agfs_server_dir / "build" / lib_name if agfs_built_lib.exists(): self._copy_binary(agfs_built_lib, agfs_target_lib) print("[OK] AGFS binding library built successfully") @@ -141,11 +134,11 @@ def build_agfs(self): # 2. Ensure binaries are copied to the build directory (where wheel is packaged from) if self.build_lib: - agfs_bin_dir_build = Path(self.build_lib) / "openviking/bin" + agfs_bin_dir_build = Path(self.build_lib) / "openviking" if agfs_target_binary.exists(): - self._copy_binary(agfs_target_binary, agfs_bin_dir_build / binary_name) + self._copy_binary(agfs_target_binary, agfs_bin_dir_build / "bin" / binary_name) if agfs_target_lib.exists(): - self._copy_binary(agfs_target_lib, agfs_bin_dir_build / lib_name) + self._copy_binary(agfs_target_lib, agfs_bin_dir_build / "lib" / lib_name) def build_extension(self, ext): """Build a single C++ extension module using CMake.""" diff --git a/third_party/agfs/agfs-sdk/python/pyagfs/binding_client.py b/third_party/agfs/agfs-sdk/python/pyagfs/binding_client.py index cd7c84ce..0078c7ec 100644 --- a/third_party/agfs/agfs-sdk/python/pyagfs/binding_client.py +++ b/third_party/agfs/agfs-sdk/python/pyagfs/binding_client.py @@ -26,12 +26,12 @@ def _find_library() -> str: search_paths = [ Path(__file__).parent / "lib" / lib_name, Path(__file__).parent.parent / "lib" / lib_name, + Path(__file__).parent.parent.parent / "lib" / lib_name, Path("/usr/local/lib") / lib_name, Path("/usr/lib") / lib_name, Path(os.environ.get("AGFS_LIB_PATH", "")) / lib_name if os.environ.get("AGFS_LIB_PATH") else None, - Path("/tmp") / lib_name, ] for path in search_paths: diff --git a/third_party/agfs/agfs-server/Makefile b/third_party/agfs/agfs-server/Makefile index 4f78e97b..cc80a2b0 100644 --- a/third_party/agfs/agfs-server/Makefile +++ b/third_party/agfs/agfs-server/Makefile @@ -24,6 +24,28 @@ build: ## Build the server binary $(GO) build $(GOFLAGS) $(LDFLAGS) -o $(BUILD_DIR)/$(BINARY_NAME) $(CMD_DIR)/main.go @echo "Build complete: $(BUILD_DIR)/$(BINARY_NAME)" +build-lib: ## Build AGFS binding library + @echo "Detecting OS for building binding library..." + @OS=$$(uname -s | tr '[:upper:]' '[:lower:]'); \ + LIB_NAME=libagfsbinding; \ + CMD_PYBINDING_DIR=cmd/pybinding; \ + case "$$OS" in \ + darwin*) \ + echo "Building for macOS..."; \ + CGO_ENABLED=1 $(GO) build -buildmode=c-shared -o $(BUILD_DIR)/$$LIB_NAME.dylib $$CMD_PYBINDING_DIR/main.go; \ + ;; \ + linux*) \ + echo "Building for Linux..."; \ + CGO_ENABLED=1 $(GO) build -buildmode=c-shared -o $(BUILD_DIR)/$$LIB_NAME.so $$CMD_PYBINDING_DIR/main.go; \ + ;; \ + msys*|cygwin*|mingw*) \ + echo "Building for Windows..."; \ + CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ AR=x86_64-w64-mingw32-ar GOOS=windows GOARCH=amd64 CGO_ENABLED=1 $(GO) build -buildmode=c-shared -o $(BUILD_DIR)/$$LIB_NAME.dll $$CMD_PYBINDING_DIR/main.go; \ + ;; \ + *) echo "Unsupported OS: $$OS"; exit 1 ;; \ + esac; \ + echo "Build complete in $(BUILD_DIR)" + run: build @echo "Starting $(BINARY_NAME) on $(ADDR)..." ./$(BUILD_DIR)/$(BINARY_NAME) -addr $(ADDR) diff --git a/third_party/agfs/agfs-server/cmd/pybinding/Makefile b/third_party/agfs/agfs-server/cmd/pybinding/Makefile deleted file mode 100644 index 99f40592..00000000 --- a/third_party/agfs/agfs-server/cmd/pybinding/Makefile +++ /dev/null @@ -1,47 +0,0 @@ -.PHONY: all build-macos build-linux build-windows clean install test - -GO=go -BUILD_DIR=build -LIB_NAME=libagfsbinding -BIN_DIR=../../../bin - -all: build-macos - -build-macos: - @echo "Building shared library for macOS..." - @mkdir -p $(BUILD_DIR) - @mkdir -p $(BIN_DIR) - CGO_ENABLED=1 $(GO) build -buildmode=c-shared -o $(BUILD_DIR)/$(LIB_NAME).dylib . - @cp $(BUILD_DIR)/$(LIB_NAME).dylib $(BIN_DIR)/ - @echo "Build complete: $(BIN_DIR)/$(LIB_NAME).dylib" - -build-linux: - @echo "Building shared library for Linux..." - @mkdir -p $(BUILD_DIR) - @mkdir -p $(BIN_DIR) - CGO_ENABLED=1 $(GO) build -buildmode=c-shared -o $(BUILD_DIR)/$(LIB_NAME).so . - @cp $(BUILD_DIR)/$(LIB_NAME).so $(BIN_DIR)/ - @echo "Build complete: $(BIN_DIR)/$(LIB_NAME).so" - -build-windows: - @echo "Building shared library for Windows..." - @mkdir -p $(BUILD_DIR) - @mkdir -p $(BIN_DIR) - CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ AR=x86_64-w64-mingw32-ar GOOS=windows GOARCH=amd64 CGO_ENABLED=1 $(GO) build -buildmode=c-shared -o $(BUILD_DIR)/$(LIB_NAME).dll . - @cp $(BUILD_DIR)/$(LIB_NAME).dll $(BIN_DIR)/ - @echo "Build complete: $(BIN_DIR)/$(LIB_NAME).dll" - -clean: - @echo "Cleaning..." - @rm -rf $(BUILD_DIR) - @rm -f *.so *.dylib *.dll *.h - @rm -f $(BIN_DIR)/$(LIB_NAME).so $(BIN_DIR)/$(LIB_NAME).dylib $(BIN_DIR)/$(LIB_NAME).dll - @echo "Clean complete" - -install: build-macos - @echo "Installing..." - @cp $(BUILD_DIR)/$(LIB_NAME).dylib /usr/local/lib/ 2>/dev/null || true - @echo "Install complete" - -test: - $(GO) test -v ./... diff --git a/third_party/agfs/bin/libagfsbinding.dll b/third_party/agfs/bin/libagfsbinding.dll deleted file mode 100644 index 68b4f724..00000000 Binary files a/third_party/agfs/bin/libagfsbinding.dll and /dev/null differ diff --git a/third_party/agfs/bin/libagfsbinding.dylib b/third_party/agfs/bin/libagfsbinding.dylib deleted file mode 100644 index df134494..00000000 Binary files a/third_party/agfs/bin/libagfsbinding.dylib and /dev/null differ diff --git a/third_party/agfs/bin/libagfsbinding.h b/third_party/agfs/bin/libagfsbinding.h deleted file mode 100644 index 51f5e228..00000000 --- a/third_party/agfs/bin/libagfsbinding.h +++ /dev/null @@ -1,134 +0,0 @@ -/* Code generated by cmd/cgo; DO NOT EDIT. */ - -/* package github.com/c4pt0r/agfs/agfs-server/cmd/pybinding */ - - -#line 1 "cgo-builtin-export-prolog" - -#include - -#ifndef GO_CGO_EXPORT_PROLOGUE_H -#define GO_CGO_EXPORT_PROLOGUE_H - -#ifndef GO_CGO_GOSTRING_TYPEDEF -typedef struct { const char *p; ptrdiff_t n; } _GoString_; -extern size_t _GoStringLen(_GoString_ s); -extern const char *_GoStringPtr(_GoString_ s); -#endif - -#endif - -/* Start of preamble from import "C" comments. */ - - -#line 3 "main.go" - -#include -#include -#include - -#line 1 "cgo-generated-wrapper" - - -/* End of preamble from import "C" comments. */ - - -/* Start of boilerplate cgo prologue. */ -#line 1 "cgo-gcc-export-header-prolog" - -#ifndef GO_CGO_PROLOGUE_H -#define GO_CGO_PROLOGUE_H - -typedef signed char GoInt8; -typedef unsigned char GoUint8; -typedef short GoInt16; -typedef unsigned short GoUint16; -typedef int GoInt32; -typedef unsigned int GoUint32; -typedef long long GoInt64; -typedef unsigned long long GoUint64; -typedef GoInt64 GoInt; -typedef GoUint64 GoUint; -typedef size_t GoUintptr; -typedef float GoFloat32; -typedef double GoFloat64; -#ifdef _MSC_VER -#if !defined(__cplusplus) || _MSVC_LANG <= 201402L -#include -typedef _Fcomplex GoComplex64; -typedef _Dcomplex GoComplex128; -#else -#include -typedef std::complex GoComplex64; -typedef std::complex GoComplex128; -#endif -#else -typedef float _Complex GoComplex64; -typedef double _Complex GoComplex128; -#endif - -/* - static assertion to make sure the file is being used on architecture - at least with matching size of GoInt. -*/ -typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; - -#ifndef GO_CGO_GOSTRING_TYPEDEF -typedef _GoString_ GoString; -#endif -typedef void *GoMap; -typedef void *GoChan; -typedef struct { void *t; void *v; } GoInterface; -typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; - -#endif - -/* End of boilerplate cgo prologue. */ - -#ifdef __cplusplus -extern "C" { -#endif - -extern GoInt64 AGFS_NewClient(void); -extern void AGFS_FreeClient(GoInt64 clientID); -extern char* AGFS_GetLastError(GoInt64 errorID); -extern void AGFS_FreeString(char* s); -extern int AGFS_Health(GoInt64 clientID); -extern char* AGFS_GetCapabilities(GoInt64 clientID); -extern char* AGFS_Ls(GoInt64 clientID, char* path); -extern int64_t AGFS_Read(GoInt64 clientID, char* path, int64_t offset, int64_t size, char** outData, int64_t* outSize); -extern char* AGFS_Write(GoInt64 clientID, char* path, void* data, int64_t dataSize); -extern char* AGFS_Create(GoInt64 clientID, char* path); -extern char* AGFS_Mkdir(GoInt64 clientID, char* path, unsigned int mode); -extern char* AGFS_Rm(GoInt64 clientID, char* path, int recursive); -extern char* AGFS_Stat(GoInt64 clientID, char* path); -extern char* AGFS_Mv(GoInt64 clientID, char* oldPath, char* newPath); -extern char* AGFS_Chmod(GoInt64 clientID, char* path, unsigned int mode); -extern char* AGFS_Touch(GoInt64 clientID, char* path); -extern char* AGFS_Mounts(GoInt64 clientID); -extern char* AGFS_Mount(GoInt64 clientID, char* fstype, char* path, char* configJSON); -extern char* AGFS_Unmount(GoInt64 clientID, char* path); -extern char* AGFS_LoadPlugin(GoInt64 clientID, char* libraryPath); -extern char* AGFS_UnloadPlugin(GoInt64 clientID, char* libraryPath); -extern char* AGFS_ListPlugins(GoInt64 clientID); -extern int64_t AGFS_OpenHandle(GoInt64 clientID, char* path, int flags, unsigned int mode, int lease); -extern char* AGFS_CloseHandle(int64_t handleID); - -/* Return type for AGFS_HandleRead */ -struct AGFS_HandleRead_return { - char* r0; - int64_t r1; - int64_t r2; -}; -extern struct AGFS_HandleRead_return AGFS_HandleRead(int64_t handleID, int64_t size, int64_t offset, int hasOffset); -extern char* AGFS_HandleWrite(int64_t handleID, void* data, int64_t dataSize, int64_t offset, int hasOffset); -extern char* AGFS_HandleSeek(int64_t handleID, int64_t offset, int whence); -extern char* AGFS_HandleSync(int64_t handleID); -extern char* AGFS_HandleStat(int64_t handleID); -extern char* AGFS_ListHandles(GoInt64 clientID); -extern char* AGFS_GetHandleInfo(int64_t handleID); -extern void* AGFS_GetPluginLoader(void); - -#ifdef __cplusplus -} -#endif diff --git a/third_party/agfs/bin/libagfsbinding.so b/third_party/agfs/bin/libagfsbinding.so deleted file mode 100644 index 85a64bc7..00000000 Binary files a/third_party/agfs/bin/libagfsbinding.so and /dev/null differ