Add tests for chunk DAG root node creation and UnixFS data extraction fallback

.github/workflows/ci.yml

@@ -2,9 +2,9 @@ name: CI
 
 on:
   push:
-    branches: [ main, develop ]
+    branches: [ main, dev ]
   pull_request:
-    branches: [ main, develop ]
+    branches: [ main, dev ]
 
 jobs:
   test:
@@ -13,7 +13,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.9', '3.10', '3.11', '3.12']
 
     steps:
     - name: Checkout code

.github/workflows/code-quality.yml

@@ -2,9 +2,9 @@ name: Code Quality
 
 on:
   push:
-    branches: [ main, develop ]
+    branches: [ main, dev ]
   pull_request:
-    branches: [ main, develop ]
+    branches: [ main, dev ]
 
 jobs:
   lint:

README.md

@@ -189,42 +189,6 @@ finally:
     print("Connection closed")
 ```
 
-### Streaming API Usage
-
-```python
-from akavesdk import SDK, SDKError
-
-config = SDKConfig(
-    address="connect.akave.ai:5500",
-    max_concurrency=10,
-    block_part_size=1 * 1024 * 1024,  # 1MB
-    use_connection_pool=True
-)
-
-# Initialize the SDK
-sdk = SDK(config)
-
-try:
-    # Get streaming API
-    streaming = sdk.streaming_api()
-
-    # List files in a bucket
-    files = streaming.list_files({}, "my-bucket")
-    for file in files:
-        print(f"File: {file.name}, Size: {file.size} bytes")
-
-    # Get file info
-    file_info = streaming.file_info({}, "my-bucket", "my-file.txt")
-    print(f"File info: {file_info}")
-except SDKError as e:
-    # handle sdk exception
-    pass
-except Exception as e:
-    # handle generic exception
-    pass
-finally:
-    sdk.close()
-```
 
 ## File Size Requirements
 

akavesdk/__init__.py

@@ -2,21 +2,20 @@
 import os
 
 # Add private directory to path
-PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__))
+PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 PRIVATE_PATH = os.path.join(PROJECT_ROOT, "private")
 if PRIVATE_PATH not in sys.path:
     sys.path.append(PRIVATE_PATH)
 
 # Import and expose main SDK classes
 from sdk.sdk import SDK, BucketCreateResult, Bucket, SDKError, SDKConfig
-from sdk.sdk_streaming import StreamingAPI
 from sdk.sdk_ipc import IPC
-from sdk.erasure_code import ErasureCode
+from private.cids import verify_raw, verify, CIDError
 
 
 # Make SDKError appear under akavesdk in tracebacks
 SDKError.__module__ = "akavesdk"
 
 # Define what gets imported with "from akavesdk import *"
-__all__ = ["SDK", "SDKError", "SDKConfig", "StreamingAPI", "IPC", 
-           "BucketCreateResult", "Bucket", "ErasureCode"]
+__all__ = ["SDK", "SDKError", "SDKConfig", "IPC", 
+           "BucketCreateResult", "Bucket", "verify_raw", "verify", "CIDError"]

akavesdk/akavesdk.py

@@ -15,9 +15,7 @@
 # Import SDK classes using absolute imports
 from sdk.sdk import SDK, BucketCreateResult, Bucket
 from sdk.config import SDKError
-from sdk.sdk_streaming import StreamingAPI
 from sdk.sdk_ipc import IPC
-from sdk.erasure_code import ErasureCode
 # Export all classes
-__all__ = ["SDK", "SDKError", "StreamingAPI", "IPC",
-           "BucketCreateResult", "Bucket", "ErasureCode"]
+__all__ = ["SDK", "SDKError", "IPC",
+           "BucketCreateResult", "Bucket"]

private/cids/__init__.py

@@ -0,0 +1,5 @@
+
+from .cids import verify_raw, verify, CIDError
+
+__all__ = ["verify_raw", "verify", "CIDError"]
+

private/cids/cids.py

@@ -0,0 +1,67 @@
+from typing import Union
+
+try:
+    from multiformats import CID, multihash
+    MULTIFORMATS_AVAILABLE = True
+except ImportError:
+    MULTIFORMATS_AVAILABLE = False
+    CID = None
+
+
+class CIDError(Exception):
+    pass
+
+
+def verify_raw(provided_cid: str, data: bytes) -> None:
+    if not MULTIFORMATS_AVAILABLE:
+        raise CIDError("multiformats library is not available")
+
+    try:
+        parsed_cid = CID.decode(provided_cid)
+    except Exception as e:
+        raise CIDError(f"failed to decode provided CID: {e}")
+
+    verify(parsed_cid, data)
+
+
+def verify(c: 'CID', data: bytes) -> None:
+    if not MULTIFORMATS_AVAILABLE:
+        raise CIDError("multiformats library is not available")
+
+    calculated_cid = _calculate_standard_cid(c, data)
+
+    if calculated_cid != c:
+        raise CIDError(
+            f"CID mismatch: provided {str(c)}, calculated {str(calculated_cid)}"
+        )
+
+
+def _calculate_standard_cid(c: 'CID', data: bytes) -> 'CID':
+
+    if not MULTIFORMATS_AVAILABLE:
+        raise CIDError("multiformats library is not available")
+
+    version = c.version
+    codec = c.codec
+
+    if hasattr(c, 'hashfun'):
+        # hashfun is a multicodec object, convert to string
+        hash_code = str(c.hashfun).replace("multihash.get('", "").replace("')", "")
+    else:
+        hash_code = "sha2-256"
+
+    try:
+        digest = multihash.digest(data, hash_code)
+    except Exception as e:
+        raise CIDError(f"failed to create multihash: {e}")
+
+    if version == 0:
+        return CID("base58btc", 0, "dag-pb", digest)
+    elif version == 1:
+        base = "base32"  
+        if hasattr(c, 'base'):
+            base = c.base
+        return CID(base, 1, codec, digest)
+    else:
+        raise CIDError(f"unsupported CID version: {version}")
+

private/cids/cids_test.py

@@ -0,0 +1,136 @@
+import pytest
+import secrets
+import sys
+import os
+
+project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+if project_root not in sys.path:
+    sys.path.insert(0, project_root)
+
+try:
+    from multiformats import CID, multihash
+    MULTIFORMATS_AVAILABLE = True
+except ImportError:
+    MULTIFORMATS_AVAILABLE = False
+    pytest.skip("multiformats library not available", allow_module_level=True)
+
+try:
+    from private.cids.cids import verify_raw, verify, CIDError
+except ImportError:
+    from .cids import verify_raw, verify, CIDError
+
+
+def test_verify_raw_valid_cidv0_matches():
+    test_data = secrets.token_bytes(128)
+
+    v0hash = multihash.digest(test_data, "sha2-256")
+    cidv0 = CID("base58btc", 0, "dag-pb", v0hash)
+
+    verify_raw(str(cidv0), test_data)
+
+
+def test_verify_raw_valid_cidv1_matches():
+    test_data = secrets.token_bytes(128)
+
+    hash_digest = multihash.digest(test_data, "sha2-256")
+    expected_cid = CID("base32", 1, "dag-pb", hash_digest)
+
+    verify_raw(str(expected_cid), test_data)
+
+
+def test_verify_raw_cid_mismatch():
+    test_data = secrets.token_bytes(128)
+
+    hash_digest = multihash.digest(test_data, "sha2-256")
+    expected_cid = CID("base32", 1, "dag-pb", hash_digest)
+
+    wrong_data = b"different data"
+
+    with pytest.raises(CIDError) as exc_info:
+        verify_raw(str(expected_cid), wrong_data)
+
+    assert "CID mismatch" in str(exc_info.value)
+
+
+def test_verify_raw_invalid_cid_format():
+    test_data = secrets.token_bytes(128)
+
+    with pytest.raises(CIDError) as exc_info:
+        verify_raw("invalid-cid", test_data)
+
+    assert "failed to decode provided CID" in str(exc_info.value)
+
+
+def test_verify_raw_empty_data():
+    empty_data = b""
+
+    # Calculate CID for empty data
+    hash_digest = multihash.digest(empty_data, "sha2-256")
+    empty_cid = CID("base32", 1, "dag-pb", hash_digest)
+
+    verify_raw(str(empty_cid), empty_data)
+
+
+def test_verify_valid_cidv1_matches():
+    test_data = secrets.token_bytes(127)
+
+    hash_digest = multihash.digest(test_data, "sha2-256")
+    expected_cid = CID("base32", 1, "dag-pb", hash_digest)
+
+    verify(expected_cid, test_data)
+
+
+def test_verify_valid_cidv0_matches():
+    test_data = secrets.token_bytes(127)
+
+    hash_digest = multihash.digest(test_data, "sha2-256")
+    cidv0 = CID("base58btc", 0, "dag-pb", hash_digest)
+
+    verify(cidv0, test_data)
+
+
+def test_verify_cid_mismatch():
+    test_data = secrets.token_bytes(127)
+
+    hash_digest = multihash.digest(test_data, "sha2-256")
+    expected_cid = CID("base32", 1, "dag-pb", hash_digest)
+
+    wrong_data = b"different data"
+
+    with pytest.raises(CIDError) as exc_info:
+        verify(expected_cid, wrong_data)
+
+    assert "CID mismatch" in str(exc_info.value)
+
+
+def test_verify_different_hash_algorithms():
+    test_data = secrets.token_bytes(64)
+
+    hash_256 = multihash.digest(test_data, "sha2-256")
+    cid_256 = CID("base32", 1, "dag-pb", hash_256)
+    verify(cid_256, test_data)
+
+    hash_512 = multihash.digest(test_data, "sha2-512")
+    cid_512 = CID("base32", 1, "dag-pb", hash_512)
+    verify(cid_512, test_data)
+
+
+def test_verify_different_codecs():
+    test_data = secrets.token_bytes(64)
+
+    hash_digest = multihash.digest(test_data, "sha2-256")
+    cid_dagpb = CID("base32", 1, "dag-pb", hash_digest)
+    verify(cid_dagpb, test_data)
+
+    cid_raw = CID("base32", 1, "raw", hash_digest)
+    verify(cid_raw, test_data)
+
+
+def test_verify_large_data():
+    large_data = secrets.token_bytes(1024 * 1024)
+
+    hash_digest = multihash.digest(large_data, "sha2-256")
+    expected_cid = CID("base32", 1, "dag-pb", hash_digest)
+
+    verify(expected_cid, large_data)
+

private/eip712/__init__.py

@@ -0,0 +1,5 @@
+
+from .eip712 import Domain, TypedData, sign, recover_signer_address
+
+__all__ = ["Domain", "TypedData", "sign", "recover_signer_address"]
+