Release 4.3.0 (#300)

* generative flow improvements

* update readme with contributiong section

* update prompt-gen pipeline (#296)

* update prompt-gen pipeline

* flash attention, TypeError when wrong cache dir is read

* a bit less opportunistic verification

---------

Co-authored-by: kenobijon <kjmiyachi@gmail.com>
Co-authored-by: dylan <dylan@bitmind.ai>

* Preserve c2pa in gen miner services (#299)

* presrve c2pa in openrouter and openai services

* separate configurable servies for image/video

---------

Co-authored-by: dylan <dylan@bitmind.ai>

* only check for duplicates within same prompt id to avoid overhead of entire db

* fix: burn rate by excluding special UIDs from normalization (#297)

* library version updates

* fix c2pareader call

* option to save gen media locally

* conditional loading of local models depending on IMAGE_SERVICE and VIDEO_SERVICE

* preserve openrouter c2pa

* option to specify none for a service

* nanobanana pro default

* ttl on miner types

* move deepwiki to end of readme

* clean up fix comments

* bump version

* updating c2pa services

* temp cache clearing for this release

* get cache dir from .env.validator

---------

Co-authored-by: kenobijon <kjmiyachi@gmail.com>
Co-authored-by: dylan <dylan@bitmind.ai>
Co-authored-by: Kenobi <108417131+kenobijon@users.noreply.github.com>
Co-authored-by: aka James4u <smart.jamesjin@gmail.com>

Author: 4 people

.env.gen_miner.template

@@ -4,6 +4,19 @@
 # Copy this file to .env.miner and configure your settings
 # cp .env.gen_miner.template .env.gen_miner
 
+# =============================================================================
+# GENERATION SERVICE SELECTION
+# =============================================================================
+# Choose which service to use for each modality:
+#   openai     - DALL-E 3 (requires OPENAI_API_KEY)
+#   openrouter - Google Gemini via OpenRouter (requires OPEN_ROUTER_API_KEY)
+#   local      - Local Stable Diffusion models (requires GPU)
+#
+# If not set, loads ALL available services (wastes memory if using API services)
+IMAGE_SERVICE=
+VIDEO_SERVICE=
+
+
 # =============================================================================
 # API KEYS (configure the services you want to use)
 # =============================================================================
@@ -49,9 +62,12 @@ MINER_MAX_CONCURRENT_TASKS=5
 MINER_TASK_TIMEOUT=300
 
 # Storage
-MINER_OUTPUT_DIR=/workspace/miner_generated_content
 HF_HOME=/workspace/.cache/huggingface
 
+# Set to true to save generated images/videos to MINER_OUTPUT_DIR
+MINER_SAVE_LOCALLY=true
+MINER_OUTPUT_DIR=./miner_generated_content
+
 
 # =============================================================================
 # LOGGING

README.md

@@ -109,8 +109,8 @@ Validators are responsible for challenging and scoring both miner types. Generat
   </a>
 </p> 
 
-[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/BitMind-AI/bitmind-subnet)
-
 ## Contributing
 
 Contributions are welcome and can be made via a pull request to the `testnet` branch.
+
+[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/BitMind-AI/bitmind-subnet)

VERSION

@@ -1 +1 @@
-4.2.2
+4.3.0

gas/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "4.2.2"
+__version__ = "4.3.0"
 
 version_split = __version__.split(".")
 __spec_version__ = (

gas/cache/content_db.py

@@ -270,6 +270,55 @@ def _init_database(self):
             except Exception as e:
                 bt.logging.error(f"Error updating NULL uploaded values: {e}")
 
+            # Add perceptual_hash column for duplicate detection (migration)
+            try:
+                conn.execute("ALTER TABLE media ADD COLUMN perceptual_hash TEXT")
+                bt.logging.info("Added 'perceptual_hash' column to media table")
+                conn.execute(
+                    "CREATE INDEX IF NOT EXISTS idx_media_perceptual_hash ON media (perceptual_hash)"
+                )
+                conn.commit()
+            except Exception as e:
+                if "duplicate column name" not in str(e).lower() and "already exists" not in str(e).lower():
+                    bt.logging.error(f"Unexpected error adding 'perceptual_hash' column: {e}")
+                else:
+                    try:
+                        conn.execute(
+                            "CREATE INDEX IF NOT EXISTS idx_media_perceptual_hash ON media (perceptual_hash)"
+                        )
+                        conn.commit()
+                    except Exception:
+                        pass
+
+            # Add c2pa_verified column for C2PA content credentials verification (migration)
+            try:
+                conn.execute("ALTER TABLE media ADD COLUMN c2pa_verified BOOLEAN DEFAULT 0")
+                bt.logging.info("Added 'c2pa_verified' column to media table")
+                conn.execute(
+                    "CREATE INDEX IF NOT EXISTS idx_media_c2pa_verified ON media (c2pa_verified)"
+                )
+                conn.commit()
+            except Exception as e:
+                if "duplicate column name" not in str(e).lower() and "already exists" not in str(e).lower():
+                    bt.logging.error(f"Unexpected error adding 'c2pa_verified' column: {e}")
+                else:
+                    try:
+                        conn.execute(
+                            "CREATE INDEX IF NOT EXISTS idx_media_c2pa_verified ON media (c2pa_verified)"
+                        )
+                        conn.commit()
+                    except Exception:
+                        pass
+
+            # Add c2pa_issuer column for C2PA issuer tracking (migration)
+            try:
+                conn.execute("ALTER TABLE media ADD COLUMN c2pa_issuer TEXT")
+                bt.logging.info("Added 'c2pa_issuer' column to media table")
+                conn.commit()
+            except Exception as e:
+                if "duplicate column name" not in str(e).lower() and "already exists" not in str(e).lower():
+                    bt.logging.error(f"Unexpected error adding 'c2pa_issuer' column: {e}")
+
             # Temporary data hygiene: prune prompts whose source_media_id no longer exists in media table
             # Note: This only checks DB-level association, not filesystem existence.
             try:
@@ -405,6 +454,21 @@ def _row_to_media_entry(self, row) -> MediaEntry:
                 if "rewarded" in row.keys() and row["rewarded"] is not None
                 else False
             ),
+            perceptual_hash=(
+                row["perceptual_hash"]
+                if "perceptual_hash" in row.keys() and row["perceptual_hash"] is not None
+                else None
+            ),
+            c2pa_verified=(
+                bool(row["c2pa_verified"])
+                if "c2pa_verified" in row.keys() and row["c2pa_verified"] is not None
+                else False
+            ),
+            c2pa_issuer=(
+                row["c2pa_issuer"]
+                if "c2pa_issuer" in row.keys() and row["c2pa_issuer"] is not None
+                else None
+            ),
             created_at=row["created_at"],
             generation_args=(
                 json.loads(row["generation_args"]) if row["generation_args"] else None
@@ -436,6 +500,9 @@ def add_media_entry(
         resolution: Optional[tuple[int, int]] = None,
         file_size: Optional[int] = None,
         format: Optional[str] = None,
+        perceptual_hash: Optional[str] = None,
+        c2pa_verified: Optional[bool] = None,
+        c2pa_issuer: Optional[str] = None,
     ) -> str:
         media_id = str(uuid.uuid4())
         created_at = time.time()
@@ -449,8 +516,9 @@ def add_media_entry(
                 INSERT INTO media (id, prompt_id, file_path, modality, media_type, source_type,
                                  model_name, download_url, scraper_name, dataset_name, 
                                  dataset_source_file, dataset_index, uid, hotkey, verified,
-                                 created_at, generation_args, mask_path, timestamp, resolution, file_size, format)
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                                 created_at, generation_args, mask_path, timestamp, resolution, 
+                                 file_size, format, perceptual_hash, c2pa_verified, c2pa_issuer)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
             """,
                 (
                     media_id,
@@ -475,6 +543,9 @@ def add_media_entry(
                     json.dumps(list(resolution)) if resolution else None,
                     file_size,
                     format,
+                    perceptual_hash,
+                    c2pa_verified,
+                    c2pa_issuer,
                 ),
             )
             conn.commit()

gas/cache/content_manager.py

@@ -147,6 +147,9 @@ def write_miner_media(
 		content_type: str,
 		task_id: str,
 		model_name: Optional[str] = None,
+		perceptual_hash: Optional[str] = None,
+		c2pa_verified: Optional[bool] = None,
+		c2pa_issuer: Optional[str] = None,
 	) -> Optional[str]:
 		"""
 		Write miner-generated binary media to storage.
@@ -159,6 +162,9 @@ def write_miner_media(
 			content_type: MIME content type (e.g., "image/png", "video/mp4")
 			task_id: Unique task identifier for filename
 			model_name: Optional model name
+			perceptual_hash: Pre-computed perceptual hash for duplicate detection
+			c2pa_verified: Whether C2PA verification passed
+			c2pa_issuer: C2PA issuer name if verified
 
 		Returns:
 			Path to saved file if successful, None if failed
@@ -194,6 +200,9 @@ def write_miner_media(
 				resolution=resolution,
 				file_size=file_size,
 				format=media_data.format,
+				perceptual_hash=perceptual_hash,
+				c2pa_verified=c2pa_verified,
+				c2pa_issuer=c2pa_issuer,
 			)
 
 			bt.logging.info(f"Saved miner media to {save_path} with database ID: {media_id}")
@@ -870,3 +879,54 @@ def needs_more_data(self, source_type: SourceType, source_name: str) -> bool:
 			True if the source needs more data
 		"""
 		return self.get_source_count(source_type, source_name) < self.min_source_threshold
+
+	def check_duplicate(
+		self,
+		perceptual_hash: str,
+		threshold: int = 8,
+		limit: int = 1000,
+		prompt_id: Optional[str] = None,
+	) -> Optional[tuple]:
+		"""
+		Check if a perceptual hash has duplicates in the database.
+
+		Args:
+			perceptual_hash: Hash to check for duplicates
+			threshold: Maximum Hamming distance to consider as duplicate
+			limit: Maximum number of hashes to check
+			prompt_id: If provided, only check duplicates within this prompt
+
+		Returns:
+			Tuple of (media_id, hamming_distance) for closest match, or None if no duplicate
+		"""
+		from gas.verification.duplicate_detection import check_duplicate_in_db
+		return check_duplicate_in_db(self.content_db, perceptual_hash, threshold, limit, prompt_id=prompt_id)
+
+	def compute_and_check_duplicate(
+		self,
+		media_content: bytes,
+		modality: str,
+		threshold: int = 8,
+	) -> tuple:
+		"""
+		Compute perceptual hash for media and check for duplicates.
+
+		Args:
+			media_content: Raw media bytes
+			modality: "image" or "video"
+			threshold: Hamming distance threshold for duplicates
+
+		Returns:
+			Tuple of (perceptual_hash, is_duplicate, duplicate_info)
+			where duplicate_info is (media_id, distance) if duplicate found, else None
+		"""
+		from gas.verification.duplicate_detection import compute_media_hash
+
+		perceptual_hash = compute_media_hash(media_content, modality=modality)
+		if not perceptual_hash:
+			return None, False, None
+
+		duplicate_info = self.check_duplicate(perceptual_hash, threshold)
+		is_duplicate = duplicate_info is not None
+
+		return perceptual_hash, is_duplicate, duplicate_info

gas/cache/types.py

@@ -79,6 +79,13 @@ class MediaEntry:
 
     prompt_content: Optional[str] = None  # for hf uploads
 
+    # Duplicate detection
+    perceptual_hash: Optional[str] = None  # pHash for duplicate detection
+
+    # C2PA content credentials
+    c2pa_verified: Optional[bool] = False  # C2PA validation passed
+    c2pa_issuer: Optional[str] = None  # Issuer name if C2PA verified
+
     # Common fields
     created_at: float = None
     resolution: Optional[tuple[int, int]] = None  # (width, height)