From 1fa8280d6f9a76fc9749c005c9eb40c973633877 Mon Sep 17 00:00:00 2001 From: mannyuncharted Date: Sun, 13 Jul 2025 20:22:23 +0100 Subject: [PATCH 1/5] REFACTOR: refactored the code to fix the issues on documentation --- .vscode/settings.json | 3 + cmd/server/main.go | 92 +++++++++------------ go.mod | 1 + go.sum | 2 + internal/sdk/sdk.go | 51 ++++++------ replica/main.go | 180 ++++++++++++++++++++++++++++++++++++++++++ replica/sdk.go | 104 ++++++++++++++++++++++++ 7 files changed, 355 insertions(+), 78 deletions(-) create mode 100644 .vscode/settings.json create mode 100644 replica/main.go create mode 100644 replica/sdk.go diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..7709bc6 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "jira-plugin.workingProject": "" +} \ No newline at end of file diff --git a/cmd/server/main.go b/cmd/server/main.go index e16f4c5..41c9b10 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -1,85 +1,65 @@ -// cmd/server/main.go -// cmd/server/main.go -package main // Keep this as 'main' for the executable entry point // Keep this as 'main' for the executable entry point +// Package cmd provides the HTTP server that manages AkaveLink buckets and files. +// +// It exposes RESTful endpoints for health checks, bucket management, and file operations. +package main import ( "log" "net/http" "os" + "github.com/gorilla/mux" + akavesdk "github.com/akave-ai/go-akavelink/internal/sdk" - "github.com/akave-ai/go-akavelink/internal/utils" // Import your new utils package + "github.com/akave-ai/go-akavelink/internal/utils" ) -// server holds the application's dependencies, like our Akave client. +// server encapsulates dependencies for HTTP handlers. type server struct { client *akavesdk.Client } -// MainFunc contains the core logic of your application, -// making it testable by allowing external calls. -func MainFunc() { - // Load environment variables using the reusable function +// healthHandler responds with a simple status OK message. +func (s *server) healthHandler(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed) + return + } + + w.WriteHeader(http.StatusOK) + w.Write([]byte("ok")) +} + + +// main initializes the server and routes. +func main() { utils.LoadEnvConfig() - // 0. Read the private key from the environment variable. - privateKey := os.Getenv("AKAVE_PRIVATE_KEY") - if privateKey == "" { - log.Fatal("FATAL: AKAVE_PRIVATE_KEY environment variable not set. This is required.") - } - nodeAddress := os.Getenv("AKAVE_NODE_ADDRESS") - if nodeAddress == "" { - log.Fatal("FATAL: AKAVE_NODE_ADDRESS environment variable not set. This is required.") + key := os.Getenv("AKAVE_PRIVATE_KEY") + node := os.Getenv("AKAVE_NODE_ADDRESS") + if key == "" || node == "" { + log.Fatal("AKAVE_PRIVATE_KEY and AKAVE_NODE_ADDRESS are required") } - // 1. Configure and initialize our Akave client wrapper. cfg := akavesdk.Config{ - NodeAddress: nodeAddress, + NodeAddress: node, MaxConcurrency: 10, - BlockPartSize: 1024 * 1024, // 1MB + BlockPartSize: 1 << 20, UseConnectionPool: true, - PrivateKeyHex: privateKey, + PrivateKeyHex: key, } - client, err := akavesdk.NewClient(cfg) if err != nil { - log.Fatalf("Fatal error initializing Akave client: %v", err) + log.Fatalf("client initialization failed: %v", err) } - defer func() { - log.Println("Closing Akave SDK connection...") - if closeErr := client.Close(); closeErr != nil { - log.Printf("Error closing Akave SDK connection: %v", closeErr) - } else { - log.Println("Akave SDK connection closed successfully.") - } - }() - - // 2. Create a new server instance with the initialized client. - srv := &server{ - client: client, - } - - // 3. Register the handlers. - mux := http.NewServeMux() - mux.HandleFunc("/health", srv.healthHandler) + defer client.Close() + r := mux.NewRouter() + srv := &server{client: client} - log.Println("Starting go-akavelink server on :8080...") - if err := http.ListenAndServe(":8080", mux); err != nil { - log.Fatalf("Server failed: %v", err) - } - -} + r.HandleFunc("/health", srv.healthHandler).Methods("GET") -// The actual main entry point for the executable. -func main() { - MainFunc() + log.Println("Server listening on :8080") + log.Fatal(http.ListenAndServe(":8080", r)) } - - -// healthHandler is a method on the server. -func (s *server) healthHandler(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - w.Write([]byte("ok")) -} \ No newline at end of file diff --git a/go.mod b/go.mod index 4ffd959..aa4b18a 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.23.5 require ( github.com/akave-ai/akavesdk v0.2.0 + github.com/gorilla/mux v1.8.1 github.com/joho/godotenv v1.5.1 github.com/stretchr/testify v1.10.0 ) diff --git a/go.sum b/go.sum index 35c6fcc..9d4b125 100644 --- a/go.sum +++ b/go.sum @@ -131,6 +131,8 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= diff --git a/internal/sdk/sdk.go b/internal/sdk/sdk.go index 40784d1..9d3c813 100644 --- a/internal/sdk/sdk.go +++ b/internal/sdk/sdk.go @@ -1,3 +1,5 @@ +// Package sdk provides a high-level client for interacting with the AkaveLink IPC API. +// It encapsulates bucket and file operations and manages the underlying SDK connection. package sdk import ( @@ -6,58 +8,63 @@ import ( "github.com/akave-ai/akavesdk/sdk" ) +// Config defines parameters for initializing the IPC client. type Config struct { + // NodeAddress is the gRPC endpoint for the AkaveLink node, e.g., "localhost:9090". NodeAddress string + // MaxConcurrency controls parallelism for upload/download streams. MaxConcurrency int + // BlockPartSize specifies the size of each chunk in bytes. BlockPartSize int64 + // UseConnectionPool toggles the SDK's connection pool. UseConnectionPool bool + // PrivateKeyHex is the hex-encoded private key used for signing transactions. PrivateKeyHex string } +// Client wraps the AkaveLink IPC API and manages its SDK lifecycle. type Client struct { *sdk.IPC - sdk *sdk.SDK + core *sdk.SDK } +// NewClient initializes the AkaveLink SDK and returns a configured IPC client. +// It returns an error if the private key is missing or the SDK cannot initialize. func NewClient(cfg Config) (*Client, error) { - // Add a check to ensure the private key is provided. if cfg.PrivateKeyHex == "" { - return nil, fmt.Errorf("private key is required for IPC client but was not provided") + return nil, fmt.Errorf("configuration error: missing PrivateKeyHex") } - sdkOpts := []sdk.Option{ + opts := []sdk.Option{ sdk.WithPrivateKey(cfg.PrivateKeyHex), } - // Initialize the main Akave SDK with the base config AND our new options. - newSDK, err := sdk.New( + core, err := sdk.New( cfg.NodeAddress, cfg.MaxConcurrency, cfg.BlockPartSize, cfg.UseConnectionPool, - sdkOpts..., + opts..., ) if err != nil { - // If New() fails, it's likely due to an invalid key or other config. - return nil, fmt.Errorf("failed to initialize Akave SDK: %w", err) + return nil, fmt.Errorf("SDK initialization failed: %w", err) } - // Now, this call will succeed because newSDK already holds the private key. - ipc, err := newSDK.IPC() + ipcClient, err := core.IPC() if err != nil { - newSDK.Close() - return nil, fmt.Errorf("failed to get IPC client from Akave SDK: %w", err) + core.Close() + return nil, fmt.Errorf("failed to obtain IPC interface: %w", err) } - return &Client{ - IPC: ipc, - sdk: newSDK, - }, nil + return &Client{IPC: ipcClient, core: core}, nil } -// Close gracefully shuts down the connection to the Akave SDK. -// It's important to call this on application shutdown to release resources. +// NewIPC returns a fresh IPC interface instance with an updated transaction nonce. +func (c *Client) NewIPC() (*sdk.IPC, error) { + return c.core.IPC() +} + +// Close terminates all underlying SDK connections. func (c *Client) Close() error { - fmt.Println("Closing Akave SDK connection...") - return c.sdk.Close() -} \ No newline at end of file + return c.core.Close() +} diff --git a/replica/main.go b/replica/main.go new file mode 100644 index 0000000..bb5772c --- /dev/null +++ b/replica/main.go @@ -0,0 +1,180 @@ +// cmd/server/main.go + +package main + +import ( + "encoding/json" + "log" + "net/http" + "os" + "strings" + + akavesdk "github.com/akave-ai/go-akavelink/internal/sdk" + "github.com/akave-ai/go-akavelink/internal/utils" +) + +// AkaveResponse is our JSON envelope. +type AkaveResponse struct { + Success bool `json:"success"` + Data interface{} `json:"data,omitempty"` + Error string `json:"error,omitempty"` +} + +type server struct { + client *akavesdk.Client +} + +func (s *server) healthHandler(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + w.WriteHeader(http.StatusOK) + w.Write([]byte("ok")) +} + +func (s *server) bucketsHandler(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + buckets, err := s.client.ListBuckets() + if err != nil { + http.Error(w, "failed to list buckets: "+err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(AkaveResponse{Success: true, Data: buckets}) +} + +func (s *server) uploadHandler(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + // extract bucketName from /files/upload/{bucketName} + parts := strings.Split(r.URL.Path, "/") + if len(parts) < 4 || parts[3] == "" { + http.Error(w, "bucketName missing in path", http.StatusBadRequest) + return + } + bucketName := parts[3] + + if err := r.ParseMultipartForm(32 << 20); err != nil { + http.Error(w, "failed to parse form: "+err.Error(), http.StatusBadRequest) + return + } + + file, handler, err := r.FormFile("file") + if err != nil { + http.Error(w, "file retrieval error: "+err.Error(), http.StatusBadRequest) + return + } + defer file.Close() + + ctx := r.Context() + // Attempt to initialize upload stream + uploadStream, err := s.client.CreateFileUpload(ctx, bucketName, handler.Filename) + if err != nil { + // if bucket doesn't exist, create it and retry + if strings.Contains(err.Error(), "BucketNonexists") { + if err2 := s.client.CreateBucket(ctx, bucketName); err2 != nil { + http.Error(w, "bucket creation failed: "+err2.Error(), http.StatusInternalServerError) + return + } + // retry upload stream initialization + uploadStream, err = s.client.CreateFileUpload(ctx, bucketName, handler.Filename) + } + if err != nil { + http.Error(w, "upload init failed: "+err.Error(), http.StatusInternalServerError) + return + } + } + + // Now stream the file + meta, err := s.client.Upload(ctx, uploadStream, file) + if err != nil { + http.Error(w, "upload failed: "+err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + resp := map[string]interface{}{ + "message": "File uploaded successfully", + "rootCID": meta.RootCID, + "bucketName": meta.BucketName, + "fileName": meta.Name, + "size": meta.Size, + "encodedSize": meta.EncodedSize, + "committedAt": meta.CommittedAt, + } + json.NewEncoder(w).Encode(AkaveResponse{Success: true, Data: resp}) +} + + +func (s *server) downloadHandler(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + // extract bucketName & fileName from /files/download/{bucketName}/{fileName} + parts := strings.Split(r.URL.Path, "/") + if len(parts) < 5 { + http.Error(w, "path must be /files/download/{bucket}/{file}", http.StatusBadRequest) + return + } + bucketName := parts[3] + fileName := parts[4] + + dlStream, err := s.client.CreateFileDownload(r.Context(), bucketName, fileName) + if err != nil { + http.Error(w, "download init failed: "+err.Error(), http.StatusInternalServerError) + return + } + + if err := s.client.Download(r.Context(), dlStream, w); err != nil { + log.Printf("download error: %v", err) + } +} + +func MainFunc() { + utils.LoadEnvConfig() + + key := os.Getenv("AKAVE_PRIVATE_KEY") + node := os.Getenv("AKAVE_NODE_ADDRESS") + if key == "" || node == "" { + log.Fatal("AKAVE_PRIVATE_KEY and AKAVE_NODE_ADDRESS must be set") + } + + cfg := akavesdk.Config{ + NodeAddress: node, + MaxConcurrency: 10, + BlockPartSize: 1 << 20, + UseConnectionPool: true, + PrivateKeyHex: key, + } + client, err := akavesdk.NewClient(cfg) + if err != nil { + log.Fatalf("client init error: %v", err) + } + defer client.Close() + + srv := &server{client: client} + mux := http.NewServeMux() + mux.HandleFunc("/health", srv.healthHandler) + mux.HandleFunc("/buckets", srv.bucketsHandler) + mux.HandleFunc("/files/upload/", srv.uploadHandler) + mux.HandleFunc("/files/download/", srv.downloadHandler) + + log.Println("Server listening on :8080") + log.Fatal(http.ListenAndServe(":8080", mux)) +} + +func main() { + MainFunc() +} diff --git a/replica/sdk.go b/replica/sdk.go new file mode 100644 index 0000000..03ce328 --- /dev/null +++ b/replica/sdk.go @@ -0,0 +1,104 @@ +// internal/sdk/sdk.go + +package sdk + +import ( + "context" + "fmt" + "io" + + "github.com/akave-ai/akavesdk/sdk" +) + +// Config holds configuration for the Akave SDK client. +type Config struct { + NodeAddress string + MaxConcurrency int + BlockPartSize int64 + UseConnectionPool bool + PrivateKeyHex string +} + +// Client wraps the official Akave SDK's IPC interface and core instance. +type Client struct { + *sdk.IPC + sdk *sdk.SDK +} + +// NewClient constructs and returns a Client using the provided Config. +func NewClient(cfg Config) (*Client, error) { + if cfg.PrivateKeyHex == "" { + return nil, fmt.Errorf("private key is required for IPC client but was not provided") + } + + sdkOpts := []sdk.Option{ + sdk.WithPrivateKey(cfg.PrivateKeyHex), + } + + core, err := sdk.New( + cfg.NodeAddress, + cfg.MaxConcurrency, + cfg.BlockPartSize, + cfg.UseConnectionPool, + sdkOpts..., + ) + if err != nil { + return nil, fmt.Errorf("failed to initialize Akave SDK: %w", err) + } + + ipc, err := core.IPC() + if err != nil { + core.Close() + return nil, fmt.Errorf("failed to get IPC client from Akave SDK: %w", err) + } + + return &Client{IPC: ipc, sdk: core}, nil +} + +// Close gracefully shuts down the underlying SDK connection. +func (c *Client) Close() error { + return c.sdk.Close() +} + +// CreateBucket provisions a new bucket under the caller’s key. +func (c *Client) CreateBucket(ctx context.Context, bucketName string) error { + // call through to the IPC layer + if _, err := c.IPC.CreateBucket(ctx, bucketName); err != nil { + return fmt.Errorf("failed to create bucket %q: %w", bucketName, err) + } + return nil +} + +// ListBuckets returns the names of all buckets accessible to this client. +func (c *Client) ListBuckets() ([]string, error) { + buckets, err := c.IPC.ListBuckets(context.Background()) + if err != nil { + return nil, fmt.Errorf("failed to list buckets: %w", err) + } + + names := make([]string, len(buckets)) + for i, b := range buckets { + names[i] = b.Name + } + return names, nil +} + +// CreateFileUpload opens a new upload session for the given bucket and file name. +func (c *Client) CreateFileUpload(ctx context.Context, bucket, fileName string) (*sdk.IPCFileUpload, error) { + return c.IPC.CreateFileUpload(ctx, bucket, fileName) +} + +// Upload streams the given reader into the established upload session. +func (c *Client) Upload(ctx context.Context, upload *sdk.IPCFileUpload, reader io.Reader) (sdk.IPCFileMetaV2, error) { + return c.IPC.Upload(ctx, upload, reader) +} + +// CreateFileDownload opens a download session for the specified bucket and file. +func (c *Client) CreateFileDownload(ctx context.Context, bucket, fileName string) (sdk.IPCFileDownload, error) { + return c.IPC.CreateFileDownload(ctx, bucket, fileName) +} + +// Download writes the content of the download session to the provided writer. +func (c *Client) Download(ctx context.Context, download sdk.IPCFileDownload, writer io.Writer) error { + return c.IPC.Download(ctx, download, writer) +} From e38fcfbcb6a35e105cd7af8093d8b5751b7f5c40 Mon Sep 17 00:00:00 2001 From: Emmanuel Appah Date: Mon, 14 Jul 2025 12:01:55 +0100 Subject: [PATCH 2/5] Updated the documentation --- README.md | 92 ++++++++++++++++++++++++++----------------------------- 1 file changed, 44 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index cb25f9b..7215115 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,15 @@ + # go-akavelink -πŸš€ A Go-based HTTP server that wraps the Akave SDK, exposing Akave APIs over REST. The previous version of this repo was a CLI wrapper around the Akave SDK; refer to [akavelink](https://github.com/akave-ai/akavelink). +πŸš€ A Go-based HTTP server that wraps the Akave SDK, exposing Akave APIs over REST. The previous version of this repository was a CLI wrapper around the Akave SDK; refer to [akavelink](https://github.com/akave-ai/akavelink). ## Project Goals - - Provide a production-ready HTTP layer around Akave SDK - - Replace dependency on CLI-based wrappers - - Make it easy to integrate Akave storage into other systems via simple REST APIs +* Provide a production-ready HTTP layer around the Akave SDK. +* Replace dependency on CLI-based wrappers. +* Facilitate integration of Akave storage into other systems via simple REST APIs. + +--- ## Dev Setup @@ -15,19 +18,19 @@ Follow these steps to set up and run `go-akavelink` locally: 1. **Clone the Repository:** ```bash - git clone https://github.com/akave-ai/go-akavelink + git clone [https://github.com/akave-ai/go-akavelink](https://github.com/akave-ai/go-akavelink) cd go-akavelink ``` -2. **Get Akave Tokens and Private Key:** +2. **Obtain Akave Tokens and Private Key:** - * Go to the Akave Faucet: [https://faucet.akave.ai/](https://faucet.akave.ai/) - * Add the Akave network to your wallet. - * Claim your tokens. - * Obtain your private key from your wallet. + * Access the Akave Faucet: [https://faucet.akave.ai/](https://faucet.akave.ai/) + * Add the Akave network to a wallet. + * Claim tokens. + * Obtain the private key from the wallet. 3. **Configure Environment Variables:** - Create a `.env` file in the root of your `go-akavelink` directory with the following content, replacing `YOUR_PRIVATE_KEY_HERE` with the private key you obtained: + Create a `.env` file in the root of the `go-akavelink` directory with the following content, replacing `YOUR_PRIVATE_KEY_HERE` with the obtained private key: ``` AKAVE_PRIVATE_KEY="YOUR_PRIVATE_KEY_HERE" @@ -39,49 +42,40 @@ Follow these steps to set up and run `go-akavelink` locally: The `scripts/` directory contains helper scripts (`setup.sh` and `setup.bat`) to automate the environment variable export process. **For macOS/Linux:** - Navigate to the `scripts` directory and give execute permissions, then run the script: + Navigate to the `scripts` directory and grant execute permissions, then run the script: ```bash chmod +x scripts/setup.sh ./scripts/setup.sh - # Or if you prefer bash specifically - # chmod +x scripts/setup.bat - # ./scripts/setup.bat ``` - This script will export the variables from your `.env` file into your current terminal session. To verify, you can run: + This script exports the variables from the `.env` file into the current terminal session. To verify, run: ```bash echo $AKAVE_PRIVATE_KEY echo $AKAVE_NODE_ADDRESS ``` - These variables will persist for your current terminal session. For permanent environment variables, consider adding them to your shell's configuration file (e.g., `~/.bashrc`, `~/.zshrc`, or `~/.profile`). + These variables will persist for the current terminal session. For permanent environment variables, consider adding them to a shell's configuration file (e.g., `~/.bashrc`, `~/.zshrc`, or `~/.profile`). **For Windows PowerShell:** - You might need to adjust your PowerShell execution policy to run scripts. Open PowerShell as an administrator and run: + PowerShell execution policy might need adjustment to run scripts. Open PowerShell as an administrator and run: ```powershell Set-ExecutionPolicy RemoteSigned -Scope CurrentUser ``` - Then, you can run the script (note: Windows Subsystem for Linux (WSL) is recommended for running `.sh` scripts on Windows for a more native experience): + Then, it is recommended to use Windows Subsystem for Linux (WSL) or Git Bash to run the `.sh` script for a more native experience. + Alternatively, to manually export variables in PowerShell: ```powershell - # If using PowerShell directly, you might need to execute it like this: - # powershell -File .\scripts\setup.ps1 (assuming a PowerShell equivalent script) - - # However, for .sh or .bat scripts, it's best to use Git Bash or WSL. - # If you have Git Bash installed: - # bash scripts/setup.sh - # Or, manually export variables as described below if you prefer not to use WSL/Git Bash: Get-Content .env | ForEach-Object { $line = $_.Trim() if (-not ([string]::IsNullOrEmpty($line)) -and -not $line.StartsWith("#")) { $parts = $line.Split('=', 2) if ($parts.Length -eq 2) { $varName = $parts[0] - $varValue = $parts[1].Trim('"') # Remove quotes if present + $varValue = $parts[1].Trim('"') [System.Environment]::SetEnvironmentVariable($varName, $varValue, "Process") Write-Host "Exported variable: $varName" } @@ -89,14 +83,14 @@ Follow these steps to set up and run `go-akavelink` locally: } ``` - To verify they are loaded in your current PowerShell session, you can run: + To verify variables are loaded in the current PowerShell session, run: ```powershell Get-Item Env:AKAVE_PRIVATE_KEY Get-Item Env:AKAVE_NODE_ADDRESS ``` - These variables will persist for this PowerShell session. For permanent environment variables, refer to Windows documentation on setting system or user-specific environment variables. + These variables will persist for the current PowerShell session. For permanent environment variables, refer to Windows documentation on setting system or user-specific environment variables. 5. **Install Go Modules:** Before running the server, ensure all Go modules are tidy and downloaded: @@ -111,41 +105,43 @@ Follow these steps to set up and run `go-akavelink` locally: go run ./cmd/server ``` - You should see output similar to: + Output similar to the following should appear: ``` 2025/07/07 03:17:14 Starting go-akavelink server on :8080... ``` 7. **Verify Installation:** - Visit `http://localhost:8080/health` in your web browser to verify that the server is running correctly. + Visit `http://localhost:8080/health` in a web browser to verify that the server is running correctly. ------ +--- ## Project Structure -``` +```markdown + go-akavelink/ -β”œβ”€β”€ cmd/ # Main entrypoint +β”œβ”€β”€ cmd/ \# Main entrypoint for executables β”‚ └── server/ -β”‚ └── main.go # Starts HTTP server -β”œβ”€β”€ internal/ # Internal logic, not exported -β”‚ └── sdk/ # Wrapper around Akave SDK -β”œβ”€β”€ pkg/ # Public packages (if needed) -β”œβ”€β”€ docs/ # Architecture, design, etc. -β”œβ”€β”€ scripts/ # Helper scripts (e.g., setup.sh for env vars) +β”‚ └── main.go \# Starts the HTTP server +β”œβ”€β”€ internal/ \# Internal logic, not intended for external consumption +β”‚ └── sdk/ \# Wrapper around the Akave SDK +β”œβ”€β”€ pkg/ \# Public packages (if needed) +β”œβ”€β”€ docs/ \# Architecture, design, and other documentation +β”œβ”€β”€ scripts/ \# Helper scripts (e.g., setup.sh for environment variables) β”‚ β”œβ”€β”€ setup.sh β”‚ └── setup.bat -β”œβ”€β”€ go.mod # Go module definition -β”œβ”€β”€ README.md # This file -β”œβ”€β”€ CONTRIBUTING.md # Guide for contributors +β”œβ”€β”€ go.mod \# Go module definition file +β”œβ”€β”€ README.md \# Project overview and setup instructions +└── CONTRIBUTING.md \# Guide for project contributors + ``` -## Contributing +--- -This repo is open to contributions\! See [`CONTRIBUTING.md`](https://www.google.com/search?q=./CONTRIBUTING.md). +## Contributing - - Check the [issue tracker](https://github.com/akave-ai/go-akavelink/issues) for `good first issue` and `help wanted` labels. - - Follow the PR checklist and formatting conventions. +This repository is open to contributions! See [`CONTRIBUTING.md`](./CONTRIBUTING.md). ------ \ No newline at end of file +* Check the [issue tracker](https://github.com/akave-ai/go-akavelink/issues) for `good first issue` and `help wanted` labels. +* Follow the pull request checklist and formatting conventions. From cfdb55915e60e3a4c5dafd67b65b7b08e2c14829 Mon Sep 17 00:00:00 2001 From: manny-uncharted Date: Thu, 23 Oct 2025 11:57:10 +0100 Subject: [PATCH 3/5] feat: implemented the validation tests --- README.md | 77 ++++- docs/VALIDATION.md | 468 ++++++++++++++++++++++++++++ go.mod | 1 + go.sum | 2 + internal/handlers/buckets.go | 19 +- internal/handlers/files.go | 102 ++++-- internal/handlers/router.go | 14 +- internal/middleware/validation.go | 158 ++++++++++ internal/validation/validator.go | 284 +++++++++++++++++ test/middleware_test.go | 261 ++++++++++++++++ test/validation_integration_test.go | 402 ++++++++++++++++++++++++ test/validation_test.go | 465 +++++++++++++++++++++++++++ 12 files changed, 2225 insertions(+), 28 deletions(-) create mode 100644 docs/VALIDATION.md create mode 100644 internal/middleware/validation.go create mode 100644 internal/validation/validator.go create mode 100644 test/middleware_test.go create mode 100644 test/validation_integration_test.go create mode 100644 test/validation_test.go diff --git a/README.md b/README.md index a6a1456..19ed952 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,72 @@ Implemented routes (see `internal/handlers/`): --- +## Input Validation & Security + +The API implements comprehensive input validation and sanitization to protect against common attacks: + +### Bucket Name Validation + +- **Length**: 1-63 characters +- **Allowed characters**: Alphanumeric (a-z, A-Z, 0-9), hyphens (-), underscores (_) +- **Format**: Must start and end with an alphanumeric character +- **Security**: Prevents path traversal attacks, special characters, and malicious patterns + +**Examples:** +- βœ… Valid: `my-bucket`, `data_store_123`, `bucket1` +- ❌ Invalid: `../etc`, `my bucket`, `bucket!@#`, `-bucket`, `bucket-` + +### File Name Validation + +- **Length**: Maximum 255 characters +- **Allowed characters**: Alphanumeric (a-z, A-Z, 0-9), dots (.), hyphens (-), underscores (_) +- **Format**: Must start and end with an alphanumeric character +- **Security**: Prevents path traversal, null bytes, and invalid filename characters + +**Examples:** +- βœ… Valid: `document.pdf`, `my-file_v2.txt`, `data.backup.tar.gz` +- ❌ Invalid: `../../../etc/passwd`, `file<>name.txt`, `.hidden`, `file.` + +### File Upload Validation + +- **Maximum file size**: 100 MB (104,857,600 bytes) +- **Minimum file size**: Must not be empty (> 0 bytes) +- **Allowed MIME types**: + - `application/octet-stream` (binary/default) + - `text/plain`, `text/csv` + - `application/json`, `application/pdf` + - `image/jpeg`, `image/png`, `image/gif`, `image/webp` + - `video/mp4` + - `audio/mpeg` + +**Security features:** +- Content-Length validation before parsing +- MIME type verification +- File name sanitization +- Protection against malicious file uploads + +### Error Responses + +Validation errors return HTTP 400 (Bad Request) with a JSON response: + +```json +{ + "error": "Validation Error", + "field": "bucketName", + "message": "bucket name must contain only alphanumeric characters, hyphens, and underscores" +} +``` + +### Security Headers + +All responses include security headers: +- `X-Content-Type-Options: nosniff` - Prevents MIME type sniffing +- `X-Frame-Options: DENY` - Prevents clickjacking +- `X-XSS-Protection: 1; mode=block` - Enables XSS protection +- `Content-Security-Policy: default-src 'self'` - Restricts resource loading + +--- + ## Project Structure ```markdown @@ -120,11 +186,15 @@ go-akavelink/ β”‚ └── architecture.md β”œβ”€β”€ internal/ β”‚ β”œβ”€β”€ handlers/ -β”‚ β”‚ β”œβ”€β”€ router.go # Wires routes only +β”‚ β”‚ β”œβ”€β”€ router.go # Wires routes and middleware β”‚ β”‚ β”œβ”€β”€ response.go # JSON envelope + helpers β”‚ β”‚ β”œβ”€β”€ health.go # /health β”‚ β”‚ β”œβ”€β”€ buckets.go # bucket endpoints β”‚ β”‚ └── files.go # file endpoints +β”‚ β”œβ”€β”€ middleware/ +β”‚ β”‚ └── validation.go # Validation middleware +β”‚ β”œβ”€β”€ validation/ +β”‚ β”‚ └── validator.go # Input validation & sanitization β”‚ β”œβ”€β”€ sdk/ β”‚ β”‚ └── sdk.go β”‚ └── utils/ @@ -133,7 +203,10 @@ go-akavelink/ β”œβ”€β”€ http_endpoints_test.go β”œβ”€β”€ integration_sdk_test.go β”œβ”€β”€ main_test.go - └── sdk_test.go + β”œβ”€β”€ sdk_test.go + β”œβ”€β”€ validation_test.go + β”œβ”€β”€ middleware_test.go + └── validation_integration_test.go ``` --- diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md new file mode 100644 index 0000000..b1e20e2 --- /dev/null +++ b/docs/VALIDATION.md @@ -0,0 +1,468 @@ +# Input Validation & Security + +This document describes the comprehensive input validation and sanitization system implemented in go-akavelink. + +## Overview + +The API implements multiple layers of security to protect against common attacks: + +1. **Input Validation** - Validates all user inputs against strict rules +2. **Input Sanitization** - Cleans and normalizes inputs for safe processing +3. **Security Headers** - Adds HTTP security headers to all responses +4. **Request Logging** - Logs all incoming requests for audit trails + +## Validation Rules + +### Bucket Names + +Bucket names must adhere to the following rules: + +- **Length**: Between 1 and 63 characters +- **Allowed Characters**: + - Lowercase letters (a-z) + - Uppercase letters (A-Z) + - Numbers (0-9) + - Hyphens (-) + - Underscores (_) +- **Format Requirements**: + - Must start with an alphanumeric character + - Must end with an alphanumeric character + - Cannot contain consecutive dots (..) + - Cannot contain path separators (/ or \) + - Cannot contain special characters (!@#$%^&*()+=[]{}|;:'",<>?) + +**Valid Examples:** +``` +my-bucket +data_store_123 +bucket1 +MyBucket-2024 +user_data_v2 +``` + +**Invalid Examples:** +``` +../etc/passwd # Path traversal +my bucket # Contains space +bucket!@# # Special characters +-mybucket # Starts with hyphen +mybucket- # Ends with hyphen +a1234567890...123 # Too long (>63 chars) +``` + +### File Names + +File names must adhere to the following rules: + +- **Length**: Maximum 255 characters +- **Allowed Characters**: + - Lowercase letters (a-z) + - Uppercase letters (A-Z) + - Numbers (0-9) + - Dots (.) + - Hyphens (-) + - Underscores (_) +- **Format Requirements**: + - Must start with an alphanumeric character + - Must end with an alphanumeric character + - Cannot contain path traversal patterns (..) + - Cannot contain path separators (/ or \) + - Cannot contain null bytes (\x00) + - Cannot contain invalid filename characters (<>:"|?*) + +**Valid Examples:** +``` +document.pdf +my-file_v2.txt +data.backup.tar.gz +image123.jpg +report-2024-01-15.csv +``` + +**Invalid Examples:** +``` +../../../etc/passwd # Path traversal +/etc/passwd # Absolute path +file<>name.txt # Invalid characters +file\x00name.txt # Null byte +.hidden # Starts with dot +file. # Ends with dot +path/to/file.txt # Contains path separator +``` + +### File Uploads + +File uploads are validated for size, type, and content: + +#### Size Limits + +- **Maximum Size**: 100 MB (104,857,600 bytes) +- **Minimum Size**: Must not be empty (> 0 bytes) + +#### Allowed MIME Types + +The following MIME types are permitted: + +| Category | MIME Type | Description | +|----------|-----------|-------------| +| Binary | `application/octet-stream` | Generic binary data | +| Text | `text/plain` | Plain text files | +| Text | `text/csv` | CSV files | +| JSON | `application/json` | JSON data | +| PDF | `application/pdf` | PDF documents | +| Images | `image/jpeg` | JPEG images | +| Images | `image/png` | PNG images | +| Images | `image/gif` | GIF images | +| Images | `image/webp` | WebP images | +| Video | `video/mp4` | MP4 videos | +| Audio | `audio/mpeg` | MP3 audio | + +**Note**: MIME type validation extracts the base type (before semicolon) to handle charset and other parameters. + +#### Upload Validation Process + +1. **Content-Length Check**: Validates the Content-Length header before parsing +2. **Multipart Form Parsing**: Parses the multipart form data (max 32 MiB in memory) +3. **File Header Validation**: Validates the file header metadata +4. **File Name Validation**: Validates and sanitizes the filename +5. **Size Validation**: Ensures file size is within limits +6. **MIME Type Validation**: Verifies the Content-Type header + +## Sanitization + +In addition to validation, the API sanitizes inputs for extra safety: + +### Bucket Name Sanitization + +```go +// Removes whitespace +// Converts to lowercase +// Removes invalid characters +// Truncates to max length +sanitized := validation.SanitizeBucketName(input) +``` + +**Example:** +``` +Input: " My-Bucket!@# " +Output: "my-bucket" +``` + +### File Name Sanitization + +```go +// Extracts base filename (removes path) +// Removes null bytes +// Removes path traversal patterns +// Removes path separators +// Truncates to max length (preserving extension) +sanitized := validation.SanitizeFileName(input) +``` + +**Example:** +``` +Input: "/path/to/../file.txt" +Output: "file.txt" +``` + +## Security Features + +### Path Traversal Prevention + +The validation system detects and blocks various path traversal attempts: + +- Parent directory references (`..`) +- Absolute paths (`/` or `\`) +- URL-encoded traversal (`%2e%2e`, `%2f`, `%5c`) +- Null bytes (`\x00`) +- Invalid filename characters + +### Attack Patterns Detected + +The following malicious patterns are detected and rejected: + +``` +../etc/passwd +..\..\windows\system32 +%2e%2e%2f +file\x00.txt + +../../../../../../etc/shadow +``` + +### Security Headers + +All API responses include the following security headers: + +```http +X-Content-Type-Options: nosniff +X-Frame-Options: DENY +X-XSS-Protection: 1; mode=block +Content-Security-Policy: default-src 'self' +``` + +#### Header Descriptions + +- **X-Content-Type-Options**: Prevents browsers from MIME-sniffing responses +- **X-Frame-Options**: Prevents the page from being embedded in frames (clickjacking protection) +- **X-XSS-Protection**: Enables browser XSS filtering +- **Content-Security-Policy**: Restricts resource loading to same origin + +## Error Responses + +### Validation Errors + +When validation fails, the API returns HTTP 400 (Bad Request) with a structured error response: + +```json +{ + "error": "Validation Error", + "field": "bucketName", + "message": "bucket name must contain only alphanumeric characters, hyphens, and underscores, and must start and end with an alphanumeric character" +} +``` + +### Error Response Fields + +- **error**: Always "Validation Error" for validation failures +- **field**: The name of the field that failed validation (e.g., "bucketName", "fileName", "file") +- **message**: A human-readable description of the validation error + +### Common Validation Errors + +#### Bucket Name Errors + +```json +{ + "error": "Validation Error", + "field": "bucketName", + "message": "bucket name is required" +} +``` + +```json +{ + "error": "Validation Error", + "field": "bucketName", + "message": "bucket name must be between 1 and 63 characters" +} +``` + +```json +{ + "error": "Validation Error", + "field": "bucketName", + "message": "bucket name contains invalid characters or patterns" +} +``` + +#### File Name Errors + +```json +{ + "error": "Validation Error", + "field": "fileName", + "message": "file name is required" +} +``` + +```json +{ + "error": "Validation Error", + "field": "fileName", + "message": "file name contains invalid characters or path traversal patterns" +} +``` + +#### File Upload Errors + +```json +{ + "error": "Validation Error", + "field": "file", + "message": "file size exceeds maximum allowed size of 104857600 bytes" +} +``` + +```json +{ + "error": "Validation Error", + "field": "file", + "message": "file is empty" +} +``` + +```json +{ + "error": "Validation Error", + "field": "file", + "message": "file type 'application/x-msdownload' is not allowed" +} +``` + +## Implementation Details + +### Package Structure + +``` +internal/ +β”œβ”€β”€ validation/ +β”‚ └── validator.go # Core validation logic +β”œβ”€β”€ middleware/ +β”‚ └── validation.go # HTTP middleware +└── handlers/ + β”œβ”€β”€ buckets.go # Bucket handlers with validation + └── files.go # File handlers with validation +``` + +### Validation Package + +The `internal/validation` package provides: + +- `ValidateBucketName(name string) error` +- `ValidateFileName(name string) error` +- `ValidateFileUpload(header *multipart.FileHeader) error` +- `ValidateContentLength(contentLength int64) error` +- `SanitizeBucketName(name string) string` +- `SanitizeFileName(name string) string` + +### Middleware Package + +The `internal/middleware` package provides: + +- `ValidateBucketName(next http.Handler) http.Handler` +- `ValidateFileName(next http.Handler) http.Handler` +- `ValidateBucketAndFileName(next http.Handler) http.Handler` +- `ValidateContentLength(next http.Handler) http.Handler` +- `SecurityHeaders(next http.Handler) http.Handler` +- `LogRequest(next http.Handler) http.Handler` + +### Handler Integration + +All handlers validate and sanitize inputs: + +```go +// Validate bucket name +if err := validation.ValidateBucketName(bucketName); err != nil { + s.writeErrorResponse(w, http.StatusBadRequest, err.Error()) + return +} + +// Sanitize bucket name for extra safety +bucketName = validation.SanitizeBucketName(bucketName) +``` + +## Testing + +Comprehensive test coverage includes: + +### Unit Tests + +- `test/validation_test.go` - Tests all validation functions +- `test/middleware_test.go` - Tests all middleware functions + +### Integration Tests + +- `test/validation_integration_test.go` - Tests validation in HTTP handlers + +### Test Coverage + +The test suite covers: + +- Valid inputs (positive tests) +- Invalid inputs (negative tests) +- Edge cases (boundary conditions) +- Security attacks (path traversal, injection, etc.) +- Sanitization behavior +- Error message accuracy + +### Running Tests + +```bash +# Run all tests +go test ./test/... + +# Run validation tests only +go test ./test/ -run Validation + +# Run with coverage +go test ./test/... -cover + +# Run with verbose output +go test ./test/... -v +``` + +## Best Practices + +### For API Users + +1. **Always validate inputs client-side** before sending requests +2. **Use descriptive names** that follow the validation rules +3. **Handle validation errors gracefully** in your application +4. **Check file sizes** before attempting uploads +5. **Use appropriate MIME types** for your files + +### For Developers + +1. **Always validate before sanitizing** to catch malicious inputs +2. **Log validation failures** for security monitoring +3. **Never trust user input** - validate everything +4. **Keep validation rules consistent** across all endpoints +5. **Update tests** when adding new validation rules +6. **Document validation changes** in the changelog + +## Security Considerations + +### Defense in Depth + +The validation system implements multiple layers of security: + +1. **Input Validation** - First line of defense +2. **Input Sanitization** - Second layer of protection +3. **Security Headers** - Browser-level protection +4. **Request Logging** - Audit trail for security incidents + +### Threat Mitigation + +The system protects against: + +- **Path Traversal Attacks** - Prevented by pattern detection +- **Malicious File Uploads** - Blocked by size and MIME type validation +- **Injection Attacks** - Mitigated by input sanitization +- **XSS Attacks** - Prevented by security headers +- **Clickjacking** - Blocked by X-Frame-Options header +- **MIME Sniffing** - Prevented by X-Content-Type-Options header + +### Logging and Monitoring + +All validation failures are logged for security monitoring: + +``` +Warning: Bucket name was sanitized from '../etc' to 'etc' +``` + +Monitor these logs for: +- Repeated validation failures from the same IP +- Path traversal attempts +- Unusual file upload patterns +- Suspicious input patterns + +## Future Enhancements + +Potential improvements to the validation system: + +1. **Rate Limiting** - Implement per-IP rate limiting +2. **Content Scanning** - Add virus/malware scanning for uploads +3. **Advanced MIME Detection** - Use magic bytes for MIME type verification +4. **Configurable Limits** - Make size limits configurable via environment variables +5. **Custom Validation Rules** - Allow users to define custom validation rules +6. **Audit Logging** - Enhanced logging with structured audit trails +7. **Metrics** - Track validation failure rates and patterns + +## References + +- [OWASP Input Validation Cheat Sheet](https://cheatsheetseries.owasp.org/cheatsheets/Input_Validation_Cheat_Sheet.html) +- [OWASP Path Traversal](https://owasp.org/www-community/attacks/Path_Traversal) +- [OWASP Secure Headers Project](https://owasp.org/www-project-secure-headers/) +- [CWE-22: Path Traversal](https://cwe.mitre.org/data/definitions/22.html) +- [CWE-434: Unrestricted Upload of File with Dangerous Type](https://cwe.mitre.org/data/definitions/434.html) diff --git a/go.mod b/go.mod index ee35346..062376d 100644 --- a/go.mod +++ b/go.mod @@ -63,6 +63,7 @@ require ( github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect github.com/spacemonkeygo/monkit/v3 v3.0.24 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect github.com/supranational/blst v0.3.14 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect diff --git a/go.sum b/go.sum index f46aa31..b405993 100644 --- a/go.sum +++ b/go.sum @@ -357,6 +357,8 @@ github.com/spacemonkeygo/monkit/v3 v3.0.24/go.mod h1:XkZYGzknZwkD0AKUnZaSXhRiVTL github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= diff --git a/internal/handlers/buckets.go b/internal/handlers/buckets.go index fd42345..64b79ca 100644 --- a/internal/handlers/buckets.go +++ b/internal/handlers/buckets.go @@ -5,6 +5,7 @@ import ( "log" "net/http" + "github.com/akave-ai/go-akavelink/internal/validation" "github.com/gorilla/mux" ) @@ -17,11 +18,16 @@ func (s *Server) createBucketHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucketName := vars["bucketName"] - if bucketName == "" { - s.writeErrorResponse(w, http.StatusBadRequest, "bucketName is required") + + // Validate bucket name + if err := validation.ValidateBucketName(bucketName); err != nil { + s.writeErrorResponse(w, http.StatusBadRequest, err.Error()) return } + // Sanitize bucket name for extra safety + bucketName = validation.SanitizeBucketName(bucketName) + if err := s.client.CreateBucket(r.Context(), bucketName); err != nil { s.writeErrorResponse(w, http.StatusInternalServerError, "Failed to create bucket") log.Printf("create bucket error: %v", err) @@ -42,11 +48,16 @@ func (s *Server) deleteBucketHandler(w http.ResponseWriter, r *http.Request) { } vars := mux.Vars(r) bucketName := vars["bucketName"] - if bucketName == "" { - s.writeErrorResponse(w, http.StatusBadRequest, "bucketName is required") + + // Validate bucket name + if err := validation.ValidateBucketName(bucketName); err != nil { + s.writeErrorResponse(w, http.StatusBadRequest, err.Error()) return } + // Sanitize bucket name for extra safety + bucketName = validation.SanitizeBucketName(bucketName) + ctx := r.Context() files, err := s.client.ListFiles(ctx, bucketName) if err != nil { diff --git a/internal/handlers/files.go b/internal/handlers/files.go index d7d8fa5..69c1c12 100644 --- a/internal/handlers/files.go +++ b/internal/handlers/files.go @@ -4,6 +4,7 @@ import ( "log" "net/http" + "github.com/akave-ai/go-akavelink/internal/validation" "github.com/gorilla/mux" ) @@ -12,11 +13,23 @@ func (s *Server) fileInfoHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucketName := vars["bucketName"] fileName := vars["fileName"] - if bucketName == "" || fileName == "" { - s.writeErrorResponse(w, http.StatusBadRequest, "bucketName and fileName are required") + + // Validate bucket name + if err := validation.ValidateBucketName(bucketName); err != nil { + s.writeErrorResponse(w, http.StatusBadRequest, err.Error()) + return + } + + // Validate file name + if err := validation.ValidateFileName(fileName); err != nil { + s.writeErrorResponse(w, http.StatusBadRequest, err.Error()) return } + // Sanitize inputs + bucketName = validation.SanitizeBucketName(bucketName) + fileName = validation.SanitizeFileName(fileName) + info, err := s.client.FileInfo(r.Context(), bucketName, fileName) if err != nil { s.writeErrorResponse(w, http.StatusInternalServerError, "Failed to retrieve file info") @@ -31,11 +44,16 @@ func (s *Server) fileInfoHandler(w http.ResponseWriter, r *http.Request) { func (s *Server) listFilesHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucketName := vars["bucketName"] - if bucketName == "" { - s.writeErrorResponse(w, http.StatusBadRequest, "bucketName is required") + + // Validate bucket name + if err := validation.ValidateBucketName(bucketName); err != nil { + s.writeErrorResponse(w, http.StatusBadRequest, err.Error()) return } + // Sanitize bucket name + bucketName = validation.SanitizeBucketName(bucketName) + files, err := s.client.ListFiles(r.Context(), bucketName) if err != nil { s.writeErrorResponse(w, http.StatusInternalServerError, "Failed to list files") @@ -50,8 +68,19 @@ func (s *Server) listFilesHandler(w http.ResponseWriter, r *http.Request) { func (s *Server) uploadHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucketName := vars["bucketName"] - if bucketName == "" { - s.writeErrorResponse(w, http.StatusBadRequest, "bucketName is required") + + // Validate bucket name + if err := validation.ValidateBucketName(bucketName); err != nil { + s.writeErrorResponse(w, http.StatusBadRequest, err.Error()) + return + } + + // Sanitize bucket name + bucketName = validation.SanitizeBucketName(bucketName) + + // Validate content length before parsing + if err := validation.ValidateContentLength(r.ContentLength); err != nil { + s.writeErrorResponse(w, http.StatusBadRequest, err.Error()) return } @@ -73,8 +102,17 @@ func (s *Server) uploadHandler(w http.ResponseWriter, r *http.Request) { } }() + // Validate file upload (size, MIME type, filename) + if err := validation.ValidateFileUpload(handler); err != nil { + s.writeErrorResponse(w, http.StatusBadRequest, err.Error()) + return + } + + // Sanitize filename + fileName := validation.SanitizeFileName(handler.Filename) + ctx := r.Context() - upload, err := s.client.CreateFileUpload(ctx, bucketName, handler.Filename) + upload, err := s.client.CreateFileUpload(ctx, bucketName, fileName) if err != nil { s.writeErrorResponse(w, http.StatusInternalServerError, "Failed to create file upload stream") log.Printf("create upload error: %v", err) @@ -89,14 +127,16 @@ func (s *Server) uploadHandler(w http.ResponseWriter, r *http.Request) { } resp := map[string]interface{}{ - "message": "File uploaded successfully", - "rootCID": meta.RootCID, - "bucketName": meta.BucketName, - "fileName": meta.Name, - "size": meta.Size, - "encodedSize": meta.EncodedSize, - "createdAt": meta.CreatedAt, - "committedAt": meta.CommittedAt, + "message": "File uploaded successfully", + "rootCID": meta.RootCID, + "bucketName": meta.BucketName, + "fileName": meta.Name, + "originalName": handler.Filename, + "sanitizedName": fileName, + "size": meta.Size, + "encodedSize": meta.EncodedSize, + "createdAt": meta.CreatedAt, + "committedAt": meta.CommittedAt, } s.writeSuccessResponse(w, http.StatusCreated, resp) } @@ -106,11 +146,23 @@ func (s *Server) downloadHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucketName := vars["bucketName"] fileName := vars["fileName"] - if bucketName == "" || fileName == "" { - s.writeErrorResponse(w, http.StatusBadRequest, "bucketName and fileName are required") + + // Validate bucket name + if err := validation.ValidateBucketName(bucketName); err != nil { + s.writeErrorResponse(w, http.StatusBadRequest, err.Error()) + return + } + + // Validate file name + if err := validation.ValidateFileName(fileName); err != nil { + s.writeErrorResponse(w, http.StatusBadRequest, err.Error()) return } + // Sanitize inputs + bucketName = validation.SanitizeBucketName(bucketName) + fileName = validation.SanitizeFileName(fileName) + dl, err := s.client.CreateFileDownload(r.Context(), bucketName, fileName) if err != nil { s.writeErrorResponse(w, http.StatusNotFound, "Failed to create file download") @@ -131,11 +183,23 @@ func (s *Server) fileDeleteHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucketName := vars["bucketName"] fileName := vars["fileName"] - if bucketName == "" || fileName == "" { - s.writeErrorResponse(w, http.StatusBadRequest, "bucketName and fileName are required") + + // Validate bucket name + if err := validation.ValidateBucketName(bucketName); err != nil { + s.writeErrorResponse(w, http.StatusBadRequest, err.Error()) + return + } + + // Validate file name + if err := validation.ValidateFileName(fileName); err != nil { + s.writeErrorResponse(w, http.StatusBadRequest, err.Error()) return } + // Sanitize inputs + bucketName = validation.SanitizeBucketName(bucketName) + fileName = validation.SanitizeFileName(fileName) + if err := s.client.FileDelete(r.Context(), bucketName, fileName); err != nil { s.writeErrorResponse(w, http.StatusInternalServerError, "Failed to delete file") log.Printf("file delete error: %v", err) diff --git a/internal/handlers/router.go b/internal/handlers/router.go index 4b520f0..d794d30 100644 --- a/internal/handlers/router.go +++ b/internal/handlers/router.go @@ -6,6 +6,7 @@ import ( "io" "net/http" + "github.com/akave-ai/go-akavelink/internal/middleware" "github.com/gorilla/mux" sdksym "github.com/akave-ai/akavesdk/sdk" @@ -41,16 +42,23 @@ func NewRouter(client ClientAPI) http.Handler { r := mux.NewRouter().StrictSlash(true) s := &Server{client: client} + // Apply global middleware + r.Use(middleware.SecurityHeaders) + r.Use(middleware.LogRequest) + + // Health check (no validation needed) r.HandleFunc("/health", s.healthHandler).Methods("GET") + // Buckets r.HandleFunc("/buckets/{bucketName}", s.createBucketHandler).Methods("POST") r.HandleFunc("/buckets/{bucketName}", s.deleteBucketHandler).Methods("DELETE") - // Normalized route without trailing slash; keep both for compatibility r.HandleFunc("/buckets", s.viewBucketHandler).Methods("GET") - // r.HandleFunc("/buckets/", viewBucketHandler).Methods("GET") - // Files + + // Files - List and Upload (bucket name only) r.HandleFunc("/buckets/{bucketName}/files", s.listFilesHandler).Methods("GET") r.HandleFunc("/buckets/{bucketName}/files", s.uploadHandler).Methods("POST") + + // Files - Operations requiring both bucket and file name r.HandleFunc("/buckets/{bucketName}/files/{fileName}", s.fileInfoHandler).Methods("GET") r.HandleFunc("/buckets/{bucketName}/files/{fileName}/download", s.downloadHandler).Methods("GET") r.HandleFunc("/buckets/{bucketName}/files/{fileName}", s.fileDeleteHandler).Methods("DELETE") diff --git a/internal/middleware/validation.go b/internal/middleware/validation.go new file mode 100644 index 0000000..cd1d4fb --- /dev/null +++ b/internal/middleware/validation.go @@ -0,0 +1,158 @@ +// Package middleware provides HTTP middleware for the AkaveLink API. +package middleware + +import ( + "encoding/json" + "log" + "net/http" + + "github.com/akave-ai/go-akavelink/internal/validation" + "github.com/gorilla/mux" +) + +// ErrorResponse represents an error response. +type ErrorResponse struct { + Error string `json:"error"` + Message string `json:"message,omitempty"` + Field string `json:"field,omitempty"` +} + +// writeValidationError writes a validation error response. +func writeValidationError(w http.ResponseWriter, err error) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusBadRequest) + + response := ErrorResponse{ + Error: "Validation Error", + } + + if validationErr, ok := err.(*validation.ValidationError); ok { + response.Field = validationErr.Field + response.Message = validationErr.Message + } else { + response.Message = err.Error() + } + + if encodeErr := json.NewEncoder(w).Encode(response); encodeErr != nil { + log.Printf("Error encoding validation error response: %v", encodeErr) + } +} + +// ValidateBucketName is a middleware that validates bucket name from URL parameters. +func ValidateBucketName(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + bucketName := vars["bucketName"] + + if err := validation.ValidateBucketName(bucketName); err != nil { + writeValidationError(w, err) + return + } + + // Sanitize the bucket name for extra safety + sanitized := validation.SanitizeBucketName(bucketName) + if sanitized != bucketName { + log.Printf("Warning: Bucket name was sanitized from '%s' to '%s'", bucketName, sanitized) + } + + next.ServeHTTP(w, r) + }) +} + +// ValidateFileName is a middleware that validates file name from URL parameters. +func ValidateFileName(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + fileName := vars["fileName"] + + if err := validation.ValidateFileName(fileName); err != nil { + writeValidationError(w, err) + return + } + + // Sanitize the file name for extra safety + sanitized := validation.SanitizeFileName(fileName) + if sanitized != fileName { + log.Printf("Warning: File name was sanitized from '%s' to '%s'", fileName, sanitized) + } + + next.ServeHTTP(w, r) + }) +} + +// ValidateBucketAndFileName is a middleware that validates both bucket and file names. +func ValidateBucketAndFileName(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + bucketName := vars["bucketName"] + fileName := vars["fileName"] + + // Validate bucket name + if err := validation.ValidateBucketName(bucketName); err != nil { + writeValidationError(w, err) + return + } + + // Validate file name + if err := validation.ValidateFileName(fileName); err != nil { + writeValidationError(w, err) + return + } + + next.ServeHTTP(w, r) + }) +} + +// ValidateContentLength is a middleware that validates Content-Length header. +func ValidateContentLength(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + contentLength := r.ContentLength + + if err := validation.ValidateContentLength(contentLength); err != nil { + writeValidationError(w, err) + return + } + + next.ServeHTTP(w, r) + }) +} + +// RateLimitByIP is a placeholder for rate limiting middleware. +// This can be implemented with a proper rate limiting library like golang.org/x/time/rate +func RateLimitByIP(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // TODO: Implement rate limiting based on IP address + // For now, just pass through + next.ServeHTTP(w, r) + }) +} + +// SecurityHeaders adds security-related HTTP headers. +func SecurityHeaders(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Prevent MIME type sniffing + w.Header().Set("X-Content-Type-Options", "nosniff") + + // Prevent clickjacking + w.Header().Set("X-Frame-Options", "DENY") + + // Enable XSS protection + w.Header().Set("X-XSS-Protection", "1; mode=block") + + // Enforce HTTPS (if applicable) + // w.Header().Set("Strict-Transport-Security", "max-age=31536000; includeSubDomains") + + // Content Security Policy + w.Header().Set("Content-Security-Policy", "default-src 'self'") + + next.ServeHTTP(w, r) + }) +} + +// LogRequest logs incoming HTTP requests. +func LogRequest(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + log.Printf("%s %s %s", r.Method, r.RequestURI, r.RemoteAddr) + next.ServeHTTP(w, r) + }) +} diff --git a/internal/validation/validator.go b/internal/validation/validator.go new file mode 100644 index 0000000..60b0f7c --- /dev/null +++ b/internal/validation/validator.go @@ -0,0 +1,284 @@ +// Package validation provides input validation and sanitization for the AkaveLink API. +package validation + +import ( + "fmt" + "mime/multipart" + "path/filepath" + "regexp" + "strings" +) + +const ( + // Bucket name constraints + MinBucketNameLength = 1 + MaxBucketNameLength = 63 + + // File name constraints + MaxFileNameLength = 255 + MaxFileSize = 100 * 1024 * 1024 // 100 MB + + // Allowed MIME types for uploads + AllowedMIMETypes = "application/octet-stream|text/plain|text/csv|application/json|application/pdf|image/jpeg|image/png|image/gif|image/webp|video/mp4|audio/mpeg" +) + +var ( + // bucketNameRegex validates bucket names: alphanumeric, hyphens, underscores only + bucketNameRegex = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_-]*[a-zA-Z0-9]$`) + + // fileNameRegex validates file names: alphanumeric, hyphens, underscores, dots only + fileNameRegex = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]*[a-zA-Z0-9]$`) + + // pathTraversalPatterns detects path traversal attempts + pathTraversalPatterns = []*regexp.Regexp{ + regexp.MustCompile(`\.\.`), // Parent directory reference + regexp.MustCompile(`^/`), // Absolute path + regexp.MustCompile(`\\`), // Windows path separator + regexp.MustCompile(`%2e%2e`), // URL encoded .. + regexp.MustCompile(`%2f`), // URL encoded / + regexp.MustCompile(`%5c`), // URL encoded \ + regexp.MustCompile(`\x00`), // Null byte + regexp.MustCompile(`[<>:"|?*\x00-\x1f]`), // Invalid filename characters + } + + // allowedMIMETypesMap for quick lookup + allowedMIMETypesMap = buildMIMETypeMap() +) + +// ValidationError represents a validation error with details. +type ValidationError struct { + Field string + Message string +} + +// Error implements the error interface. +func (e *ValidationError) Error() string { + return fmt.Sprintf("%s: %s", e.Field, e.Message) +} + +// buildMIMETypeMap creates a map of allowed MIME types. +func buildMIMETypeMap() map[string]bool { + types := strings.Split(AllowedMIMETypes, "|") + m := make(map[string]bool, len(types)) + for _, t := range types { + m[t] = true + } + return m +} + +// ValidateBucketName validates bucket name according to rules: +// - Length between 1-63 characters +// - Alphanumeric characters, hyphens, and underscores only +// - Must start and end with alphanumeric character +// - No path traversal patterns +func ValidateBucketName(name string) error { + if name == "" { + return &ValidationError{Field: "bucketName", Message: "bucket name is required"} + } + + if len(name) < MinBucketNameLength || len(name) > MaxBucketNameLength { + return &ValidationError{ + Field: "bucketName", + Message: fmt.Sprintf("bucket name must be between %d and %d characters", MinBucketNameLength, MaxBucketNameLength), + } + } + + // Check for path traversal patterns first + if containsPathTraversal(name) { + return &ValidationError{ + Field: "bucketName", + Message: "bucket name contains invalid characters or patterns", + } + } + + // For single character names, just check if alphanumeric + if len(name) == 1 { + if !((name[0] >= 'a' && name[0] <= 'z') || (name[0] >= 'A' && name[0] <= 'Z') || (name[0] >= '0' && name[0] <= '9')) { + return &ValidationError{ + Field: "bucketName", + Message: "bucket name must contain only alphanumeric characters, hyphens, and underscores, and must start and end with an alphanumeric character", + } + } + return nil + } + + if !bucketNameRegex.MatchString(name) { + return &ValidationError{ + Field: "bucketName", + Message: "bucket name must contain only alphanumeric characters, hyphens, and underscores, and must start and end with an alphanumeric character", + } + } + + return nil +} + +// ValidateFileName validates file name according to rules: +// - Not empty +// - Length <= 255 characters +// - No path traversal patterns +// - Valid characters only +func ValidateFileName(name string) error { + if name == "" { + return &ValidationError{Field: "fileName", Message: "file name is required"} + } + + if len(name) > MaxFileNameLength { + return &ValidationError{ + Field: "fileName", + Message: fmt.Sprintf("file name must not exceed %d characters", MaxFileNameLength), + } + } + + // Check for path traversal patterns + if containsPathTraversal(name) { + return &ValidationError{ + Field: "fileName", + Message: "file name contains invalid characters or path traversal patterns", + } + } + + // Validate file name format + if !fileNameRegex.MatchString(name) { + return &ValidationError{ + Field: "fileName", + Message: "file name must contain only alphanumeric characters, dots, hyphens, and underscores, and must start and end with an alphanumeric character", + } + } + + return nil +} + +// ValidateFileUpload validates file upload including size and MIME type. +func ValidateFileUpload(header *multipart.FileHeader) error { + if header == nil { + return &ValidationError{Field: "file", Message: "file is required"} + } + + // Validate file name + if err := ValidateFileName(header.Filename); err != nil { + return err + } + + // Validate file size + if header.Size > MaxFileSize { + return &ValidationError{ + Field: "file", + Message: fmt.Sprintf("file size exceeds maximum allowed size of %d bytes", MaxFileSize), + } + } + + if header.Size == 0 { + return &ValidationError{Field: "file", Message: "file is empty"} + } + + // Validate MIME type + contentType := header.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" // Default + } + + // Extract base MIME type (remove charset and other parameters) + baseMIMEType := strings.Split(contentType, ";")[0] + baseMIMEType = strings.TrimSpace(baseMIMEType) + + if !allowedMIMETypesMap[baseMIMEType] { + return &ValidationError{ + Field: "file", + Message: fmt.Sprintf("file type '%s' is not allowed", baseMIMEType), + } + } + + return nil +} + +// SanitizeBucketName sanitizes bucket name by removing invalid characters. +// This should be used after validation for extra safety. +func SanitizeBucketName(name string) string { + // Remove any whitespace + name = strings.TrimSpace(name) + + // Convert to lowercase for consistency + name = strings.ToLower(name) + + // Remove any characters that aren't alphanumeric, hyphen, or underscore + var sanitized strings.Builder + for _, r := range name { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '-' || r == '_' { + sanitized.WriteRune(r) + } + } + + result := sanitized.String() + + // Ensure it doesn't exceed max length + if len(result) > MaxBucketNameLength { + result = result[:MaxBucketNameLength] + } + + return result +} + +// SanitizeFileName sanitizes file name by removing path components and invalid characters. +func SanitizeFileName(name string) string { + // Extract just the filename (remove any path components) + name = filepath.Base(name) + + // Remove any null bytes + name = strings.ReplaceAll(name, "\x00", "") + + // Remove leading/trailing whitespace + name = strings.TrimSpace(name) + + // Remove any path traversal patterns + name = strings.ReplaceAll(name, "..", "") + name = strings.ReplaceAll(name, "/", "") + name = strings.ReplaceAll(name, "\\", "") + + // Ensure it doesn't exceed max length + if len(name) > MaxFileNameLength { + // Try to preserve the extension + ext := filepath.Ext(name) + if len(ext) > 0 && len(ext) < 10 { + baseName := name[:len(name)-len(ext)] + maxBaseLength := MaxFileNameLength - len(ext) + if len(baseName) > maxBaseLength { + baseName = baseName[:maxBaseLength] + } + name = baseName + ext + } else { + name = name[:MaxFileNameLength] + } + } + + return name +} + +// containsPathTraversal checks if a string contains path traversal patterns. +func containsPathTraversal(s string) bool { + // Convert to lowercase for case-insensitive matching + lower := strings.ToLower(s) + + for _, pattern := range pathTraversalPatterns { + if pattern.MatchString(lower) { + return true + } + } + + return false +} + +// ValidateContentLength validates the Content-Length header for uploads. +func ValidateContentLength(contentLength int64) error { + if contentLength < 0 { + return &ValidationError{Field: "Content-Length", Message: "invalid content length"} + } + + if contentLength > MaxFileSize { + return &ValidationError{ + Field: "Content-Length", + Message: fmt.Sprintf("content length exceeds maximum allowed size of %d bytes", MaxFileSize), + } + } + + return nil +} diff --git a/test/middleware_test.go b/test/middleware_test.go new file mode 100644 index 0000000..9196334 --- /dev/null +++ b/test/middleware_test.go @@ -0,0 +1,261 @@ +package test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/akave-ai/go-akavelink/internal/middleware" + "github.com/gorilla/mux" + "github.com/stretchr/testify/assert" +) + +func TestValidateBucketNameMiddleware(t *testing.T) { + tests := []struct { + name string + bucketName string + expectedStatus int + }{ + { + name: "valid bucket name", + bucketName: "my-bucket-123", + expectedStatus: http.StatusOK, + }, + { + name: "invalid bucket name with special chars", + bucketName: "my-bucket!@#", + expectedStatus: http.StatusBadRequest, + }, + { + name: "invalid bucket name starting with hyphen", + bucketName: "-bucket", + expectedStatus: http.StatusBadRequest, + }, + { + name: "bucket name too long", + bucketName: "a1234567890123456789012345678901234567890123456789012345678901234567890", + expectedStatus: http.StatusBadRequest, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create a test handler + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + // Wrap with middleware + r := mux.NewRouter() + r.Handle("/buckets/{bucketName}", middleware.ValidateBucketName(handler)) + + // Create test request + req := httptest.NewRequest("GET", "/buckets/"+tt.bucketName, nil) + rr := httptest.NewRecorder() + + // Serve the request + r.ServeHTTP(rr, req) + + // Check status code + assert.Equal(t, tt.expectedStatus, rr.Code) + }) + } +} + +func TestValidateFileNameMiddleware(t *testing.T) { + tests := []struct { + name string + fileName string + expectedStatus int + }{ + { + name: "valid file name", + fileName: "document.pdf", + expectedStatus: http.StatusOK, + }, + { + name: "invalid file name starting with dot", + fileName: ".hidden", + expectedStatus: http.StatusBadRequest, + }, + { + name: "invalid file name ending with dot", + fileName: "file.", + expectedStatus: http.StatusBadRequest, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create a test handler + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + // Wrap with middleware + r := mux.NewRouter() + r.Handle("/files/{fileName}", middleware.ValidateFileName(handler)) + + // Create test request + req := httptest.NewRequest("GET", "/files/"+tt.fileName, nil) + rr := httptest.NewRecorder() + + // Serve the request + r.ServeHTTP(rr, req) + + // Check status code + assert.Equal(t, tt.expectedStatus, rr.Code) + }) + } +} + +func TestValidateBucketAndFileNameMiddleware(t *testing.T) { + tests := []struct { + name string + bucketName string + fileName string + expectedStatus int + }{ + { + name: "valid bucket and file names", + bucketName: "my-bucket", + fileName: "document.pdf", + expectedStatus: http.StatusOK, + }, + { + name: "invalid bucket name", + bucketName: "-bucket", + fileName: "document.pdf", + expectedStatus: http.StatusBadRequest, + }, + { + name: "invalid file name", + bucketName: "my-bucket", + fileName: ".hidden", + expectedStatus: http.StatusBadRequest, + }, + { + name: "both invalid", + bucketName: "-bucket", + fileName: "file.", + expectedStatus: http.StatusBadRequest, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create a test handler + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + // Wrap with middleware + r := mux.NewRouter() + r.Handle("/buckets/{bucketName}/files/{fileName}", + middleware.ValidateBucketAndFileName(handler)) + + // Create test request + req := httptest.NewRequest("GET", + "/buckets/"+tt.bucketName+"/files/"+tt.fileName, nil) + rr := httptest.NewRecorder() + + // Serve the request + r.ServeHTTP(rr, req) + + // Check status code + assert.Equal(t, tt.expectedStatus, rr.Code) + }) + } +} + +func TestValidateContentLengthMiddleware(t *testing.T) { + tests := []struct { + name string + contentLength string + expectedStatus int + }{ + { + name: "valid content length", + contentLength: "1024", + expectedStatus: http.StatusOK, + }, + { + name: "content length at max", + contentLength: "104857600", // 100 MB + expectedStatus: http.StatusOK, + }, + { + name: "zero content length", + contentLength: "0", + expectedStatus: http.StatusOK, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create a test handler + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + // Wrap with middleware + wrappedHandler := middleware.ValidateContentLength(handler) + + // Create test request + req := httptest.NewRequest("POST", "/upload", nil) + if tt.contentLength != "" { + req.Header.Set("Content-Length", tt.contentLength) + } + rr := httptest.NewRecorder() + + // Serve the request + wrappedHandler.ServeHTTP(rr, req) + + // Check status code + assert.Equal(t, tt.expectedStatus, rr.Code) + }) + } +} + +func TestSecurityHeadersMiddleware(t *testing.T) { + // Create a test handler + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + // Wrap with middleware + wrappedHandler := middleware.SecurityHeaders(handler) + + // Create test request + req := httptest.NewRequest("GET", "/", nil) + rr := httptest.NewRecorder() + + // Serve the request + wrappedHandler.ServeHTTP(rr, req) + + // Check security headers + assert.Equal(t, "nosniff", rr.Header().Get("X-Content-Type-Options")) + assert.Equal(t, "DENY", rr.Header().Get("X-Frame-Options")) + assert.Equal(t, "1; mode=block", rr.Header().Get("X-XSS-Protection")) + assert.Equal(t, "default-src 'self'", rr.Header().Get("Content-Security-Policy")) +} + +func TestLogRequestMiddleware(t *testing.T) { + // Create a test handler + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + // Wrap with middleware + wrappedHandler := middleware.LogRequest(handler) + + // Create test request + req := httptest.NewRequest("GET", "/test", nil) + rr := httptest.NewRecorder() + + // Serve the request (should not panic or error) + wrappedHandler.ServeHTTP(rr, req) + + // Check that handler was called + assert.Equal(t, http.StatusOK, rr.Code) +} diff --git a/test/validation_integration_test.go b/test/validation_integration_test.go new file mode 100644 index 0000000..99050b2 --- /dev/null +++ b/test/validation_integration_test.go @@ -0,0 +1,402 @@ +package test + +import ( + "bytes" + "context" + "io" + "mime/multipart" + "net/http" + "net/http/httptest" + "testing" + + "github.com/akave-ai/go-akavelink/internal/handlers" + sdksym "github.com/akave-ai/akavesdk/sdk" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +// MockClient is a mock implementation of handlers.ClientAPI +type MockClient struct { + mock.Mock +} + +func (m *MockClient) CreateBucket(ctx context.Context, name string) error { + args := m.Called(ctx, name) + return args.Error(0) +} + +func (m *MockClient) DeleteBucket(ctx context.Context, name string) error { + args := m.Called(ctx, name) + return args.Error(0) +} + +func (m *MockClient) ListBuckets() ([]string, error) { + args := m.Called() + return args.Get(0).([]string), args.Error(1) +} + +func (m *MockClient) ListFiles(ctx context.Context, bucket string) ([]sdksym.IPCFileListItem, error) { + args := m.Called(ctx, bucket) + return args.Get(0).([]sdksym.IPCFileListItem), args.Error(1) +} + +func (m *MockClient) CreateFileUpload(ctx context.Context, bucket, fileName string) (*sdksym.IPCFileUpload, error) { + args := m.Called(ctx, bucket, fileName) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).(*sdksym.IPCFileUpload), args.Error(1) +} + +func (m *MockClient) Upload(ctx context.Context, up *sdksym.IPCFileUpload, r io.Reader) (sdksym.IPCFileMetaV2, error) { + args := m.Called(ctx, up, r) + return args.Get(0).(sdksym.IPCFileMetaV2), args.Error(1) +} + +func (m *MockClient) CreateFileDownload(ctx context.Context, bucket, fileName string) (sdksym.IPCFileDownload, error) { + args := m.Called(ctx, bucket, fileName) + return args.Get(0).(sdksym.IPCFileDownload), args.Error(1) +} + +func (m *MockClient) Download(ctx context.Context, dl sdksym.IPCFileDownload, w io.Writer) error { + args := m.Called(ctx, dl, w) + return args.Error(0) +} + +func (m *MockClient) FileInfo(ctx context.Context, bucket, fileName string) (sdksym.IPCFileMeta, error) { + args := m.Called(ctx, bucket, fileName) + return args.Get(0).(sdksym.IPCFileMeta), args.Error(1) +} + +func (m *MockClient) FileDelete(ctx context.Context, bucket, fileName string) error { + args := m.Called(ctx, bucket, fileName) + return args.Error(0) +} + +func (m *MockClient) NewIPC() (*sdksym.IPC, error) { + args := m.Called() + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).(*sdksym.IPC), args.Error(1) +} + +func TestCreateBucketValidation(t *testing.T) { + tests := []struct { + name string + bucketName string + expectedStatus int + }{ + { + name: "valid bucket name", + bucketName: "valid-bucket-123", + expectedStatus: http.StatusCreated, + }, + { + name: "invalid bucket name with special chars", + bucketName: "invalid!bucket", + expectedStatus: http.StatusBadRequest, + }, + { + name: "invalid bucket name starting with hyphen", + bucketName: "-bucket", + expectedStatus: http.StatusBadRequest, + }, + { + name: "bucket name too long", + bucketName: "a1234567890123456789012345678901234567890123456789012345678901234567890", + expectedStatus: http.StatusBadRequest, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockClient := new(MockClient) + if tt.expectedStatus == http.StatusCreated { + mockClient.On("CreateBucket", mock.Anything, mock.Anything).Return(nil) + } + + router := handlers.NewRouter(mockClient) + + req := httptest.NewRequest("POST", "/buckets/"+tt.bucketName, nil) + rr := httptest.NewRecorder() + + router.ServeHTTP(rr, req) + + assert.Equal(t, tt.expectedStatus, rr.Code) + }) + } +} + +func TestDeleteBucketValidation(t *testing.T) { + tests := []struct { + name string + bucketName string + expectedStatus int + }{ + { + name: "valid bucket name", + bucketName: "valid-bucket", + expectedStatus: http.StatusOK, + }, + { + name: "invalid bucket name", + bucketName: "-bucket", + expectedStatus: http.StatusBadRequest, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockClient := new(MockClient) + if tt.expectedStatus == http.StatusOK { + mockClient.On("ListFiles", mock.Anything, mock.Anything). + Return([]sdksym.IPCFileListItem{}, nil) + mockClient.On("DeleteBucket", mock.Anything, mock.Anything).Return(nil) + } + + router := handlers.NewRouter(mockClient) + + req := httptest.NewRequest("DELETE", "/buckets/"+tt.bucketName, nil) + rr := httptest.NewRecorder() + + router.ServeHTTP(rr, req) + + assert.Equal(t, tt.expectedStatus, rr.Code) + }) + } +} + +func TestFileInfoValidation(t *testing.T) { + tests := []struct { + name string + bucketName string + fileName string + expectedStatus int + }{ + { + name: "valid bucket and file names", + bucketName: "my-bucket", + fileName: "document.pdf", + expectedStatus: http.StatusOK, + }, + { + name: "invalid bucket name", + bucketName: "-bucket", + fileName: "document.pdf", + expectedStatus: http.StatusBadRequest, + }, + { + name: "invalid file name starting with dot", + bucketName: "my-bucket", + fileName: ".hidden", + expectedStatus: http.StatusBadRequest, + }, + { + name: "invalid file name ending with dot", + bucketName: "my-bucket", + fileName: "file.", + expectedStatus: http.StatusBadRequest, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockClient := new(MockClient) + if tt.expectedStatus == http.StatusOK { + mockClient.On("FileInfo", mock.Anything, mock.Anything, mock.Anything). + Return(sdksym.IPCFileMeta{}, nil) + } + + router := handlers.NewRouter(mockClient) + + req := httptest.NewRequest("GET", + "/buckets/"+tt.bucketName+"/files/"+tt.fileName, nil) + rr := httptest.NewRecorder() + + router.ServeHTTP(rr, req) + + assert.Equal(t, tt.expectedStatus, rr.Code) + }) + } +} + +func TestFileUploadValidation(t *testing.T) { + tests := []struct { + name string + bucketName string + fileName string + fileSize int64 + contentType string + expectedStatus int + }{ + { + name: "valid file upload", + bucketName: "my-bucket", + fileName: "document.pdf", + fileSize: 1024, + contentType: "application/pdf", + expectedStatus: http.StatusCreated, + }, + { + name: "invalid bucket name", + bucketName: "-bucket", + fileName: "document.pdf", + fileSize: 1024, + contentType: "application/pdf", + expectedStatus: http.StatusBadRequest, + }, + { + name: "file too large", + bucketName: "my-bucket", + fileName: "large.bin", + fileSize: 101 * 1024 * 1024, // 101 MB + contentType: "application/octet-stream", + expectedStatus: http.StatusBadRequest, + }, + { + name: "invalid file name", + bucketName: "my-bucket", + fileName: ".hidden", + fileSize: 1024, + contentType: "text/plain", + expectedStatus: http.StatusBadRequest, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockClient := new(MockClient) + if tt.expectedStatus == http.StatusCreated { + mockClient.On("CreateFileUpload", mock.Anything, mock.Anything, mock.Anything). + Return(&sdksym.IPCFileUpload{}, nil) + mockClient.On("Upload", mock.Anything, mock.Anything, mock.Anything). + Return(sdksym.IPCFileMetaV2{}, nil) + } + + router := handlers.NewRouter(mockClient) + + // Create multipart form + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + + // Create file part + part, err := writer.CreateFormFile("file", tt.fileName) + assert.NoError(t, err) + + // Write file content + fileContent := make([]byte, tt.fileSize) + _, err = part.Write(fileContent) + assert.NoError(t, err) + + err = writer.Close() + assert.NoError(t, err) + + req := httptest.NewRequest("POST", "/buckets/"+tt.bucketName+"/files", body) + req.Header.Set("Content-Type", writer.FormDataContentType()) + rr := httptest.NewRecorder() + + router.ServeHTTP(rr, req) + + assert.Equal(t, tt.expectedStatus, rr.Code) + }) + } +} + +func TestFileDownloadValidation(t *testing.T) { + tests := []struct { + name string + bucketName string + fileName string + expectedStatus int + }{ + { + name: "valid download", + bucketName: "my-bucket", + fileName: "document.pdf", + expectedStatus: http.StatusOK, + }, + { + name: "invalid bucket name", + bucketName: "-bucket", + fileName: "document.pdf", + expectedStatus: http.StatusBadRequest, + }, + { + name: "invalid file name", + bucketName: "my-bucket", + fileName: ".hidden", + expectedStatus: http.StatusBadRequest, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockClient := new(MockClient) + if tt.expectedStatus == http.StatusOK { + mockClient.On("CreateFileDownload", mock.Anything, mock.Anything, mock.Anything). + Return(sdksym.IPCFileDownload{}, nil) + mockClient.On("Download", mock.Anything, mock.Anything, mock.Anything). + Return(nil) + } + + router := handlers.NewRouter(mockClient) + + req := httptest.NewRequest("GET", + "/buckets/"+tt.bucketName+"/files/"+tt.fileName+"/download", nil) + rr := httptest.NewRecorder() + + router.ServeHTTP(rr, req) + + assert.Equal(t, tt.expectedStatus, rr.Code) + }) + } +} + +func TestFileDeleteValidation(t *testing.T) { + tests := []struct { + name string + bucketName string + fileName string + expectedStatus int + }{ + { + name: "valid delete", + bucketName: "my-bucket", + fileName: "document.pdf", + expectedStatus: http.StatusOK, + }, + { + name: "invalid bucket name", + bucketName: "-bucket", + fileName: "document.pdf", + expectedStatus: http.StatusBadRequest, + }, + { + name: "invalid file name", + bucketName: "my-bucket", + fileName: "file.", + expectedStatus: http.StatusBadRequest, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockClient := new(MockClient) + if tt.expectedStatus == http.StatusOK { + mockClient.On("FileDelete", mock.Anything, mock.Anything, mock.Anything). + Return(nil) + } + + router := handlers.NewRouter(mockClient) + + req := httptest.NewRequest("DELETE", + "/buckets/"+tt.bucketName+"/files/"+tt.fileName, nil) + rr := httptest.NewRecorder() + + router.ServeHTTP(rr, req) + + assert.Equal(t, tt.expectedStatus, rr.Code) + }) + } +} diff --git a/test/validation_test.go b/test/validation_test.go new file mode 100644 index 0000000..95ecf0d --- /dev/null +++ b/test/validation_test.go @@ -0,0 +1,465 @@ +package test + +import ( + "bytes" + "mime/multipart" + "net/textproto" + "testing" + + "github.com/akave-ai/go-akavelink/internal/validation" + "github.com/stretchr/testify/assert" +) + +func TestValidateBucketName(t *testing.T) { + tests := []struct { + name string + input string + wantError bool + errorMsg string + }{ + { + name: "valid bucket name", + input: "my-bucket-123", + wantError: false, + }, + { + name: "valid bucket name with underscores", + input: "my_bucket_123", + wantError: false, + }, + { + name: "empty bucket name", + input: "", + wantError: true, + errorMsg: "bucket name is required", + }, + { + name: "bucket name too long", + input: "a123456789012345678901234567890123456789012345678901234567890123456789", + wantError: true, + errorMsg: "bucket name must be between", + }, + { + name: "bucket name with spaces", + input: "my bucket", + wantError: true, + errorMsg: "bucket name must contain only alphanumeric", + }, + { + name: "bucket name with special characters", + input: "my-bucket!@#", + wantError: true, + errorMsg: "bucket name must contain only alphanumeric", + }, + { + name: "bucket name with path traversal", + input: "../etc/passwd", + wantError: true, + errorMsg: "bucket name contains invalid characters", + }, + { + name: "bucket name with forward slash", + input: "my/bucket", + wantError: true, + errorMsg: "bucket name must contain only alphanumeric", + }, + { + name: "bucket name with backslash", + input: "my\\bucket", + wantError: true, + errorMsg: "bucket name contains invalid characters", + }, + { + name: "bucket name starting with hyphen", + input: "-mybucket", + wantError: true, + errorMsg: "bucket name must contain only alphanumeric", + }, + { + name: "bucket name ending with hyphen", + input: "mybucket-", + wantError: true, + errorMsg: "bucket name must contain only alphanumeric", + }, + { + name: "single character bucket name", + input: "a", + wantError: false, + }, + { + name: "max length bucket name", + input: "a12345678901234567890123456789012345678901234567890123456789012", + wantError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validation.ValidateBucketName(tt.input) + if tt.wantError { + assert.Error(t, err) + if tt.errorMsg != "" { + assert.Contains(t, err.Error(), tt.errorMsg) + } + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestValidateFileName(t *testing.T) { + tests := []struct { + name string + input string + wantError bool + errorMsg string + }{ + { + name: "valid file name", + input: "document.pdf", + wantError: false, + }, + { + name: "valid file name with multiple dots", + input: "my.file.name.txt", + wantError: false, + }, + { + name: "empty file name", + input: "", + wantError: true, + errorMsg: "file name is required", + }, + { + name: "file name too long", + input: string(make([]byte, 256)), + wantError: true, + errorMsg: "file name must not exceed", + }, + { + name: "file name with path traversal", + input: "../../../etc/passwd", + wantError: true, + errorMsg: "file name contains invalid characters", + }, + { + name: "file name with absolute path", + input: "/etc/passwd", + wantError: true, + errorMsg: "file name contains invalid characters", + }, + { + name: "file name with null byte", + input: "file\x00name.txt", + wantError: true, + errorMsg: "file name contains invalid characters", + }, + { + name: "file name with special characters", + input: "file<>name.txt", + wantError: true, + errorMsg: "file name contains invalid characters", + }, + { + name: "file name with backslash", + input: "path\\file.txt", + wantError: true, + errorMsg: "file name contains invalid characters", + }, + { + name: "file name starting with dot", + input: ".hidden", + wantError: true, + errorMsg: "file name must contain only alphanumeric", + }, + { + name: "file name ending with dot", + input: "file.", + wantError: true, + errorMsg: "file name must contain only alphanumeric", + }, + { + name: "valid file name with hyphens and underscores", + input: "my-file_name-123.txt", + wantError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validation.ValidateFileName(tt.input) + if tt.wantError { + assert.Error(t, err) + if tt.errorMsg != "" { + assert.Contains(t, err.Error(), tt.errorMsg) + } + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestValidateFileUpload(t *testing.T) { + tests := []struct { + name string + filename string + size int64 + contentType string + wantError bool + errorMsg string + }{ + { + name: "valid file upload", + filename: "document.pdf", + size: 1024, + contentType: "application/pdf", + wantError: false, + }, + { + name: "valid text file", + filename: "data.txt", + size: 512, + contentType: "text/plain", + wantError: false, + }, + { + name: "valid image file", + filename: "photo.jpg", + size: 2048, + contentType: "image/jpeg", + wantError: false, + }, + { + name: "file too large", + filename: "large.bin", + size: 101 * 1024 * 1024, // 101 MB + contentType: "application/octet-stream", + wantError: true, + errorMsg: "file size exceeds maximum", + }, + { + name: "empty file", + filename: "empty.txt", + size: 0, + contentType: "text/plain", + wantError: true, + errorMsg: "file is empty", + }, + { + name: "invalid MIME type", + filename: "script.exe", + size: 1024, + contentType: "application/x-msdownload", + wantError: true, + errorMsg: "file type", + }, + { + name: "invalid filename with path traversal", + filename: "../../../etc/passwd", + size: 1024, + contentType: "text/plain", + wantError: true, + errorMsg: "file name contains invalid characters", + }, + { + name: "default content type", + filename: "file.bin", + size: 1024, + contentType: "", + wantError: false, + }, + { + name: "content type with charset", + filename: "data.json", + size: 1024, + contentType: "application/json; charset=utf-8", + wantError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create a multipart file header + header := &multipart.FileHeader{ + Filename: tt.filename, + Size: tt.size, + } + + // Set content type in header + if tt.contentType != "" { + header.Header = make(textproto.MIMEHeader) + header.Header.Set("Content-Type", tt.contentType) + } + + err := validation.ValidateFileUpload(header) + if tt.wantError { + assert.Error(t, err) + if tt.errorMsg != "" { + assert.Contains(t, err.Error(), tt.errorMsg) + } + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestSanitizeBucketName(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "already clean", + input: "mybucket", + expected: "mybucket", + }, + { + name: "uppercase to lowercase", + input: "MyBucket", + expected: "mybucket", + }, + { + name: "with whitespace", + input: " my-bucket ", + expected: "my-bucket", + }, + { + name: "remove special characters", + input: "my!@#bucket", + expected: "mybucket", + }, + { + name: "preserve hyphens and underscores", + input: "my-bucket_123", + expected: "my-bucket_123", + }, + { + name: "truncate long name", + input: "a1234567890123456789012345678901234567890123456789012345678901234567890", + expected: "a12345678901234567890123456789012345678901234567890123456789012", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := validation.SanitizeBucketName(tt.input) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestSanitizeFileName(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "already clean", + input: "document.pdf", + expected: "document.pdf", + }, + { + name: "remove path components", + input: "/path/to/file.txt", + expected: "file.txt", + }, + { + name: "remove path traversal", + input: "../../../etc/passwd", + expected: "passwd", // filepath.Base extracts the last element + }, + { + name: "remove null bytes", + input: "file\x00name.txt", + expected: "filename.txt", + }, + { + name: "remove backslashes", + input: "path\\to\\file.txt", + expected: "pathtofile.txt", + }, + { + name: "trim whitespace", + input: " file.txt ", + expected: "file.txt", + }, + { + name: "truncate long filename preserving extension", + input: string(bytes.Repeat([]byte("a"), 260)) + ".txt", + expected: string(bytes.Repeat([]byte("a"), 251)) + ".txt", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := validation.SanitizeFileName(tt.input) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestValidateContentLength(t *testing.T) { + tests := []struct { + name string + length int64 + wantError bool + errorMsg string + }{ + { + name: "valid content length", + length: 1024, + wantError: false, + }, + { + name: "zero content length", + length: 0, + wantError: false, + }, + { + name: "negative content length", + length: -1, + wantError: true, + errorMsg: "invalid content length", + }, + { + name: "content length exceeds maximum", + length: 101 * 1024 * 1024, + wantError: true, + errorMsg: "content length exceeds maximum", + }, + { + name: "maximum allowed content length", + length: 100 * 1024 * 1024, + wantError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validation.ValidateContentLength(tt.length) + if tt.wantError { + assert.Error(t, err) + if tt.errorMsg != "" { + assert.Contains(t, err.Error(), tt.errorMsg) + } + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestValidationError(t *testing.T) { + err := &validation.ValidationError{ + Field: "testField", + Message: "test message", + } + + expected := "testField: test message" + assert.Equal(t, expected, err.Error()) +} From 5216ea6b43122c7bbb603be6bb9ae0661691e9a9 Mon Sep 17 00:00:00 2001 From: manny-uncharted Date: Thu, 23 Oct 2025 12:04:14 +0100 Subject: [PATCH 4/5] fix: removed unnecessary validation --- docs/VALIDATION.md | 468 --------------------------------------------- 1 file changed, 468 deletions(-) delete mode 100644 docs/VALIDATION.md diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md deleted file mode 100644 index b1e20e2..0000000 --- a/docs/VALIDATION.md +++ /dev/null @@ -1,468 +0,0 @@ -# Input Validation & Security - -This document describes the comprehensive input validation and sanitization system implemented in go-akavelink. - -## Overview - -The API implements multiple layers of security to protect against common attacks: - -1. **Input Validation** - Validates all user inputs against strict rules -2. **Input Sanitization** - Cleans and normalizes inputs for safe processing -3. **Security Headers** - Adds HTTP security headers to all responses -4. **Request Logging** - Logs all incoming requests for audit trails - -## Validation Rules - -### Bucket Names - -Bucket names must adhere to the following rules: - -- **Length**: Between 1 and 63 characters -- **Allowed Characters**: - - Lowercase letters (a-z) - - Uppercase letters (A-Z) - - Numbers (0-9) - - Hyphens (-) - - Underscores (_) -- **Format Requirements**: - - Must start with an alphanumeric character - - Must end with an alphanumeric character - - Cannot contain consecutive dots (..) - - Cannot contain path separators (/ or \) - - Cannot contain special characters (!@#$%^&*()+=[]{}|;:'",<>?) - -**Valid Examples:** -``` -my-bucket -data_store_123 -bucket1 -MyBucket-2024 -user_data_v2 -``` - -**Invalid Examples:** -``` -../etc/passwd # Path traversal -my bucket # Contains space -bucket!@# # Special characters --mybucket # Starts with hyphen -mybucket- # Ends with hyphen -a1234567890...123 # Too long (>63 chars) -``` - -### File Names - -File names must adhere to the following rules: - -- **Length**: Maximum 255 characters -- **Allowed Characters**: - - Lowercase letters (a-z) - - Uppercase letters (A-Z) - - Numbers (0-9) - - Dots (.) - - Hyphens (-) - - Underscores (_) -- **Format Requirements**: - - Must start with an alphanumeric character - - Must end with an alphanumeric character - - Cannot contain path traversal patterns (..) - - Cannot contain path separators (/ or \) - - Cannot contain null bytes (\x00) - - Cannot contain invalid filename characters (<>:"|?*) - -**Valid Examples:** -``` -document.pdf -my-file_v2.txt -data.backup.tar.gz -image123.jpg -report-2024-01-15.csv -``` - -**Invalid Examples:** -``` -../../../etc/passwd # Path traversal -/etc/passwd # Absolute path -file<>name.txt # Invalid characters -file\x00name.txt # Null byte -.hidden # Starts with dot -file. # Ends with dot -path/to/file.txt # Contains path separator -``` - -### File Uploads - -File uploads are validated for size, type, and content: - -#### Size Limits - -- **Maximum Size**: 100 MB (104,857,600 bytes) -- **Minimum Size**: Must not be empty (> 0 bytes) - -#### Allowed MIME Types - -The following MIME types are permitted: - -| Category | MIME Type | Description | -|----------|-----------|-------------| -| Binary | `application/octet-stream` | Generic binary data | -| Text | `text/plain` | Plain text files | -| Text | `text/csv` | CSV files | -| JSON | `application/json` | JSON data | -| PDF | `application/pdf` | PDF documents | -| Images | `image/jpeg` | JPEG images | -| Images | `image/png` | PNG images | -| Images | `image/gif` | GIF images | -| Images | `image/webp` | WebP images | -| Video | `video/mp4` | MP4 videos | -| Audio | `audio/mpeg` | MP3 audio | - -**Note**: MIME type validation extracts the base type (before semicolon) to handle charset and other parameters. - -#### Upload Validation Process - -1. **Content-Length Check**: Validates the Content-Length header before parsing -2. **Multipart Form Parsing**: Parses the multipart form data (max 32 MiB in memory) -3. **File Header Validation**: Validates the file header metadata -4. **File Name Validation**: Validates and sanitizes the filename -5. **Size Validation**: Ensures file size is within limits -6. **MIME Type Validation**: Verifies the Content-Type header - -## Sanitization - -In addition to validation, the API sanitizes inputs for extra safety: - -### Bucket Name Sanitization - -```go -// Removes whitespace -// Converts to lowercase -// Removes invalid characters -// Truncates to max length -sanitized := validation.SanitizeBucketName(input) -``` - -**Example:** -``` -Input: " My-Bucket!@# " -Output: "my-bucket" -``` - -### File Name Sanitization - -```go -// Extracts base filename (removes path) -// Removes null bytes -// Removes path traversal patterns -// Removes path separators -// Truncates to max length (preserving extension) -sanitized := validation.SanitizeFileName(input) -``` - -**Example:** -``` -Input: "/path/to/../file.txt" -Output: "file.txt" -``` - -## Security Features - -### Path Traversal Prevention - -The validation system detects and blocks various path traversal attempts: - -- Parent directory references (`..`) -- Absolute paths (`/` or `\`) -- URL-encoded traversal (`%2e%2e`, `%2f`, `%5c`) -- Null bytes (`\x00`) -- Invalid filename characters - -### Attack Patterns Detected - -The following malicious patterns are detected and rejected: - -``` -../etc/passwd -..\..\windows\system32 -%2e%2e%2f -file\x00.txt - -../../../../../../etc/shadow -``` - -### Security Headers - -All API responses include the following security headers: - -```http -X-Content-Type-Options: nosniff -X-Frame-Options: DENY -X-XSS-Protection: 1; mode=block -Content-Security-Policy: default-src 'self' -``` - -#### Header Descriptions - -- **X-Content-Type-Options**: Prevents browsers from MIME-sniffing responses -- **X-Frame-Options**: Prevents the page from being embedded in frames (clickjacking protection) -- **X-XSS-Protection**: Enables browser XSS filtering -- **Content-Security-Policy**: Restricts resource loading to same origin - -## Error Responses - -### Validation Errors - -When validation fails, the API returns HTTP 400 (Bad Request) with a structured error response: - -```json -{ - "error": "Validation Error", - "field": "bucketName", - "message": "bucket name must contain only alphanumeric characters, hyphens, and underscores, and must start and end with an alphanumeric character" -} -``` - -### Error Response Fields - -- **error**: Always "Validation Error" for validation failures -- **field**: The name of the field that failed validation (e.g., "bucketName", "fileName", "file") -- **message**: A human-readable description of the validation error - -### Common Validation Errors - -#### Bucket Name Errors - -```json -{ - "error": "Validation Error", - "field": "bucketName", - "message": "bucket name is required" -} -``` - -```json -{ - "error": "Validation Error", - "field": "bucketName", - "message": "bucket name must be between 1 and 63 characters" -} -``` - -```json -{ - "error": "Validation Error", - "field": "bucketName", - "message": "bucket name contains invalid characters or patterns" -} -``` - -#### File Name Errors - -```json -{ - "error": "Validation Error", - "field": "fileName", - "message": "file name is required" -} -``` - -```json -{ - "error": "Validation Error", - "field": "fileName", - "message": "file name contains invalid characters or path traversal patterns" -} -``` - -#### File Upload Errors - -```json -{ - "error": "Validation Error", - "field": "file", - "message": "file size exceeds maximum allowed size of 104857600 bytes" -} -``` - -```json -{ - "error": "Validation Error", - "field": "file", - "message": "file is empty" -} -``` - -```json -{ - "error": "Validation Error", - "field": "file", - "message": "file type 'application/x-msdownload' is not allowed" -} -``` - -## Implementation Details - -### Package Structure - -``` -internal/ -β”œβ”€β”€ validation/ -β”‚ └── validator.go # Core validation logic -β”œβ”€β”€ middleware/ -β”‚ └── validation.go # HTTP middleware -└── handlers/ - β”œβ”€β”€ buckets.go # Bucket handlers with validation - └── files.go # File handlers with validation -``` - -### Validation Package - -The `internal/validation` package provides: - -- `ValidateBucketName(name string) error` -- `ValidateFileName(name string) error` -- `ValidateFileUpload(header *multipart.FileHeader) error` -- `ValidateContentLength(contentLength int64) error` -- `SanitizeBucketName(name string) string` -- `SanitizeFileName(name string) string` - -### Middleware Package - -The `internal/middleware` package provides: - -- `ValidateBucketName(next http.Handler) http.Handler` -- `ValidateFileName(next http.Handler) http.Handler` -- `ValidateBucketAndFileName(next http.Handler) http.Handler` -- `ValidateContentLength(next http.Handler) http.Handler` -- `SecurityHeaders(next http.Handler) http.Handler` -- `LogRequest(next http.Handler) http.Handler` - -### Handler Integration - -All handlers validate and sanitize inputs: - -```go -// Validate bucket name -if err := validation.ValidateBucketName(bucketName); err != nil { - s.writeErrorResponse(w, http.StatusBadRequest, err.Error()) - return -} - -// Sanitize bucket name for extra safety -bucketName = validation.SanitizeBucketName(bucketName) -``` - -## Testing - -Comprehensive test coverage includes: - -### Unit Tests - -- `test/validation_test.go` - Tests all validation functions -- `test/middleware_test.go` - Tests all middleware functions - -### Integration Tests - -- `test/validation_integration_test.go` - Tests validation in HTTP handlers - -### Test Coverage - -The test suite covers: - -- Valid inputs (positive tests) -- Invalid inputs (negative tests) -- Edge cases (boundary conditions) -- Security attacks (path traversal, injection, etc.) -- Sanitization behavior -- Error message accuracy - -### Running Tests - -```bash -# Run all tests -go test ./test/... - -# Run validation tests only -go test ./test/ -run Validation - -# Run with coverage -go test ./test/... -cover - -# Run with verbose output -go test ./test/... -v -``` - -## Best Practices - -### For API Users - -1. **Always validate inputs client-side** before sending requests -2. **Use descriptive names** that follow the validation rules -3. **Handle validation errors gracefully** in your application -4. **Check file sizes** before attempting uploads -5. **Use appropriate MIME types** for your files - -### For Developers - -1. **Always validate before sanitizing** to catch malicious inputs -2. **Log validation failures** for security monitoring -3. **Never trust user input** - validate everything -4. **Keep validation rules consistent** across all endpoints -5. **Update tests** when adding new validation rules -6. **Document validation changes** in the changelog - -## Security Considerations - -### Defense in Depth - -The validation system implements multiple layers of security: - -1. **Input Validation** - First line of defense -2. **Input Sanitization** - Second layer of protection -3. **Security Headers** - Browser-level protection -4. **Request Logging** - Audit trail for security incidents - -### Threat Mitigation - -The system protects against: - -- **Path Traversal Attacks** - Prevented by pattern detection -- **Malicious File Uploads** - Blocked by size and MIME type validation -- **Injection Attacks** - Mitigated by input sanitization -- **XSS Attacks** - Prevented by security headers -- **Clickjacking** - Blocked by X-Frame-Options header -- **MIME Sniffing** - Prevented by X-Content-Type-Options header - -### Logging and Monitoring - -All validation failures are logged for security monitoring: - -``` -Warning: Bucket name was sanitized from '../etc' to 'etc' -``` - -Monitor these logs for: -- Repeated validation failures from the same IP -- Path traversal attempts -- Unusual file upload patterns -- Suspicious input patterns - -## Future Enhancements - -Potential improvements to the validation system: - -1. **Rate Limiting** - Implement per-IP rate limiting -2. **Content Scanning** - Add virus/malware scanning for uploads -3. **Advanced MIME Detection** - Use magic bytes for MIME type verification -4. **Configurable Limits** - Make size limits configurable via environment variables -5. **Custom Validation Rules** - Allow users to define custom validation rules -6. **Audit Logging** - Enhanced logging with structured audit trails -7. **Metrics** - Track validation failure rates and patterns - -## References - -- [OWASP Input Validation Cheat Sheet](https://cheatsheetseries.owasp.org/cheatsheets/Input_Validation_Cheat_Sheet.html) -- [OWASP Path Traversal](https://owasp.org/www-community/attacks/Path_Traversal) -- [OWASP Secure Headers Project](https://owasp.org/www-project-secure-headers/) -- [CWE-22: Path Traversal](https://cwe.mitre.org/data/definitions/22.html) -- [CWE-434: Unrestricted Upload of File with Dangerous Type](https://cwe.mitre.org/data/definitions/434.html) From de041c43c0bbd931d1a0048d8708c17fdbb7eeaa Mon Sep 17 00:00:00 2001 From: manny-uncharted Date: Wed, 3 Dec 2025 11:02:49 +0100 Subject: [PATCH 5/5] fix: remove unused server and SDK code, this. was used for testing --- replica/main.go | 180 ------------------------------------------------ replica/sdk.go | 104 ---------------------------- 2 files changed, 284 deletions(-) delete mode 100644 replica/main.go delete mode 100644 replica/sdk.go diff --git a/replica/main.go b/replica/main.go deleted file mode 100644 index bb5772c..0000000 --- a/replica/main.go +++ /dev/null @@ -1,180 +0,0 @@ -// cmd/server/main.go - -package main - -import ( - "encoding/json" - "log" - "net/http" - "os" - "strings" - - akavesdk "github.com/akave-ai/go-akavelink/internal/sdk" - "github.com/akave-ai/go-akavelink/internal/utils" -) - -// AkaveResponse is our JSON envelope. -type AkaveResponse struct { - Success bool `json:"success"` - Data interface{} `json:"data,omitempty"` - Error string `json:"error,omitempty"` -} - -type server struct { - client *akavesdk.Client -} - -func (s *server) healthHandler(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - http.Error(w, "method not allowed", http.StatusMethodNotAllowed) - return - } - w.WriteHeader(http.StatusOK) - w.Write([]byte("ok")) -} - -func (s *server) bucketsHandler(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - http.Error(w, "method not allowed", http.StatusMethodNotAllowed) - return - } - - buckets, err := s.client.ListBuckets() - if err != nil { - http.Error(w, "failed to list buckets: "+err.Error(), http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(AkaveResponse{Success: true, Data: buckets}) -} - -func (s *server) uploadHandler(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { - http.Error(w, "method not allowed", http.StatusMethodNotAllowed) - return - } - - // extract bucketName from /files/upload/{bucketName} - parts := strings.Split(r.URL.Path, "/") - if len(parts) < 4 || parts[3] == "" { - http.Error(w, "bucketName missing in path", http.StatusBadRequest) - return - } - bucketName := parts[3] - - if err := r.ParseMultipartForm(32 << 20); err != nil { - http.Error(w, "failed to parse form: "+err.Error(), http.StatusBadRequest) - return - } - - file, handler, err := r.FormFile("file") - if err != nil { - http.Error(w, "file retrieval error: "+err.Error(), http.StatusBadRequest) - return - } - defer file.Close() - - ctx := r.Context() - // Attempt to initialize upload stream - uploadStream, err := s.client.CreateFileUpload(ctx, bucketName, handler.Filename) - if err != nil { - // if bucket doesn't exist, create it and retry - if strings.Contains(err.Error(), "BucketNonexists") { - if err2 := s.client.CreateBucket(ctx, bucketName); err2 != nil { - http.Error(w, "bucket creation failed: "+err2.Error(), http.StatusInternalServerError) - return - } - // retry upload stream initialization - uploadStream, err = s.client.CreateFileUpload(ctx, bucketName, handler.Filename) - } - if err != nil { - http.Error(w, "upload init failed: "+err.Error(), http.StatusInternalServerError) - return - } - } - - // Now stream the file - meta, err := s.client.Upload(ctx, uploadStream, file) - if err != nil { - http.Error(w, "upload failed: "+err.Error(), http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusCreated) - resp := map[string]interface{}{ - "message": "File uploaded successfully", - "rootCID": meta.RootCID, - "bucketName": meta.BucketName, - "fileName": meta.Name, - "size": meta.Size, - "encodedSize": meta.EncodedSize, - "committedAt": meta.CommittedAt, - } - json.NewEncoder(w).Encode(AkaveResponse{Success: true, Data: resp}) -} - - -func (s *server) downloadHandler(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - http.Error(w, "method not allowed", http.StatusMethodNotAllowed) - return - } - - // extract bucketName & fileName from /files/download/{bucketName}/{fileName} - parts := strings.Split(r.URL.Path, "/") - if len(parts) < 5 { - http.Error(w, "path must be /files/download/{bucket}/{file}", http.StatusBadRequest) - return - } - bucketName := parts[3] - fileName := parts[4] - - dlStream, err := s.client.CreateFileDownload(r.Context(), bucketName, fileName) - if err != nil { - http.Error(w, "download init failed: "+err.Error(), http.StatusInternalServerError) - return - } - - if err := s.client.Download(r.Context(), dlStream, w); err != nil { - log.Printf("download error: %v", err) - } -} - -func MainFunc() { - utils.LoadEnvConfig() - - key := os.Getenv("AKAVE_PRIVATE_KEY") - node := os.Getenv("AKAVE_NODE_ADDRESS") - if key == "" || node == "" { - log.Fatal("AKAVE_PRIVATE_KEY and AKAVE_NODE_ADDRESS must be set") - } - - cfg := akavesdk.Config{ - NodeAddress: node, - MaxConcurrency: 10, - BlockPartSize: 1 << 20, - UseConnectionPool: true, - PrivateKeyHex: key, - } - client, err := akavesdk.NewClient(cfg) - if err != nil { - log.Fatalf("client init error: %v", err) - } - defer client.Close() - - srv := &server{client: client} - mux := http.NewServeMux() - mux.HandleFunc("/health", srv.healthHandler) - mux.HandleFunc("/buckets", srv.bucketsHandler) - mux.HandleFunc("/files/upload/", srv.uploadHandler) - mux.HandleFunc("/files/download/", srv.downloadHandler) - - log.Println("Server listening on :8080") - log.Fatal(http.ListenAndServe(":8080", mux)) -} - -func main() { - MainFunc() -} diff --git a/replica/sdk.go b/replica/sdk.go deleted file mode 100644 index 03ce328..0000000 --- a/replica/sdk.go +++ /dev/null @@ -1,104 +0,0 @@ -// internal/sdk/sdk.go - -package sdk - -import ( - "context" - "fmt" - "io" - - "github.com/akave-ai/akavesdk/sdk" -) - -// Config holds configuration for the Akave SDK client. -type Config struct { - NodeAddress string - MaxConcurrency int - BlockPartSize int64 - UseConnectionPool bool - PrivateKeyHex string -} - -// Client wraps the official Akave SDK's IPC interface and core instance. -type Client struct { - *sdk.IPC - sdk *sdk.SDK -} - -// NewClient constructs and returns a Client using the provided Config. -func NewClient(cfg Config) (*Client, error) { - if cfg.PrivateKeyHex == "" { - return nil, fmt.Errorf("private key is required for IPC client but was not provided") - } - - sdkOpts := []sdk.Option{ - sdk.WithPrivateKey(cfg.PrivateKeyHex), - } - - core, err := sdk.New( - cfg.NodeAddress, - cfg.MaxConcurrency, - cfg.BlockPartSize, - cfg.UseConnectionPool, - sdkOpts..., - ) - if err != nil { - return nil, fmt.Errorf("failed to initialize Akave SDK: %w", err) - } - - ipc, err := core.IPC() - if err != nil { - core.Close() - return nil, fmt.Errorf("failed to get IPC client from Akave SDK: %w", err) - } - - return &Client{IPC: ipc, sdk: core}, nil -} - -// Close gracefully shuts down the underlying SDK connection. -func (c *Client) Close() error { - return c.sdk.Close() -} - -// CreateBucket provisions a new bucket under the caller’s key. -func (c *Client) CreateBucket(ctx context.Context, bucketName string) error { - // call through to the IPC layer - if _, err := c.IPC.CreateBucket(ctx, bucketName); err != nil { - return fmt.Errorf("failed to create bucket %q: %w", bucketName, err) - } - return nil -} - -// ListBuckets returns the names of all buckets accessible to this client. -func (c *Client) ListBuckets() ([]string, error) { - buckets, err := c.IPC.ListBuckets(context.Background()) - if err != nil { - return nil, fmt.Errorf("failed to list buckets: %w", err) - } - - names := make([]string, len(buckets)) - for i, b := range buckets { - names[i] = b.Name - } - return names, nil -} - -// CreateFileUpload opens a new upload session for the given bucket and file name. -func (c *Client) CreateFileUpload(ctx context.Context, bucket, fileName string) (*sdk.IPCFileUpload, error) { - return c.IPC.CreateFileUpload(ctx, bucket, fileName) -} - -// Upload streams the given reader into the established upload session. -func (c *Client) Upload(ctx context.Context, upload *sdk.IPCFileUpload, reader io.Reader) (sdk.IPCFileMetaV2, error) { - return c.IPC.Upload(ctx, upload, reader) -} - -// CreateFileDownload opens a download session for the specified bucket and file. -func (c *Client) CreateFileDownload(ctx context.Context, bucket, fileName string) (sdk.IPCFileDownload, error) { - return c.IPC.CreateFileDownload(ctx, bucket, fileName) -} - -// Download writes the content of the download session to the provided writer. -func (c *Client) Download(ctx context.Context, download sdk.IPCFileDownload, writer io.Writer) error { - return c.IPC.Download(ctx, download, writer) -}