Merge remote-tracking branch 'origin/master' into feature/SK-1336

Author: carl-andersson

.ci/tests/studio/studio.sh

@@ -32,7 +32,7 @@ fi
 fedn studio login -u $STUDIO_USER -P $STUDIO_PASSWORD -H $STUDIO_HOST
 TAG=sha-$(git rev-parse HEAD | cut -c1-7)
 PROJECT_NAME=citest_$TAG
-fedn project create -n $PROJECT_NAME -H $STUDIO_HOST --branch main --repository ghcr.io/scaleoutsystems/fedn --image fedn:$TAG --no-interactive
+fedn project create -n $PROJECT_NAME -H $STUDIO_HOST --branch $STUDIO_BRANCH --repository ghcr.io/scaleoutsystems/fedn --image fedn:$TAG --no-interactive
 for i in {1..10}; do echo "Attempt $i of 10"; if ! fedn project list -H $STUDIO_HOST | awk -F'|' -v project="$PROJECT_NAME" '$1 ~ project {gsub(/^[ \t]+|[ \t]+$/, "", $4); if ($4 == "active") {exit 1} else {exit 0}}'; then echo "Status is active, exiting"; break; else echo "Status is not active, sleeping..."; sleep 5; fi; done
 FEDN_PROJECT=$(fedn project list -H $STUDIO_HOST | awk -F'|' -v project="$PROJECT_NAME" '$1 ~ project {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2}')
 fedn project set-context -id $FEDN_PROJECT -H $STUDIO_HOST

.github/dependabot.yml

@@ -12,7 +12,8 @@ updates:
     reviewers:
       - "Wrede"
       - "stefanhellander"
+      - "niklastheman"
     # To ignore a dependency, uncomment the next two lines and replace "example-package" with the name of the dependency
     #ignore:
     #  - dependency-name: "example-package"
-    #    versions: ["< 1.0.0"]
+    #    versions: ["< 1.0.0"]

.github/workflows/integration-test-studio.yaml

@@ -41,10 +41,10 @@ on:
         description: 'Array Size Factor, used in load-test, 1 = 144MB'
         required: false
         default: '1'
-      FEDN_CLIENT_TIMEOUT:
-        description: 'Client Connection Timeout (OBS - not related to round timeout)'
+      STUDIO_BRANCH:
+        description: 'Branch to test fedn chart'
         required: false
-        default: '420'
+        default: 'main'
 
 jobs:
   integration-test:
@@ -84,8 +84,9 @@ jobs:
         echo "FEDN_NR_EXPECTED_AGG=${{ github.event.inputs.FEDN_NR_EXPECTED_AGG || '2' }}" >> .ci/tests/studio/.env
         echo "FEDN_SESSION_TIMEOUT=${{ github.event.inputs.FEDN_SESSION_TIMEOUT || '420' }}" >> .ci/tests/studio/.env
         echo "FEDN_ARRAY_SIZE_FACTOR=${{ github.event.inputs.FEDN_ARRAY_SIZE_FACTOR || '1' }}" >> .ci/tests/studio/.env
-        echo "FEDN_CLIENT_TIMEOUT=${{ github.event.inputs.FEDN_CLIENT_TIMEOUT || '420' }}" >> .ci/tests/studio/.env
+        echo "FEDN_CLIENT_TIMEOUT=420" >> .ci/tests/studio/.env
         echo "FEDN_PACKAGE_EXTRACT_DIR=client" >> .ci/tests/studio/.env
+        echo "STUDIO_BRANCH=${{ github.event.inputs.STUDIO_BRANCH || 'main' }}" >> .ci/tests/studio/.env
         
     - name: Run integration tests
       env:

docs/conf.py

@@ -11,7 +11,7 @@
 author = "Scaleout Systems AB"
 
 # The full version, including alpha/beta/rc tags
-release = "0.28.1"
+release = "0.30.0"
 
 # Add any Sphinx extension module names here, as strings
 extensions = [

examples/api-tutorials/Local_Compute_Example.ipynb

@@ -26,7 +26,7 @@
         },
         {
             "cell_type": "code",
-            "execution_count": 7,
+            "execution_count": null,
             "id": "c4d3134e-f22c-40af-989d-439c9867acc0",
             "metadata": {},
             "outputs": [],

examples/async-clients/run_clients.py

@@ -18,10 +18,10 @@
 from io import BytesIO
 from multiprocessing import Process
 
+import click
 import numpy as np
 from init_seed import compile_model, make_data
 from sklearn.metrics import accuracy_score
-import click
 
 from config import settings
 from fedn import FednClient
@@ -168,7 +168,9 @@ def run_client(name="client", client_id=None, no_discovery=False, intermittent=F
     else:
         fl_client.run()
 
+
 if __name__ == "__main__":
+
     @click.command()
     @click.option("--name", "-n", default="client", help="Base name for clients (will be appended with number)")
     @click.option("--no-discovery", is_flag=True, help="Connect to combiner without discovery service")

examples/mnist-pytorch/client/validate.py

@@ -2,9 +2,9 @@
 import sys
 
 import torch
+from data import load_data
 from model import load_parameters
 
-from data import load_data
 from fedn.utils.helpers.helpers import save_metrics
 
 dir_path = os.path.dirname(os.path.realpath(__file__))

fedn/cli/client_cmd.py

@@ -269,7 +269,7 @@ def client_start_v2_cmd(
     if token and token != "":
         config["token"] = token
         if config["token"]:
-            click.echo(f"Input param token: {token} overrides value from file")
+            click.echo("Input param token overrides value from file")
 
     if name and name != "":
         config["name"] = name

fedn/cli/project_cmd.py

@@ -49,6 +49,7 @@ def delete_project(ctx, id: str = None, protocol: str = None, host: str = None,
 
 
 @click.option("-n", "--name", required=False, default=None, help="Name of new project.")
+@click.option("-n", "--nr-of-combiners", required=False, default=1, help="Number of combiners for the project (admin feature).")
 @click.option("-p", "--protocol", required=False, default=STUDIO_DEFAULTS["protocol"], help="Communication protocol of studio (api)")
 @click.option("-H", "--host", required=False, default=STUDIO_DEFAULTS["host"], help="Hostname of studio (api)")
 @click.option("--branch", required=False, default=None, help="Studio branch (default main). Requires admin in Studio")
@@ -61,6 +62,7 @@ def delete_project(ctx, id: str = None, protocol: str = None, host: str = None,
 def create_project(
     ctx,
     name: str = None,
+    nr_of_combiners: int = 1,
     protocol: str = None,
     host: str = None,
     no_interactive: bool = False,
@@ -90,7 +92,11 @@ def create_project(
     else:
         # Call the authentication API
         try:
-            response = requests.post(url, data={"name": name, "studio_branch": branch, "fedn_image": image, "fedn_repo": repository}, headers=headers)
+            response = requests.post(
+                url,
+                data={"name": name, "nr_of_combiners": nr_of_combiners, "studio_branch": branch, "fedn_image": image, "fedn_repo": repository},
+                headers=headers,
+            )
             response_message = response.json().get("message")
             if response.status_code == 201:
                 click.secho(f"Project with name '{name}' created.", fg="green")
@@ -260,3 +266,54 @@ def no_project_exists(response) -> bool:
     elif response_json.get("error"):
         return True
     return False
+
+
+@click.option("-id", "--id", required=True, help="Slug of the project to add the combiner to.")
+@click.option("-nr_of_combiners", "--nr_of_combiners", required=True, help="Total number of combiners to deploy.")
+@click.option("--branch", required=False, default=None, help="Studio branch (optional)")
+@click.option("--image", required=False, default=None, help="FEDn image (optional)")
+@click.option("--repository", required=False, default=None, help="FEDn repo URL (optional)")
+@click.option("-p", "--protocol", required=False, default=STUDIO_DEFAULTS["protocol"], help="Protocol to use for Studio API")
+@click.option("-H", "--host", required=False, default=STUDIO_DEFAULTS["host"], help="Host to use for Studio API")
+@project_cmd.command("add-combiner")
+@click.pass_context
+def add_combiner_to_project(
+    ctx,
+    id: str,
+    nr_of_combiners: str,
+    branch: str = None,
+    image: str = None,
+    repository: str = None,
+    protocol: str = None,
+    host: str = None,
+):
+    """Add a Combiner (and optional Server Functions) to an existing project."""
+    studio_api = True
+    headers = {"Content-Type": "application/json"}
+
+    _token = get_token(None, True)
+    if _token:
+        headers["Authorization"] = _token
+
+    url = get_api_url(protocol=protocol, host=host, port=None, endpoint="projects/add_combiner", usr_api=studio_api)
+
+    payload = {
+        "project_slug": id,
+        "nr_of_combiners": nr_of_combiners,
+    }
+    if branch:
+        payload["studio_branch"] = branch
+    if image:
+        payload["fedn_image"] = image
+    if repository:
+        payload["fedn_repo"] = repository
+
+    try:
+        response = requests.post(url, json=payload, headers=headers)
+        if response.status_code in [200, 201, 202]:
+            click.secho(f"Combiner deployment initiated for project '{id}'.", fg="green")
+        else:
+            click.secho(f"Failed to initiate combiner deployment: {response.status_code}", fg="red")
+            print_response(response, "error", True)
+    except requests.exceptions.RequestException as e:
+        click.secho(f"Request failed: {e}", fg="red")

fedn/cli/shared.py

@@ -71,7 +71,9 @@ def get_project_url(protocol: str, host: str, port: str, endpoint: str) -> str:
 
 
 def get_token(token: str, usr_token: bool) -> str:
-    _token = token or os.environ.get("FEDN_AUTH_TOKEN", None)
+    _token = None
+    if token and isinstance(token, str) and len(token) > 0:
+        _token = token
 
     if _token is None:
         context_path = os.path.join(HOME_DIR, ".fedn")
@@ -82,8 +84,12 @@ def get_token(token: str, usr_token: bool) -> str:
             else:
                 _token = context_data.get("Active project tokens").get("access")
         except Exception as e:
+            _token = None
             click.secho(f"Encountered error {e}. Make sure you are logged in and have activated a project.", fg="red")
 
+    if _token is None:
+        _token = token or os.environ.get("FEDN_AUTH_TOKEN", None)
+
     scheme = os.environ.get("FEDN_AUTH_SCHEME", "Bearer")
 
     return f"{scheme} {_token}"