update READMEs and clean

Author: dfulu

README.md

@@ -6,29 +6,18 @@
 [![contributors badge](https://img.shields.io/github/contributors/openclimatefix/cloudcasting-app?color=FFFFFF)](https://github.com/openclimatefix/cloudcasting-app/graphs/contributors)
 [![ease of contribution: hard](https://img.shields.io/badge/ease%20of%20contribution:%20hard-bb2629)](https://github.com/openclimatefix#how-easy-is-it-to-get-involved)
 
-This repo is used to run the OCF-ATI cloudcasting model live in production. This model takes
-previous frames of EUMETSAT satellite images and forecasts the future frames to come.
+This repo is used to run the OCF-ATI cloudcasting model live in production and meausure its 
+performance. The model takes previous frames of EUMETSAT satellite images and forecasts the future 
+frames to come.
 
-The repo associated with training the models run here is https://github.com/openclimatefix/sat_pred
-
-The model checkpoints are hosted at:
-https://huggingface.co/openclimatefix/cloudcasting_uk
-
-## Environment Variables
-
-The following environment variables are used in the app:
-
-- `SATELLITE_ZARR_PATH`: The path to the satellite data in Zarr format.
-- `OUTPUT_PREDICTION_DIRECTORY`: The directory where results are saved. 
-
-### Optional Environment Variables
-
-- `SATELLITE_SCALE_FACTOR`: The scale factor for the satellite data. Defaults to 1023. 
-- `SATELLITE_15_ZARR_PATH`: The path to the 15 minute satellite data in Zarr format. If 
-this is not set then the `SATELLITE_ZARR_PATH` is used by `.zarr` is repalced with `_15.zarr`
+This repo contains two different packages:
+ - `cloudcasting_app`: Used to run inference
+ - `cloudcasting_metrics`: Used to score the predictions against ground truth
 
 ## Installation
 
+Both packages will be installed simultaneously using
+
 ## Setup / Installation
 
 ```bash
@@ -38,22 +27,21 @@ pip install .
 ```
 
 
-## Example usage
+## Usage and environmental variables
 
-### Running the app locally
-
-It is possbile to run the app locally by setting the required environment variables listed at the top of the [app](src/cloudcasting_app/app.py), these should point to the relevant paths for loading satellite data 
-and saving predicitons.
+See the READMEs in `src/cloudcasting_app` and `src/cloudcasting_metrics`.
 
 ## Development
 
 ### Running the test suite
 
-The test suite is via pytest and can be run from command line using
+The test suite is via pytest and can be run from command line using.
 
 ```
 pytest
 ```
+
+This will run tests for both packages.
 
 
 ## Contributing and community

pyproject.toml

@@ -36,7 +36,6 @@ dependencies = [
     "torchvision @ https://download.pytorch.org/whl/cpu/torchvision-0.18.1-cp312-cp312-macosx_11_0_arm64.whl ; platform_system == 'Darwin' and platform_machine == 'arm64'",
     "torch @ https://download.pytorch.org/whl/cpu/torch-2.3.1%2Bcpu-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64'",
     "torchvision @ https://download.pytorch.org/whl/cpu/torchvision-0.18.1%2Bcpu-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64'",
-    "typer==0.15.1",
     "xarray==2025.7.1",
     "zarr==3.1.1",
 ]
@@ -53,7 +52,8 @@ dev = [
 
 [project.scripts]
 # Put entrypoints in here
-cloudcasting-app = "cloudcasting_app.app:main"
+cloudcasting-app = "cloudcasting_app.app:app"
+cloudcasting-metrics = "cloudcasting_metrics.app:app"
 
 [project.urls]
 repository = "https://github.com/openclimatefix/cloudcasting-app"

src/cloudcasting_app/README.md

@@ -0,0 +1,29 @@
+# cloudcasting-app
+
+This package is used to run inference on the model. This model takes previous frames of EUMETSAT 
+satellite images and forecasts the future frames to come.
+
+The repo associated with training the models run here is https://github.com/openclimatefix/sat_pred
+
+The model checkpoints are hosted at:
+https://huggingface.co/openclimatefix/cloudcasting_uk
+
+## Environment Variables
+
+The following environment variables are used in the app:
+
+- `SATELLITE_ZARR_PATH`: The path to the satellite data in Zarr format.
+- `PREDICTION_SAVE_DIRECTORY`: The directory where predictions will be saved. 
+
+### Optional Environment Variables
+
+- `SATELLITE_15_ZARR_PATH`: The path to the 15 minute satellite data in Zarr format. If 
+this is not set then the `SATELLITE_ZARR_PATH` is used by `.zarr` is repalced with `_15.zarr`
+
+## Example usage
+
+### Running the app locally
+
+It is possbile to run the app locally by setting the required environment variables listed at the
+top of the [app](src/cloudcasting_app/app.py), these should point to the relevant paths for loading
+satellite data and saving predicitons.

src/cloudcasting_app/__init__.py

@@ -43,9 +43,4 @@ def structured_formatter(record: "loguru.Record") -> str:
     loguru.logger.add(
         sys.stdout, format=structured_formatter, backtrace=True,
         level=os.getenv("LOGLEVEL", "INFO").upper(),
-    )
-
-# Uncomment and change the list to quieten external libraries
-# for logger in ["aiobotocore", "cfgrib"]:
-#    logging.getLogger(logger).setLevel(logging.WARNING)
-
+    )

src/cloudcasting_app/app.py

@@ -2,7 +2,7 @@
 
 This app expects these environmental variables to be available:
     SATELLITE_ZARR_PATH (str): The path of the input satellite data
-    OUTPUT_PREDICTION_DIRECTORY (str): The path of the directory to save the predictions to
+    PREDICTION_SAVE_DIRECTORY (str): The path of the directory to save the predictions to
 """
 
 import os
@@ -12,7 +12,6 @@
 import hydra
 import pandas as pd
 import torch
-import typer
 import xarray as xr
 import yaml
 from huggingface_hub import snapshot_download
@@ -33,13 +32,11 @@
 # Model will use GPU if available
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
-
 # Model revision on huggingface
 REPO_ID = "openclimatefix/cloudcasting_uk"
 REVISION = "47643e89000e64e0150f7359ccc0cb6524948712"
 
 
-
 def app(t0=None):
     """Inference function for production
 
@@ -123,7 +120,7 @@ def app(t0=None):
     ds_y_hat.sat_pred.attrs.update(ds.data.attrs)
 
     # Save predictions to the latest path and to path with timestring
-    out_dir = os.environ["OUTPUT_PREDICTION_DIRECTORY"]
+    out_dir = os.environ["PREDICTION_SAVE_DIRECTORY"]
 
     if satellite_downloader.use_5_minute:
         latest_zarr_path = f"{out_dir}/latest.zarr"
@@ -141,11 +138,3 @@ def app(t0=None):
             fs.rm(path, recursive=True)
 
         ds_y_hat.to_zarr(path)
-
-
-def main() -> None:
-    """Entrypoint to the application."""
-    typer.run(app)
-
-if __name__ == "__main__":
-    main()

src/cloudcasting_metrics/README.md

@@ -0,0 +1,13 @@
+# cloudcasting-metrics
+
+This package is used to score the predictions made by the model. By default, the scoring is run for 
+all init-times made the day before.
+
+## Environment Variables
+
+The following environment variables are used in the app:
+
+- `SATELLITE_ICECHUNK_ARCHIVE`: The path to the satellite archive in icechunk formast
+- `PREDICTION_SAVE_DIRECTORY`: The directory where the cloudcasting predictions are saved. 
+  i.e. set to the same as `PREDICTION_SAVE_DIRECTORY` in `cloudcasting_app`.
+- `METRIC_ZARR_PATH`: Where to save metrics zarr

tests/test_cloudcasting_app/test_app.py

@@ -14,7 +14,7 @@ def test_app(sat_5_data, tmp_path, init_time):
     # In production sat zarr is zipped
     os.environ["SATELLITE_ZARR_PATH"] = "temp_sat.zarr.zip"
 
-    os.environ["OUTPUT_PREDICTION_DIRECTORY"] = f"{tmp_path}"
+    os.environ["PREDICTION_SAVE_DIRECTORY"] = f"{tmp_path}"
 
     with zarr.storage.ZipStore("temp_sat.zarr.zip", mode="x") as store:
         sat_5_data.to_zarr(store)