From ddf0ff5483bf0162ee6348b7a7df5616ec60d005 Mon Sep 17 00:00:00 2001 From: Mark Ruvald Pedersen Date: Sat, 14 Jun 2025 00:22:57 +0200 Subject: [PATCH 1/3] Refine docs build and update docs --- .github/workflows/ci.yml | 17 ++++++++++++ .gitignore | 1 + default.nix | 2 ++ docs/_static/.gitkeep | 0 docs/_templates/.gitkeep | 0 docs/conf.py | 39 +++++++++++++++++++++++++++ docs/index.rst | 58 ++++++++++++++++++++++++++++++++++++++++ justfile | 4 +++ readme.md | 10 +++++++ 9 files changed, 131 insertions(+) create mode 100644 docs/_static/.gitkeep create mode 100644 docs/_templates/.gitkeep create mode 100644 docs/conf.py create mode 100644 docs/index.rst diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c00c34..a456647 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,3 +22,20 @@ jobs: env: TZ: Europe/Copenhagen run: nix-shell --pure --run "just unittest" + + docs: + needs: test + runs-on: ubuntu-latest + permissions: + contents: read + pages: write + id-token: write + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v25 + - name: Build documentation + run: nix-shell --pure --run "just docs-html" + - uses: actions/upload-pages-artifact@v2 + with: + path: docs/_build/html + - uses: actions/deploy-pages@v2 diff --git a/.gitignore b/.gitignore index 7ef6f73..6bfff68 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ __pycache__ tmp/ build/ +docs/_build/ dist/ git_recycle_bin.egg-info/ diff --git a/default.nix b/default.nix index 1059043..6852ded 100644 --- a/default.nix +++ b/default.nix @@ -17,6 +17,8 @@ pkgs.python311Packages.buildPythonApplication rec { colorama dateparser pytest + sphinx + sphinx-material ]; postInstall = '' diff --git a/docs/_static/.gitkeep b/docs/_static/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docs/_templates/.gitkeep b/docs/_templates/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..794ecd6 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,39 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information +import os +import sys +sys.path.insert(0, os.path.abspath('..')) + +project = 'git-recycle-bin' +copyright = '2025, git recycle bin' +author = 'git recycle bin' + +version = '0.2.5' +release = '0.2.5' + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [] + +templates_path = ['_templates'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'sphinx_material' +html_static_path = ['_static'] +html_theme_options = { + 'nav_title': 'git-recycle-bin Docs', + 'color_primary': 'blue', + 'color_accent': 'light-blue', + 'globaltoc_depth': 5, +} diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..9ad93ad --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,58 @@ +.. _index: + +git-recycle-bin +================ + +git-recycle-bin publishes build outputs to a dedicated git repository +while keeping full traceability back to the source commit. Artifacts +can expire automatically yet can be prolonged or removed using normal +git commands. Discovery of available artifacts is done via git notes. + +How to install +-------------- + +Enter the provided Nix shell and install the package: + +.. code-block:: bash + + nix-shell shell.nix --pure + pip install . + +Usage +----- + +Run the tool inside your build directory. For command-line help: + +.. code-block:: bash + + git_recycle_bin.py --help + +Further Examples +---------------- + +See the ``demos/`` directory for practical usage scenarios. + +Motivation +---------- + +Storing binaries in git offers a simple, auditable artifact repository +without extra infrastructure. + +Implementation details +---------------------- + +Artifacts are stored as commits with a commit-message schema as +outlined in the README. Refspecs control their placement and git notes +provide quick lookup information. + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/justfile b/justfile index 3c1900c..de0f4cf 100644 --- a/justfile +++ b/justfile @@ -6,6 +6,10 @@ unittest: demo0: git_recycle_bin.py --help +# Build HTML documentation +docs-html: + sphinx-build -b html docs docs/_build/html + # for general flags look at push.justfile # the other examples are only command specific diff --git a/readme.md b/readme.md index 1e913d7..df063c0 100644 --- a/readme.md +++ b/readme.md @@ -17,6 +17,16 @@ Unlike many artifact management systems out there, the artifacts published here * Locally or CI-side, this tool creates and pushes artifacts, see `--help` and examples below. * Garbage collection of expired artifacts is done at server-side. +## Documentation +Documentation is built with [Sphinx](https://www.sphinx-doc.org/). A +`docs-html` target is provided in the `justfile`. Run it from the Nix shell: + +```bash +nix-shell shell.nix --pure --run 'just docs-html' +``` + +The resulting HTML is located under `docs/_build/html` and published from CI. + ## Schema Artifacts come with meta-data, for {expiry, traceabillity, audit, placement} purposes. \ From 54c456afa4d185d2a388bb0096cab495c2c9dc0d Mon Sep 17 00:00:00 2001 From: Mark Ruvald Pedersen Date: Sat, 14 Jun 2025 00:46:52 +0200 Subject: [PATCH 2/3] Merge master --- .github/workflows/ci.yml | 102 ++++++++- .gitignore | 3 + .gitlab-ci.yml | 28 +++ .markdownlint.json | 9 + AGENTS.md | 74 +++++- CONTRIBUTING.md | 31 +++ aux/git_add_ssh_remote.sh | 2 +- contribution.md | 13 ++ default.nix | 10 + issues/0001-git-notes-integration.md | 39 +++- issues/0002-gitlfs-support.md | 24 ++ issues/0003-non-orphan-lineage.md | 23 ++ issues/0004-non-expiring-artifacts.md | 23 ++ issues/0005-library-support.md | 24 ++ issues/0006-ui-queries-overview.md | 24 ++ issues/0007-python-test-matrix.md | 25 +++ issues/0008-prepare-for-pypi.md | 25 +++ issues/0009-versioning-strategy.md | 23 ++ issues/0010-stress-testing.md | 24 ++ issues/0011-multi-python-testing.md | 18 ++ issues/0011-sibling-demo-src-bin-projects.md | 23 ++ justfile | 13 +- readme.md | 223 +++++++++++++------ setup.py | 50 +++-- shell.nix | 2 + src/commit_msg.py | 4 +- tests/test_arg_parser.py | 49 ++++ tests/test_commit_msg.py | 62 ++++++ tests/test_download.py | 27 +++ tests/test_git_recycle_bin_cmds.py | 167 ++++++++++++++ tests/test_list.py | 39 ++++ tests/test_printer.py | 20 ++ tests/test_rbgit.py | 122 ++++++++++ tests/test_util.py | 31 +++ tests/test_util_date.py | 46 ++++ tests/test_util_file.py | 18 ++ tests/test_util_string.py | 40 ++++ tests/test_util_sysinfo.py | 15 ++ 38 files changed, 1380 insertions(+), 115 deletions(-) create mode 100644 .markdownlint.json create mode 100644 CONTRIBUTING.md create mode 100644 contribution.md create mode 100644 issues/0002-gitlfs-support.md create mode 100644 issues/0003-non-orphan-lineage.md create mode 100644 issues/0004-non-expiring-artifacts.md create mode 100644 issues/0005-library-support.md create mode 100644 issues/0006-ui-queries-overview.md create mode 100644 issues/0007-python-test-matrix.md create mode 100644 issues/0008-prepare-for-pypi.md create mode 100644 issues/0009-versioning-strategy.md create mode 100644 issues/0010-stress-testing.md create mode 100644 issues/0011-multi-python-testing.md create mode 100644 issues/0011-sibling-demo-src-bin-projects.md create mode 100644 tests/test_arg_parser.py create mode 100644 tests/test_commit_msg.py create mode 100644 tests/test_download.py create mode 100644 tests/test_git_recycle_bin_cmds.py create mode 100644 tests/test_list.py create mode 100644 tests/test_printer.py create mode 100644 tests/test_util.py create mode 100644 tests/test_util_date.py create mode 100644 tests/test_util_file.py create mode 100644 tests/test_util_string.py create mode 100644 tests/test_util_sysinfo.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a456647..76b103d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ env: NIX_PATH: "nixpkgs=channel:nixos-24.05" jobs: - test: + usage: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -18,13 +18,25 @@ jobs: run: nix-shell --pure --run "git_recycle_bin.py --help" - name: Just list run: nix-shell --pure --run "just --list" + + unittest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v25 - name: Unit tests env: TZ: Europe/Copenhagen run: nix-shell --pure --run "just unittest" + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + slug: ArtifactLabs/git-recycle-bin + files: coverage.xml docs: - needs: test + needs: unittest runs-on: ubuntu-latest permissions: contents: read @@ -39,3 +51,89 @@ jobs: with: path: docs/_build/html - uses: actions/deploy-pages@v2 + + pip_install: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + - name: Install package + run: pip install . + - name: Install test deps + run: pip install pytest + - name: Run tests + run: PYTHONPATH=$PWD:$PWD/src pytest + - name: CLI help + run: git_recycle_bin.py --help + + demo_help: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v25 + - name: Demo help + run: nix-shell --pure --run "just demo0" + + demo_push: + runs-on: ubuntu-latest + strategy: + matrix: + demo: [demo1, demo1_quiet, demo1_verbose, demo1_vverbose, demo2, demo3, demo4, demo5] + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v25 + - name: Demo push + run: nix-shell --pure --run "just push::${{ matrix.demo }}" + + demo_clean: + runs-on: ubuntu-latest + strategy: + matrix: + demo: [demo1] + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v25 + - name: Demo clean + run: nix-shell --pure --run "just clean::${{ matrix.demo }}" + + demo_list: + runs-on: ubuntu-latest + strategy: + matrix: + demo: [demo1, demo2, demo3, demo4] + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v25 + - name: Demo list + run: nix-shell --pure --run "just list::${{ matrix.demo }}" + + demo_download: + runs-on: ubuntu-latest + strategy: + matrix: + demo: [demo1] + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v25 + - name: Demo download + run: nix-shell --pure --run "just download::${{ matrix.demo }}" + + lint-shell: + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v25 + - name: ShellCheck + run: nix-shell --pure --run "just lint-shell" + + lint-md: + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v25 + - name: Markdownlint + run: nix-shell --pure --run "just lint-md" diff --git a/.gitignore b/.gitignore index 6bfff68..8392ee7 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ build/ docs/_build/ dist/ git_recycle_bin.egg-info/ +.coverage +coverage.xml +htmlcov/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d005559..d908bf4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,6 +2,9 @@ stages: - test - demos +variables: + NIX_PATH: "nixpkgs=channel:nixos-24.05" + default: tags: - csfw-minion @@ -16,6 +19,18 @@ unittest: stage: test script: - nix-shell --pure --run "just unittest" + - curl -Os https://uploader.codecov.io/latest/linux/codecov + - chmod +x codecov + - ./codecov -t ${CODECOV_TOKEN} -r ArtifactLabs/git-recycle-bin -f coverage.xml + +pip_install: + stage: test + image: python:3.11 + script: + - pip install . + - pip install pytest + - PYTHONPATH=$PWD:$PWD/src pytest + - git_recycle_bin.py --help demo_help: stage: demos @@ -52,6 +67,7 @@ demo_push_note: # Not --pure so as to use host system's ssh client - nix-shell --run "just push::demo6_note" + demo_clean: stage: demos script: @@ -81,3 +97,15 @@ demo_download: matrix: - DEMO: - demo1 + +lint_shell: + stage: test + script: + - nix-shell --pure --run "just lint-shell" + allow_failure: true + +lint_md: + stage: test + script: + - nix-shell --pure --run "just lint-md" + allow_failure: true diff --git a/.markdownlint.json b/.markdownlint.json new file mode 100644 index 0000000..a62b0e5 --- /dev/null +++ b/.markdownlint.json @@ -0,0 +1,9 @@ +{ + "default": true, + "MD013": false, + "MD022": false, + "MD032": false, + "MD031": false, + "MD004": false, + "MD012": false +} diff --git a/AGENTS.md b/AGENTS.md index 1d68106..b9134d5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,41 +1,95 @@ # Repo Guidelines for Codex Agents -This project provides a collection of Python scripts and utilities for managing artifacts in a git repository. It uses Nix for reproducible development environments but should remain runnable outside of Nix as a normal Python package. +This project provides a collection of Python scripts and utilities for +managing artifacts in a git repository. +It uses Nix for reproducible development environments but should remain +runnable outside of Nix as a normal Python package. ## Project Structure + The repository is organised into a few key folders: + - `src/` contains the application code and executable entrypoints. - `tests/` holds unit and future integration tests. - `demos/` shows practical usage of the tools. -- `aux/` stores helper scripts used by the build. +- `aux/` holds helper scripts that are not part of the build itself but may be + useful. +- `issues/` contains markdown tickets that track design ideas and open tasks. + When starting new work, add a ticket here first. + +### Issue ticket style + +Tickets under `issues/` should outline: + +- **Purpose** of the idea or feature +- **Acceptance Criteria** describing what must be true to close the ticket +- **Prerequisites** needed before work can begin +- **Questions** that remain open +- **Status** of the issue (e.g. Open, Done) + +Wrap lines in these markdown files at roughly 72 characters for readability. ## Coding Conventions + - Follow PEP 8 style with four-space indentation and descriptive names. - Use type hints where sensible and keep functions short and readable. - Prefer standard library modules over additional dependencies when possible. +- Use `snake_case` for modules and functions, and `PascalCase` for classes. ## General Conventions for AGENTS.md Implementation -- Keep this document in sync with the code base. Update guidelines alongside feature or behaviour changes. -- Clarify any new requirements or conventions in this file so agents know how to contribute. + +- Keep this document in sync with the code base. + Update guidelines alongside feature or behaviour changes. +- Clarify any new requirements or conventions in this file so agents know how + to contribute. ## Testing Requirements + - Preferred: `nix-shell shell.nix --pure --run "just unittest"`. -- Non-Nix: install dependencies from `setup.py` and run `pytest` with `PYTHONPATH=$PWD:$PWD/src`. -- Integration tests comparing HEAD with prior tagged releases must pass before merging breaking behaviour changes. +- Always run `nix-shell --run 'just lint'` to verify code style. +- Non-Nix: install dependencies from `setup.py` and run `pytest` with + `PYTHONPATH=$PWD:$PWD/src`. +- Integration tests comparing HEAD with prior tagged releases must pass + before merging breaking behaviour changes. ## Environment + - Code should run with Python 3.11+. Avoid Nix-specific runtime assumptions. - Ensure the project remains installable with `pip install .`. ## Style and Quality -- Keep the code base robust and industrial readable. Use type hints and descriptive names. -- Avoid breaking changes. Introduce integration tests comparing HEAD with previous tagged releases when altering behaviour. -- Keep documentation and design notes in sync with the implementation to avoid drift. + +- Keep the code base robust and industrial readable. Use type hints and + descriptive names. +- Avoid breaking changes. Introduce integration tests comparing HEAD with + previous tagged releases when altering behaviour. +- Keep documentation and design notes in sync with the implementation to avoid + drift. +- Wrap lines in Markdown to keep them reasonably short and readable. +- Embrace the UNIX and KISS philosophies and apply POLA (principle of least + astonishment). +- Consider input/output and data-structures first, then implementation. +- Avoid refactoring unless explicitly approved. If refactoring occurs, ensure + end-to-end tests exist and add unit tests. ## Pull Request Guidelines + - Keep PRs focused and include a summary of changes and testing steps. - Ensure programmatic checks pass locally before opening a PR. ## Programmatic Checks -- Run `nix-shell shell.nix --pure --run "just unittest"` and ensure all tests succeed. + +- Run `nix-shell shell.nix --pure --run "just unittest"` and ensure all tests + succeed. +- Run `nix-shell --run 'just lint'` and fix any reported issues. - Validate that documentation updates accompany code changes to prevent drift. +- When editing CLI examples, run `git_recycle_bin.py --help` to verify option + names and arguments. + +## Continuous Integration + +The project maintains both GitHub and GitLab pipelines. +When changing one CI configuration, update the other in a comparable manner so +the steps remain logically aligned. +Keep `.gitlab-ci.yml` and `.github/workflows/ci.yml` synchronized to avoid +divergent behaviour. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..21841f0 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,31 @@ +# Contributing πŸ™Œ + +Thank you for considering a contribution! +To keep the project easy to work with please follow these steps: + +1. **Fork and branch** from `master`. + Clone your fork then create a feature branch: + + ```bash + git clone git@example.com:you/git-recycle-bin.git + cd git-recycle-bin + git checkout -b my-feature origin/master + ``` + +2. **Coding style** follows PEPΒ 8 with four-space indents + and type hints where useful. +3. **Run tests locally**: + - Preferred: `nix-shell shell.nix --pure --run "just unittest"` + - Non‑Nix: `pip install .` then `PYTHONPATH=$PWD:$PWD/src pytest` +4. **Open a pull request** with a short summary of your changes and how you + tested them. + +Please also read `AGENTS.md` for repository etiquette. + +## Installing Nix + +Nix gives you a fully reproducible build environment. Follow the official +[installation guide](https://nixos.org/download.html) if you don't have it yet. + +All kinds of improvements are welcome – documentation, tests or features. +Happy hacking! πŸš€ diff --git a/aux/git_add_ssh_remote.sh b/aux/git_add_ssh_remote.sh index a02df39..f4a8605 100755 --- a/aux/git_add_ssh_remote.sh +++ b/aux/git_add_ssh_remote.sh @@ -58,4 +58,4 @@ else git remote add "${REMOTE_NAME}_ssh" "$ssh_url" 2>/dev/null || true fi -echo "Remote '${REMOTE_NAME}_ssh' added: $(git remote get-url ${REMOTE_NAME}_ssh)" 1>&2 +echo "Remote '${REMOTE_NAME}_ssh' added: $(git remote get-url "${REMOTE_NAME}_ssh")" 1>&2 diff --git a/contribution.md b/contribution.md new file mode 100644 index 0000000..932e2dc --- /dev/null +++ b/contribution.md @@ -0,0 +1,13 @@ +# Contribution Guidelines + +This project welcomes community contributions. Before starting work on a +new feature or bug fix, check the markdown tickets under `issues/`. +If your idea is not yet covered, add a new ticket there to describe the +proposal. +Tickets should follow the style outlined in `AGENTS.md` with sections for +purpose, acceptance criteria, prerequisites, questions and status. +Follow the coding and testing conventions in `AGENTS.md` when submitting +a pull request. + +Before opening a pull request, you can run `just lint` to check shell +scripts with ShellCheck and Markdown files with `markdownlint`. diff --git a/default.nix b/default.nix index 6852ded..2295af1 100644 --- a/default.nix +++ b/default.nix @@ -17,10 +17,20 @@ pkgs.python311Packages.buildPythonApplication rec { colorama dateparser pytest + pytest-cov sphinx sphinx-material ]; + checkInputs = with pkgs.python311Packages; [ pytest pytest-cov ]; + + checkPhase = '' + runHook preCheck + export PYTHONPATH="$PYTHONPATH:$PWD:$PWD/src" + python -m pytest -vv + runHook postCheck + ''; + postInstall = '' install -Dm755 ${./aux/git_add_ssh_remote.sh} $out/bin/git_add_ssh_remote.sh ''; diff --git a/issues/0001-git-notes-integration.md b/issues/0001-git-notes-integration.md index 8826718..2fc40a1 100644 --- a/issues/0001-git-notes-integration.md +++ b/issues/0001-git-notes-integration.md @@ -1,14 +1,29 @@ # Git notes integration for artifact discovery -## Background -`git notes` allows attaching auxiliary metadata to commits without changing commit hashes. The project now uses git notes to advertise available binary artifacts. - -## Proposed design -- Maintain a dedicated notes namespace such as `refs/notes/artifact/` or `refs/notes/artifact//`. Each note will attach to the corresponding source commit SHA. -- Note contents will be sorted lines of simple JSON with fields: - `{date:, sha:, target:, ttl:, remote:}` -- Clients can fetch and display these notes to discover artifacts related to a source commit without knowing the binary remote upfront. -- Additional helper commands will be added to create and read these notes when pushing or listing artifacts. - -This issue tracked implementation of the git notes scheme described above and in `design_notes.txt`. -Implementation landed in commit 5ef2128. +## Purpose + +Allow discovery of binary artifacts by storing metadata under +`git notes`. + +## Acceptance Criteria + +- Dedicated namespace such as `refs/notes/artifact/` or + `refs/notes/artifact//` stores a note for each + relevant commit. +- Notes contain sorted JSON lines describing the artifact: + `{date:, sha:, target:, ttl:, + remote:}`. +- CLI commands expose creation and reading of notes. + +## Prerequisites + +- Familiarity with `git notes` and the project's artifact scheme. + +## Questions + +- Should notes also capture artifact expiry information? + +## Status + +Done. Implementation was completed in commit `5ef2128` and described +in `design_notes.txt`. diff --git a/issues/0002-gitlfs-support.md b/issues/0002-gitlfs-support.md new file mode 100644 index 0000000..958aa41 --- /dev/null +++ b/issues/0002-gitlfs-support.md @@ -0,0 +1,24 @@ +# Support for GitLFS and garbage collection + +## Purpose + +Explore storing artifacts with GitLFS and understand how LFS objects can +be cleaned up when artifacts expire. + +## Acceptance Criteria + +- Describe how GitLFS performs garbage collection. +- Provide commands that remove unused LFS objects together with the + orphan-branch cleanup. + +## Prerequisites + +- GitLFS installed and configured in the development environment. + +## Questions + +- Does GitLFS need explicit pruning or is it handled automatically? + +## Status + +Open diff --git a/issues/0003-non-orphan-lineage.md b/issues/0003-non-orphan-lineage.md new file mode 100644 index 0000000..f6cf56b --- /dev/null +++ b/issues/0003-non-orphan-lineage.md @@ -0,0 +1,23 @@ +# Non-orphan branch lineage preservation + +## Purpose + +Allow artifact branches to retain the source commit as their parent so the +relationship between source and artifact is visible in normal history. + +## Acceptance Criteria + +- New option to create artifact branches with a parent commit. +- Lineage clearly shown when running standard git history commands. + +## Prerequisites + +- Understanding of how orphan branches are currently produced. + +## Questions + +- Should lineage branches be enabled by default or remain optional? + +## Status + +Open diff --git a/issues/0004-non-expiring-artifacts.md b/issues/0004-non-expiring-artifacts.md new file mode 100644 index 0000000..6de3f41 --- /dev/null +++ b/issues/0004-non-expiring-artifacts.md @@ -0,0 +1,23 @@ +# Support for non-expiring artifacts + +## Purpose + +Keep certain artifacts forever, acting as releases or long-term +checkpoints that never expire. + +## Acceptance Criteria + +- Command line flag or configuration to mark an artifact as permanent. +- Garbage collection leaves these artifacts intact. + +## Prerequisites + +- Current pruning mechanism based on absolute expiry dates. + +## Questions + +- Should permanent artifacts be stored in a separate namespace? + +## Status + +Open diff --git a/issues/0005-library-support.md b/issues/0005-library-support.md new file mode 100644 index 0000000..9bf1b04 --- /dev/null +++ b/issues/0005-library-support.md @@ -0,0 +1,24 @@ +# Convert tools into a reusable library + +## Purpose + +Make the artifact management functionality available as an importable +library so other projects can reuse it programmatically. + +## Acceptance Criteria + +- Package exposes stable APIs for listing, pushing and cleaning + artifacts. +- CLI continues to operate using those APIs. + +## Prerequisites + +- Codebase currently implemented as command line scripts. + +## Questions + +- Which parts of the API need to remain backwards compatible? + +## Status + +Open diff --git a/issues/0006-ui-queries-overview.md b/issues/0006-ui-queries-overview.md new file mode 100644 index 0000000..c4e4f1a --- /dev/null +++ b/issues/0006-ui-queries-overview.md @@ -0,0 +1,24 @@ +# Extended user interface for queries and overview + +## Purpose + +Improve visibility into stored artifacts with commands that list them +and summarise repository status. + +## Acceptance Criteria + +- CLI can display available artifacts for a commit or branch. +- Users can query upcoming expiries. +- Overview command summarises repository health. + +## Prerequisites + +- Basic list and push commands already implemented. + +## Questions + +- Should the overview command display remote configuration details? + +## Status + +Open diff --git a/issues/0007-python-test-matrix.md b/issues/0007-python-test-matrix.md new file mode 100644 index 0000000..714e725 --- /dev/null +++ b/issues/0007-python-test-matrix.md @@ -0,0 +1,25 @@ +# Python version test matrix + +## Purpose + +Ensure the project works across multiple Python versions by +running automated tests against a matrix of versions. + +## Acceptance Criteria + +- CI configuration runs tests on at least three supported + Python releases. +- Failures on any version block merges. + +## Prerequisites + +- Tests runnable in a clean environment. + +## Questions + +- Which versions should be considered the minimum + supported set? + +## Status + +Open diff --git a/issues/0008-prepare-for-pypi.md b/issues/0008-prepare-for-pypi.md new file mode 100644 index 0000000..060cabe --- /dev/null +++ b/issues/0008-prepare-for-pypi.md @@ -0,0 +1,25 @@ +# Prepare project for PyPI upload + +## Purpose + +Package the code for publication on PyPI so it can be easily +installed via `pip`. + +## Acceptance Criteria + +- Build configuration produces a valid source distribution and + wheel. +- Metadata such as description and license render correctly on + PyPI. + +## Prerequisites + +- Complete versioning strategy and test coverage. + +## Questions + +- Should we use `twine` or another tool for uploading? + +## Status + +Open diff --git a/issues/0009-versioning-strategy.md b/issues/0009-versioning-strategy.md new file mode 100644 index 0000000..634d1f1 --- /dev/null +++ b/issues/0009-versioning-strategy.md @@ -0,0 +1,23 @@ +# Versioning and release strategy + +## Purpose + +Define how the project bumps versions for releases and how +pre-release builds are labelled. + +## Acceptance Criteria + +- Automated process updates version numbers when releasing. +- Strategy documented for patch, minor and major bumps. + +## Prerequisites + +- Continuous integration and packaging tools in place. + +## Questions + +- Should we follow semantic versioning or a simpler scheme? + +## Status + +Open diff --git a/issues/0010-stress-testing.md b/issues/0010-stress-testing.md new file mode 100644 index 0000000..5efe14d --- /dev/null +++ b/issues/0010-stress-testing.md @@ -0,0 +1,24 @@ +# Stress testing under heavy load + +## Purpose + +Evaluate behaviour when many clients push and pull artifacts +in parallel to uncover race conditions in expiration logic. + +## Acceptance Criteria + +- Tests simulate concurrent producers and consumers with + expiring artifacts. +- System remains stable and cleans up expired data correctly. + +## Prerequisites + +- Functioning concurrency in existing commands. + +## Questions + +- How should we coordinate workers to trigger potential races? + +## Status + +Open diff --git a/issues/0011-multi-python-testing.md b/issues/0011-multi-python-testing.md new file mode 100644 index 0000000..18320cb --- /dev/null +++ b/issues/0011-multi-python-testing.md @@ -0,0 +1,18 @@ +# Multi-version Python testing + +## Purpose +Ensure the codebase remains compatible across supported Python +releases by running the test suite against each version. + +## Acceptance Criteria +- CI matrix covers Python 3.8 through 3.11 at minimum. +- All versions pass the same tests without conditional skips. + +## Prerequisites +- Tests rely only on functionality present in all targeted versions. + +## Questions +- When should we remove older Python versions from the matrix? + +## Status +Open diff --git a/issues/0011-sibling-demo-src-bin-projects.md b/issues/0011-sibling-demo-src-bin-projects.md new file mode 100644 index 0000000..076576b --- /dev/null +++ b/issues/0011-sibling-demo-src-bin-projects.md @@ -0,0 +1,23 @@ +# Sibling demo projects for source and binary repos + +## Purpose + +Provide example projects that demonstrate using git-recycle-bin with +separate source and artifact repositories. + +## Acceptance Criteria + +- Two small repositories illustrate pushing and fetching artifacts. +- README files explain how to run the demo end-to-end. + +## Prerequisites + +- Working installation of git-recycle-bin. + +## Questions + +- Should the demos use Makefiles or plain shell scripts? + +## Status + +Open diff --git a/justfile b/justfile index de0f4cf..e609fec 100644 --- a/justfile +++ b/justfile @@ -1,6 +1,6 @@ # Run unit tests unittest: - PYTHONPATH="$PYTHONPATH:$PWD:$PWD/src" pytest + PYTHONPATH="$PYTHONPATH:$PWD:$PWD/src" pytest --cov=src --cov-report=xml # Demonstrate help demo0: @@ -17,3 +17,14 @@ mod push 'demos/push.justfile' mod list 'demos/list.justfile' mod clean 'demos/clean.justfile' mod download 'demos/download.justfile' + +# Lint shell scripts +lint-shell: + find . -name '*.sh' -print0 | xargs -0 shellcheck + +# Lint Markdown files +lint-md: + markdownlint '**/*.md' + +# Run all linters +lint: lint-shell lint-md diff --git a/readme.md b/readme.md index df063c0..919c81a 100644 --- a/readme.md +++ b/readme.md @@ -1,21 +1,32 @@ -# What? -This project provides means for pushing artifacts from a git source repo to a git binary repo. +# Git Recycle Bin ♻️ -Say you/CI build an app; now you can publish your binary to git! Not to your same source code repo, but another *binary* repo. +[![CI](https://github.com/ArtifactLabs/git-recycle-bin/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/ArtifactLabs/git-recycle-bin/actions/workflows/ci.yml) +[![codecov](https://codecov.io/gh/ArtifactLabs/git-recycle-bin/branch/master/graph/badge.svg)](https://codecov.io/gh/ArtifactLabs/git-recycle-bin) +**Use any other git repo as an artifact build cache** 🀯. +With bidirectional traceability πŸŽ‰! +Store build outputs right alongside your source and skip costly rebuilds while +keeping complete traceability. -# Why? -Unlike many artifact management systems out there, the artifacts published here will: -- Have full traceabillity back to their original source code. -- Support expiry and garbage collection - achieved via git gc of orphan branches with absolute expiry date in their ref. -- Prolong expiry (absolute or indefinite) via simple git commands. -- Permit discoverability of available artifacts via `git notes`. - Implemented as described in [issue #1](issues/0001-git-notes-integration.md). +## What is Git Recycle Bin? +Git Recycle Bin uses Git so you can manage build artifacts in a +separate repository while keeping a clear link back to the source that +produced them. Artifacts can be pushed, fetched and automatically +removed when they expire. -# Usage -* Locally or CI-side, this tool creates and pushes artifacts, see `--help` and examples below. -* Garbage collection of expired artifacts is done at server-side. +## Features + +- Push build outputs to a dedicated artifact repository +- Preserve metadata linking artifacts back to the exact source commit +- Garbage collect artifacts using expiry dates +- Create `latest` tags for easy discovery +- Enable skipping builds by downloading pre-built artifacts +- Use git-notes so source repos know which binaries exist +- Operates locally or with any remote Git server using your existing git+ssh + authentication +- Requires no additional setup or deployment of other services so you can use + your existing Git infrastructure ## Documentation Documentation is built with [Sphinx](https://www.sphinx-doc.org/). A @@ -27,69 +38,155 @@ nix-shell shell.nix --pure --run 'just docs-html' The resulting HTML is located under `docs/_build/html` and published from CI. +## Why adopt? + +- 🌱 *Self-governed*: no special server software or enterprise tools + required. Any git host works. +- ♻️ *Reuse binaries*: retrieve previous build artifacts and avoid + unnecessary rebuilds. +- πŸ” *Full traceability*: artifacts are tied to the exact source commit + via git notes. +- πŸ—‘οΈ *Garbage collect*: expired artifacts vanish with `git gc`. + +## Principle of Operation -## Schema -Artifacts come with meta-data, for {expiry, traceabillity, audit, placement} purposes. \ -Meta-data is stored as trailer fields in the commit message, forming a schema, e.g.: +Artifacts are stored in orphan branches using a structured naming +scheme: -* `artifact-schema-version: 1` : Integer. The version of the schema. -* `artifact-name: Aurora-RST-Documentation` : String. Name of the artifact. -* `artifact-mime-type: directory` : String or tuple. MIME type of the artifact. - - One of {[`mimetypes.guess_type()`](https://docs.python.org/3/library/mimetypes.html#mimetypes.guess_type), `directory`, `link`, `mount`, `unknown`}. -* `artifact-tree-prefix: obj/doc/html` : String. Files in this artifact commit all share this directory-prefix. - - Either `.` or some directory-prefix. (A directory-prefix can make merges of artifact-commits conflict-free) -* `src-git-relpath: ../obj/doc/html` : String. Relative path to artifact from source-git's root. - - Leading `../ ` means artifact resided outside the source-git. -* `src-git-commit-title: rf: twister wrapper WIP` : String. Artifact was built from this commit in source git repo. -* `src-git-commit-sha: ed6267ee5b84c894fc8490d93db0525fb2f167eb` : String. Artifact was built from this commit in source git repo. -* `src-git-commit-changeid: Ie3eb7af86c3e578d2c18631f15cf0a12e7d0f80d` : String. Artifact was built from this commit in source git repo. Optional. -* `src-git-commit-time-author: Wed, 21 Jun 2023 14:13:31 +0200` : Date. Artifact was built from this commit in source git repo. -* `src-git-commit-time-commit: Wed, 21 Jun 2023 15:42:49 +0200` : Date. Artifact was built from this commit in source git repo. -* `src-git-branch: feature/wrk_le_audio_7.0` : String. Locally-checked out branch in source git repo or `Detached HEAD`. -* `src-git-repo-name: firmware` : String. Basename of `src-git-repo-url`. -* `src-git-repo-url: ssh://gerrit.ci.demant.com:29418/firmware` : String. URL of remote in source git repo. -* `src-git-commits-ahead: ?`: Integer or `?`. How many commits source git repo branch was locally ahead of its remote upstream tracking branch. -* `src-git-commits-behind: ?`: Integer or `?`. How many commits source git repo branch was locally behind of its remote upstream tracking branch. -* `src-git-status: clean` : String or strings. Either `clean` or list of locally {modified, deleted} files in source git repo. Untracked files are ignored. +```text +artifact/expire/{EXPIRY_DATE}/{SOURCE_REPO}@{SOURCE_SHA}/{ARTIFACT_PATH} +``` -This scheme captures only what is intrinsically tied to the artifact and the sources it comes from. -Adding further meta-data should be carefully considered, so as to not compromise the repeatability/_stabillity_ of the artifact's commit SHA. +`latest` tags point at the most recent artifact for a branch: +```text +artifact/latest/{SOURCE_REPO}@{SOURCE_BRANCH}/{ARTIFACT_PATH} +``` -### Other schema ideas -It may be tempting to extend the schema above with further convenient meta-data, see below for more ideas. -However, adding such _convenient_ meta-data means we mix-in ever-changing non-reproducible machine-specific side-effects - making the SHA _unstable_. -These could still be useful but do no belong in the artifact's commit message; would fit better as a `git-note` attached to the artifact's commitish or treeish -- this remains as possible future work. +Metadata-only refs allow querying +information without downloading the full artifact tree. Git notes on +source commits advertise available artifacts and are used to implement +build avoidance. + +## Installation + +### Using Nix + +```nix +{ pkgs ? import {} }: +pkgs.mkShell { + buildInputs = [ + (pkgs.callPackage (pkgs.fetchFromGitHub { + owner = "artifactLabs"; + repo = "git-recycle-bin"; + rev = ""; + sha256 = ""; + } + "/default.nix") {}) + ]; +} +``` -* `artifact-outputs`: List of files/artifacts generated. Would give more insight into what's included beyond just the tree prefix. -* `artifact-dependencies`: List of other artifacts this one depends on. Useful for dependency management. -* `build-job`: ID of any associated CI job/build pipeline. -* `build-host`: Name/ID of machine that built the artifact. -* `build-duration`: How long the build took. Performance metric. -* `build-timestamp`: Exact UTC timestamp of when build occurred. -* `artifact-hash`: Hash or checksum of the artifact output. Improves integrity checking. -* `artifact-url`: Direct URL to download the artifact. -* `artifact-notes`: Any other information about the build - logs, warnings, etc. +Then enter the shell: +```bash +nix-shell +``` +### Via pip +```bash +pip install git+https://github.com/ArtifactLabs/git-recycle-bin.git +``` +You can also install from a local checkout: -# Usage example 1 +```bash +pip install . ``` -git_recycle_bin.py \ + +## Basic usage + +Push an artifact to a binary repository: + +```bash +git_recycle_bin.py push \ + git@example.com:documentation/generated/rst_html.git \ --path ../obj/doc/html \ - --name "Aurora-RST-Documentation" \ - --remote git@gitlab.ci.demant.com:csfw/documentation/generated/aurora_rst_html_mpeddemo.git \ - --push \ - --push-tag + --name "Example-RST-Documentation" --tag +``` + +Push with expiry: + +```bash +git_recycle_bin.py push . --path build --name demo --expire "in 1 hour" +``` + +Download an artifact back into your working tree: + +```bash +git_recycle_bin.py list . | head -n 1 | \ + xargs -I _ git_recycle_bin.py download . _ ``` -This will: +List artifacts: + +```bash +git_recycle_bin.py list . --name "Example-RST-Documentation" +``` + +Enable build avoidance in your build script by checking for an existing +artifact before building: + +```bash +if git_recycle_bin.py list . --name demo | grep -q .; then + git_recycle_bin.py list . --name demo | head -n 1 | \ + xargs -I _ git_recycle_bin.py download . _ + echo "Build skipped - using downloaded artifact" +else + make all + git_recycle_bin.py push . --path ./build --name demo +fi +``` + +## Advanced usage + +Set a custom expiry date when pushing an artifact: + +```bash +git_recycle_bin.py push . --path ./build --name demo --expire "1 month" +``` + +List all artifacts for the current repository: + +```bash +git_recycle_bin.py list . +``` + +## How it works + +`git-recycle-bin` stores artifacts in dedicated branches and +links them to source commits using git notes. Notes are non-destructive +so you can look up previous binaries and reuse them. Stale artifacts +disappear once expired and a `git gc` runs. CI pipelines can fetch +matching artifacts and skip rebuilding. + +Artifacts include metadata stored as trailer fields in the commit +message. Key fields: + +- `artifact-schema-version` +- `artifact-name` +- `artifact-mime-type` +- `artifact-tree-prefix` +- `src-git-relpath` +- `src-git-commit-sha` +- `src-git-branch` +- `src-git-repo-url` + +For a full schema see [issue #1](issues/0001-git-notes-integration.md). + +## Contributing - 1. Create a new empty git repo locally -- to ensure non-interference with source repo. - 2. Create a new binary artifact git commit for the HTML folder and assign it a {name, expiry} and record meta-data. - 3. Push the artifact to the remote, as an orphan branch named `auto/artifact/firmware@ed6267ee5b84c894fc8490d93db0525fb2f167eb/{obj/doc/html}` - (if the branch already exists, we lost the push-race and leave it alone). - 4. Push {a new, an update of} tag `auto/artifact/firmware@feature/wrk_le_audio_7.0/{obj/doc/html}` - (if the tag already exists, it will be updated if our committer time is more recent). +See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines. Run tests +with `just unittest` and check style with `just lint` +before submitting a +pull request. diff --git a/setup.py b/setup.py index 37cd4ad..268a7cb 100644 --- a/setup.py +++ b/setup.py @@ -2,28 +2,30 @@ from setuptools import setup, find_packages -setup(name='git-recycle-bin', - # Modules to import from other scripts: - packages=find_packages(), - install_requires=[ - 'maya', - 'colorama', - 'dateparser', - ], - # Executables - scripts=[ - "src/git_recycle_bin.py", - "src/rbgit.py", - "src/printer.py", - "src/util_string.py", - "src/util_file.py", - "src/util_date.py", - "src/util_sysinfo.py", - "src/arg_parser.py", - "src/list.py", - "src/download.py", - "src/commit_msg.py", - "src/util.py", - ], - ) +setup( + name="git-recycle-bin", + version="0.2.5", + # Modules to import from other scripts: + packages=find_packages(), + install_requires=[ + "maya", + "colorama", + "dateparser", + ], + # Executables + scripts=[ + "src/git_recycle_bin.py", + "src/rbgit.py", + "src/printer.py", + "src/util_string.py", + "src/util_file.py", + "src/util_date.py", + "src/util_sysinfo.py", + "src/arg_parser.py", + "src/list.py", + "src/download.py", + "src/commit_msg.py", + "src/util.py", + ], +) diff --git a/shell.nix b/shell.nix index bfe58bb..abb5890 100644 --- a/shell.nix +++ b/shell.nix @@ -11,6 +11,8 @@ pkgs.mkShell { packages = [ pkgs.just git-recycle-bin + pkgs.shellcheck + pkgs.nodePackages.markdownlint-cli ]; shellHook = '' export JUST_UNSTABLE=1 diff --git a/src/commit_msg.py b/src/commit_msg.py index 985eae5..95e7496 100644 --- a/src/commit_msg.py +++ b/src/commit_msg.py @@ -36,8 +36,8 @@ def emit_commit_msg(d: dict): """ commit_msg_body = """ - This is a (binary) artifact with expiry. Expiry can be changed. - See https://gitlab.ci.demant.com/csfw/flow/git-recycle-bin#usage + This artifact was published by git-recycle-bin - it may expire! + See https://github.com/ArtifactLabs/git-recycle-bin """ commit_msg_trailers = f""" diff --git a/tests/test_arg_parser.py b/tests/test_arg_parser.py new file mode 100644 index 0000000..9865055 --- /dev/null +++ b/tests/test_arg_parser.py @@ -0,0 +1,49 @@ +import sys +import argparse +import pytest +import arg_parser + + +def run_parse_args(argv): + old = sys.argv + sys.argv = ['prog'] + argv + try: + return arg_parser.parse_args() + finally: + sys.argv = old + + +def test_str2bool(): + assert arg_parser.str2bool('yes') is True + assert arg_parser.str2bool('no') is False + with pytest.raises(argparse.ArgumentTypeError): + arg_parser.str2bool('maybe') + + +def test_tuple1(): + f = arg_parser.tuple1('key') + assert f('val') == ('key', 'val') + + +def test_parse_args_push(): + args = run_parse_args(['push', 'https://example.com', '--path', '/tmp/foo', '--name', 'bar']) + assert args.command == 'push' + assert args.remote == 'https://example.com' + assert args.path == '/tmp/foo' + assert args.name == 'bar' + + +def test_parse_args_force_tag_requires_force_branch(): + res = run_parse_args(['push', 'https://example.com', '--path', '/tmp/foo', '--name', 'bar', '--force-tag']) + assert res is None + + +def test_parse_args_list_name(): + args = run_parse_args(['list', 'https://example.com', '--name', 'foo']) + assert args.command == 'list' + assert args.query == ('name', 'foo') + + +def test_parse_args_missing_remote(): + with pytest.raises(SystemExit): + run_parse_args(['list']) diff --git a/tests/test_commit_msg.py b/tests/test_commit_msg.py new file mode 100644 index 0000000..f993e79 --- /dev/null +++ b/tests/test_commit_msg.py @@ -0,0 +1,62 @@ +import datetime +import re +from commit_msg import extract_gerrit_change_id, parse_commit_msg, emit_commit_msg + + +def test_extract_gerrit_change_id(): + msg = "Change-Id: Iabc\nChange-Id: Idef" + assert extract_gerrit_change_id(msg) == "Idef" + + +def test_parse_commit_msg(): + msg = "key1: val1\nkey2: val2\nignored" + assert parse_commit_msg(msg) == {"key1": "val1", "key2": "val2"} + + +def test_emit_commit_msg_redacts_url(): + d = { + "artifact_name": "artifact", + "src_repo": "repo.git", + "src_sha_short": "abcdef1234", + "src_sha_title": "Commit title", + "artifact_mime": "text/plain", + "artifact_relpath_nca": "path/to", + "artifact_relpath_src": "path/to", + "src_sha": "abcdef1234567890", + "src_sha_msg": "Commit title\n\nChange-Id: Ideadbeef", + "src_time_author": "Thu, 01 Jan 1970 00:00:00 +0000", + "src_time_commit": "Thu, 01 Jan 1970 00:00:00 +0000", + "src_branch": "main", + "src_repo_url": "https://user:pass@host/repo.git", + "src_commits_ahead": "1", + "src_commits_behind": "0", + "src_status": "", + } + msg = emit_commit_msg(d) + assert "artifact: repo.git@abcdef1234: artifact @(Commit title)" in msg + assert "src-git-repo-url: https://user:REDACTED@host/repo.git" in msg + + +def test_emit_commit_msg_truncates_title_and_handles_missing_changeid(): + d = { + 'artifact_name': 'name', + 'src_repo': 'repo.git', + 'src_sha_short': 'abc', + 'src_sha_title': 'T' * 50, + 'artifact_mime': 'text/plain', + 'artifact_relpath_nca': 'p', + 'artifact_relpath_src': 'p', + 'src_sha': 'a' * 40, + 'src_sha_msg': 'Message without id', + 'src_time_author': 'Thu, 01 Jan 1970 00:00:00 +0000', + 'src_time_commit': 'Thu, 01 Jan 1970 00:00:00 +0000', + 'src_branch': 'main', + 'src_repo_url': 'url', + 'src_commits_ahead': '', + 'src_commits_behind': '', + 'src_status': '', + } + msg = emit_commit_msg(d) + truncated = 'T' * 27 + '...' + assert truncated in msg + assert 'src-git-commit-changeid:' not in msg diff --git a/tests/test_download.py b/tests/test_download.py new file mode 100644 index 0000000..d970425 --- /dev/null +++ b/tests/test_download.py @@ -0,0 +1,27 @@ +from types import SimpleNamespace +from download import download_command + +class DummyRbGit: + def __init__(self): + self.calls = [] + def cmd(self, *args): + self.calls.append(args) + if args[0] == 'checkout' and args[-1] == 'fail' and '-f' not in args: + raise RuntimeError('exists') + + +def test_download_force_false_error(capsys): + rbgit = DummyRbGit() + args = SimpleNamespace(artifacts=['fail'], force=False) + ret = download_command(args, rbgit, 'remote') + assert ret == 1 + assert ('fetch', 'remote', 'fail') in rbgit.calls + assert ('checkout', 'fail') in rbgit.calls + + +def test_download_force_true(): + rbgit = DummyRbGit() + args = SimpleNamespace(artifacts=['ok'], force=True) + ret = download_command(args, rbgit, 'remote') + assert ret is None + assert rbgit.calls == [('fetch', 'remote', 'ok'), ('checkout', '-f', 'ok')] diff --git a/tests/test_git_recycle_bin_cmds.py b/tests/test_git_recycle_bin_cmds.py new file mode 100644 index 0000000..8ee3859 --- /dev/null +++ b/tests/test_git_recycle_bin_cmds.py @@ -0,0 +1,167 @@ +import datetime +from types import SimpleNamespace +import git_recycle_bin as grb + + +def test_push_branch_force(monkeypatch): + calls = [] + dummy = SimpleNamespace( + cmd=lambda *a, **k: calls.append(a) + ) + args = SimpleNamespace(force_branch=True) + d = {'bin_branch_name': 'b', 'bin_ref_only_metadata': 'm'} + grb.push_branch(args, d, dummy, 'remote') + assert ('push', '--force', 'remote', 'b') in calls + assert ('push', '--force', 'remote', 'm') in calls + + +def test_push_branch_skip_existing(monkeypatch): + calls = [] + + class Dummy: + def cmd(self, *a, **k): + calls.append(a) + return '' + + def remote_already_has_ref(self, remote, ref): + return ref == 'b' + + dummy = Dummy() + args = SimpleNamespace(force_branch=False) + d = {'bin_branch_name': 'b', 'bin_ref_only_metadata': 'm'} + grb.push_branch(args, d, dummy, 'remote') + assert ('push', 'remote', 'm') in calls + assert ('push', 'remote', 'b') not in calls + + +def test_push_tag_new(monkeypatch): + calls = [] + + class Dummy: + def fetch_current_tag_value(self, r, t): + return None + + def cmd(self, *a, **k): + calls.append(a) + return '' + + dummy = Dummy() + args = SimpleNamespace(force_tag=False) + d = { + 'bin_tag_name': 'tag', + 'src_commits_ahead': '', + 'bin_sha_commit': 'sha', + 'src_time_commit': 'Wed, 21 Jun 2023 12:00:00 +0000', + } + grb.push_tag(args, d, dummy, 'remote') + assert ('push', 'remote', 'tag') in calls + + +def test_push_tag_force_when_newer(monkeypatch): + calls = [] + + class Dummy: + def fetch_current_tag_value(self, r, t): + return 'abc' + + def fetch_cat_pretty(self, r, ref): + return 'src-git-commit-time-commit: Wed, 21 Jun 2023 11:00:00 +0000' + + def cmd(self, *a, **k): + calls.append(a) + return '' + + dummy = Dummy() + args = SimpleNamespace(force_tag=False) + d = { + 'bin_tag_name': 'tag', + 'src_commits_ahead': '', + 'bin_sha_commit': 'ours', + 'src_time_commit': 'Wed, 21 Jun 2023 12:00:00 +0000', + } + grb.push_tag(args, d, dummy, 'remote') + assert ('push', '--force', 'remote', 'tag') in calls + + +def test_remote_delete_expired_branches(monkeypatch): + lines = ( + 'sha1\trefs/heads/artifact/expire/9999-01-01/00.00+0000/foo', + 'sha2\trefs/heads/artifact/expire/2000-01-01/00.00+0000/foo', + ) + calls = [] + + class Dummy: + def cmd(self, *a, **k): + if a[:3] == ('ls-remote', '--heads', 'remote'): + return '\n'.join(lines) + calls.append(a) + return '' + + dummy = Dummy() + grb.remote_delete_expired_branches(dummy, 'remote') + assert ('push', 'remote', '--delete', 'refs/heads/artifact/expire/2000-01-01/00.00+0000/foo') in calls + assert ('push', 'remote', '--delete', 'refs/heads/artifact/expire/9999-01-01/00.00+0000/foo') not in calls + + +def test_remote_flush_meta_for_commit(monkeypatch): + sha1 = 'a' * 40 + sha2 = 'b' * 40 + calls = [] + + class Dummy: + def cmd(self, *a, **k): + if a[:4] == ('ls-remote', '--refs', 'remote', 'refs/artifact/meta-for-commit/*'): + return f'{sha1}\trefs/artifact/meta-for-commit/{sha1}\n{sha2}\trefs/artifact/meta-for-commit/{sha2}' + if a[:3] == ('ls-remote', '--heads', 'remote'): + return f'{sha2}\trefs/heads/main' + if a[:3] == ('ls-remote', '--tags', 'remote'): + return '' + calls.append(a) + return '' + + dummy = Dummy() + grb.remote_flush_meta_for_commit(dummy, 'remote') + assert ('push', 'remote', '--delete', 'refs/artifact/meta-for-commit/' + sha1) in calls + assert all('refs/artifact/meta-for-commit/' + sha2 not in a for a in calls) + + +def test_note_append_push(monkeypatch): + calls = [] + + def fake_exec(cmd, env=None): + calls.append(("exec", cmd, env)) + return '' + + def fake_exec_nostderr(cmd, env=None): + calls.append(("exec_nostderr", cmd, env)) + return '' + + monkeypatch.setattr(grb, 'exec', fake_exec) + monkeypatch.setattr(grb, 'exec_nostderr', fake_exec_nostderr) + monkeypatch.setattr(grb, 'get_user', lambda: 'u') + monkeypatch.setattr(grb, 'get_hostname', lambda: 'h') + + args = SimpleNamespace( + remote='https://remote', + name='artifact', + src_remote_name='origin', + user_name='me', + user_email='me@example.com', + ) + d = { + 'src_status': '', + 'bin_time_commit': 't', + 'bin_branch_expire': 'exp', + 'bin_sha_commit': 'sha', + } + + grb.note_append_push(args, d) + + gitenv = calls[0][2] + expected_ref = grb.sanitize_branch_name( + f"refs/notes/artifact/{grb.sanitize_slashes(args.remote)}/{grb.sanitize_slashes(args.name)}/{d['bin_sha_commit']}-clean" + ) + assert gitenv['GIT_NOTES_REF'] == expected_ref + assert gitenv['GIT_AUTHOR_NAME'] == 'me' + assert any(c[1][:2] == ['git', 'notes'] and 'append' in c[1] for c in calls) + assert any(c[1][:2] == ['git', 'push'] for c in calls) diff --git a/tests/test_list.py b/tests/test_list.py new file mode 100644 index 0000000..aa295bd --- /dev/null +++ b/tests/test_list.py @@ -0,0 +1,39 @@ +from types import SimpleNamespace +import list as list_mod + + +def test_remote_artifacts(monkeypatch): + # patch util.exec to return known sha + monkeypatch.setattr(list_mod, 'exec', lambda cmd: 'abcd') + + def fake_cmd(*args, **kwargs): + if args[:3] == ('ls-remote', '--refs', 'remote'): + return 'm1 refs/artifact/meta-for-commit/abcd/sha1\nm2 refs/artifact/meta-for-commit/abcd/sha2\n' + raise AssertionError('unexpected') + + dummy = SimpleNamespace(cmd=fake_cmd) + res = list_mod.remote_artifacts(dummy, 'remote') + assert res == [('m1', 'sha1'), ('m2', 'sha2')] + + +def test_filter_artifacts_by_name_and_path(monkeypatch): + msgs = { + 'm1': 'artifact-name: foo\nsrc-git-relpath: path1', + 'm2': 'artifact-name: bar\nsrc-git-relpath: path2', + } + + def fake_fetch(remote, ref): + return msgs[ref] + + dummy = SimpleNamespace(fetch_cat_pretty=lambda r, ref: fake_fetch(r, ref)) + artifacts = [('m1', 'sha1'), ('m2', 'sha2')] + + filtered = list_mod.filter_artifacts( + dummy, 'r', 'foo', artifacts, list_mod.filter_funcs['name'] + ) + assert filtered == [('m1', 'sha1')] + + filtered = list_mod.filter_artifacts( + dummy, 'r', 'path2', artifacts, list_mod.filter_funcs['path'] + ) + assert filtered == [('m2', 'sha2')] diff --git a/tests/test_printer.py b/tests/test_printer.py new file mode 100644 index 0000000..5b033d6 --- /dev/null +++ b/tests/test_printer.py @@ -0,0 +1,20 @@ +from io import StringIO +from printer import Printer + + +def test_printer_levels(): + buf = StringIO() + p = Printer(verbosity=2, colorize=False) + p.high_level('A', file=buf) + p.detail('B', file=buf) + p.debug('C', file=buf) + lines = [line.strip() for line in buf.getvalue().splitlines()] + assert lines == ['A', 'B'] + + +def test_strcolor(): + p = Printer(colorize=True) + colored = p.strcolor('\x1b[31m', 'msg') + assert colored.startswith('\x1b[31m') and colored.endswith('\x1b[0m') + p.colorize = False + assert p.strcolor('x', 'msg') == 'msg' diff --git a/tests/test_rbgit.py b/tests/test_rbgit.py index d41de52..0fe8708 100644 --- a/tests/test_rbgit.py +++ b/tests/test_rbgit.py @@ -15,3 +15,125 @@ def test_tree_size_sum(): dummy = DummyRbGit() size = RbGit.tree_size(dummy, "HEAD") assert size == 579 + + +def test_remote_already_has_ref(): + class D: + def cmd(self, *args): + if args[0] == 'ls-remote': + return 'sha\tref' if args[-1] == 'ref' else '' + raise RuntimeError('bad') + + dummy = D() + assert RbGit.remote_already_has_ref(dummy, 'origin', 'ref') is True + assert RbGit.remote_already_has_ref(dummy, 'origin', 'missing') is False + + +def test_fetch_current_tag_value(): + class D: + def cmd(self, *args): + if args[:3] == ('ls-remote', '--tags', 'origin'): + return 'a1\trefs/tags/v1\nb2\trefs/tags/v2' + raise RuntimeError('bad') + + dummy = D() + assert RbGit.fetch_current_tag_value(dummy, 'origin', 'v2') == 'b2' + + +def test_fetch_cat_pretty(): + calls = [] + + class D: + def cmd(self, *args, **kwargs): + calls.append(args) + if args[0] == 'fetch': + return '' + if args[:2] == ('cat-file', '-p'): + return 'content' + raise RuntimeError('bad') + + dummy = D() + res = RbGit.fetch_cat_pretty(dummy, 'origin', 'ref') + assert calls[0] == ('fetch', 'origin', 'ref') + assert res == 'content' + + +def test_add_no_changes(tmp_path): + path = tmp_path / 'file' + path.write_text('data') + + calls = [] + + class D: + def cmd(self, *args, **kwargs): + calls.append(args) + if args[0] == 'diff-index': + return '' + return '' + + dummy = D() + res = RbGit.add(dummy, str(path)) + assert res is False + assert ('add', str(path)) in calls + + +def test_add_changes_force(tmp_path): + path = tmp_path / 'file' + path.write_text('data') + + calls = [] + + class D: + def cmd(self, *args, **kwargs): + calls.append(args) + if args[0] == 'diff-index': + raise RuntimeError('changed') + return '' + + dummy = D() + res = RbGit.add(dummy, str(path), force=True) + assert res is True + assert ('add', '--force', str(path)) in calls + + +def test_add_remote_idempotent_adds(): + calls = [] + + class D: + def cmd(self, *args, **kwargs): + calls.append(args) + return '' + + dummy = D() + RbGit.add_remote_idempotent(dummy, 'origin', 'url') + assert calls == [('remote', 'add', 'origin', 'url')] + + +def test_add_remote_idempotent_set_url(): + calls = [] + + class D: + def cmd(self, *args, **kwargs): + calls.append(args) + if args[:2] == ('remote', 'add'): + raise RuntimeError('exists') + return '' + + dummy = D() + RbGit.add_remote_idempotent(dummy, 'origin', 'url') + assert ('remote', 'set-url', 'origin', 'url') in calls + + +def test_fetch_only_tags_and_set_tag(): + calls = [] + + class D: + def cmd(self, *args, **kwargs): + calls.append(args) + return '' + + dummy = D() + RbGit.fetch_only_tags(dummy, 'origin') + RbGit.set_tag(dummy, 'v1', 'sha') + assert ('fetch', 'origin', 'refs/tags/*:refs/tags/*') in calls + assert ('tag', '--force', 'v1', 'sha') in calls diff --git a/tests/test_util.py b/tests/test_util.py new file mode 100644 index 0000000..71f1791 --- /dev/null +++ b/tests/test_util.py @@ -0,0 +1,31 @@ +import subprocess +from util import exec, exec_nostderr + + +def test_exec_env(monkeypatch): + called = {} + + def fake_check_output(cmd, env=None, text=None, stderr=None): + called['cmd'] = cmd + called['env'] = env + called['stderr'] = stderr + return 'out' + + monkeypatch.setattr(subprocess, 'check_output', fake_check_output) + res = exec(['echo', 'hi'], env={'FOO': 'BAR'}) + assert res == 'out' + assert called['cmd'] == ['echo', 'hi'] + assert called['env']['FOO'] == 'BAR' + assert called['stderr'] is None + + +def test_exec_nostderr(monkeypatch): + called = {} + + def fake_check_output(cmd, env=None, text=None, stderr=None): + called['stderr'] = stderr + return '' + + monkeypatch.setattr(subprocess, 'check_output', fake_check_output) + exec_nostderr(['echo']) + assert called['stderr'] is subprocess.DEVNULL diff --git a/tests/test_util_date.py b/tests/test_util_date.py new file mode 100644 index 0000000..bd7c568 --- /dev/null +++ b/tests/test_util_date.py @@ -0,0 +1,46 @@ +import datetime +from dateutil.tz import tzlocal +from util_date import ( + parse_fuzzy_time, + parse_expire_date, + date_formatted2unix, + format_timespan, + DATE_FMT_GIT, + date_fuzzy2expiryformat, +) + + +def test_parse_fuzzy_time_now(): + dt = parse_fuzzy_time('now') + assert isinstance(dt, datetime.datetime) + + +def test_parse_expire_date(): + s = 'pre/2024-01-02/12.00+0100' + assert parse_expire_date(s, 'pre/') == {'date': '2024-01-02', 'time': '12.00', 'tzoffset': '+0100'} + + +def test_date_formatted2unix(): + s = 'Wed, 21 Jun 2023 14:13:31 +0200' + assert date_formatted2unix(s, DATE_FMT_GIT) == 1687349611 + + +def test_format_timespan(): + a = datetime.datetime(2023,1,1, tzinfo=tzlocal()) + b = datetime.datetime(2023,1,2,3,4, tzinfo=tzlocal()) + assert format_timespan(a, b).strip() == '1days 3h 4m' + + +def test_date_fuzzy2expiryformat_absolute(): + s = '2024-01-02 12:00 UTC' + res = date_fuzzy2expiryformat(s) + parsed = parse_expire_date(res) + assert parsed['date'] == '2024-01-02' + assert parsed['time'] is not None + + +def test_date_fuzzy2expiryformat_relative(): + res = date_fuzzy2expiryformat('in 1 day') + parsed = parse_expire_date(res) + assert parsed['date'] is not None + diff --git a/tests/test_util_file.py b/tests/test_util_file.py new file mode 100644 index 0000000..6ac5c07 --- /dev/null +++ b/tests/test_util_file.py @@ -0,0 +1,18 @@ +import os +from util_file import nca_path, rel_dir, classify_path + + +def test_nca_and_rel(tmp_path): + a = tmp_path / 'a' + b = tmp_path / 'a' / 'b' + b.mkdir(parents=True) + assert nca_path(a, b) == str(a.resolve()) + assert rel_dir(a, b) == 'b' + + +def test_classify_path(tmp_path): + f = tmp_path / 'file.txt' + f.write_text('hi') + assert classify_path(str(f)) == ('text/plain', None) + assert classify_path(str(tmp_path)) == 'directory' + assert classify_path(str(tmp_path / 'missing')) == 'unknown' diff --git a/tests/test_util_string.py b/tests/test_util_string.py new file mode 100644 index 0000000..563beb3 --- /dev/null +++ b/tests/test_util_string.py @@ -0,0 +1,40 @@ +from util_string import ( + remove_empty_lines, + sanitize_slashes, + sanitize_branch_name, + trim_all_lines, + prefix_lines, + string_trunc_ellipsis, +) + + +def test_remove_empty_lines(): + s = 'a\n\n b\n' + assert remove_empty_lines(s) == 'a\n b' + + +def test_sanitize_slashes(): + assert sanitize_slashes('foo/bar') == 'foo_bar' + + +def test_sanitize_branch_name(): + assert sanitize_branch_name('foo bar') == 'foo_bar' + assert sanitize_branch_name('/start') == '_start' + assert sanitize_branch_name('foo..bar') == 'foo.bar' + + +def test_trim_all_lines(): + inp = ' a\n b ' + expected = 'a\nb' + assert trim_all_lines(inp) == expected + + +def test_prefix_lines(): + text = 'a\nb' + assert prefix_lines(text, 'p: ') == 'p: a\np: b' + + +def test_string_trunc_ellipsis(): + long = 'abcdefg' + assert string_trunc_ellipsis(10, long) == long + assert string_trunc_ellipsis(5, long) == 'ab...' diff --git a/tests/test_util_sysinfo.py b/tests/test_util_sysinfo.py new file mode 100644 index 0000000..b147f42 --- /dev/null +++ b/tests/test_util_sysinfo.py @@ -0,0 +1,15 @@ +import os +from util_sysinfo import get_user, get_hostname + + +def test_get_user_env(monkeypatch): + monkeypatch.setenv('USER', 'foo') + assert get_user() == 'foo' + monkeypatch.delenv('USER', raising=False) + monkeypatch.setenv('USERNAME', 'bar') + assert get_user() == 'bar' + + +def test_get_hostname_env(monkeypatch): + monkeypatch.setenv('HOSTNAME', 'host1') + assert get_hostname() == 'host1' From 30982a0da6a6567dbfb30f762181d08913c90ab9 Mon Sep 17 00:00:00 2001 From: Mark Ruvald Pedersen Date: Sat, 14 Jun 2025 01:06:31 +0200 Subject: [PATCH 3/3] Merge master and resolve docs conflicts --- readme.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 919c81a..8222739 100644 --- a/readme.md +++ b/readme.md @@ -2,6 +2,9 @@ [![CI](https://github.com/ArtifactLabs/git-recycle-bin/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/ArtifactLabs/git-recycle-bin/actions/workflows/ci.yml) [![codecov](https://codecov.io/gh/ArtifactLabs/git-recycle-bin/branch/master/graph/badge.svg)](https://codecov.io/gh/ArtifactLabs/git-recycle-bin) +[![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=ArtifactLabs_git-recycle-bin&metric=security_rating)](https://sonarcloud.io/summary/new_code?id=ArtifactLabs_git-recycle-bin) +[![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=ArtifactLabs_git-recycle-bin&metric=sqale_rating)](https://sonarcloud.io/summary/new_code?id=ArtifactLabs_git-recycle-bin) +[![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=ArtifactLabs_git-recycle-bin&metric=reliability_rating)](https://sonarcloud.io/summary/new_code?id=ArtifactLabs_git-recycle-bin) **Use any other git repo as an artifact build cache** 🀯. With bidirectional traceability πŸŽ‰! @@ -56,7 +59,6 @@ scheme: ```text artifact/expire/{EXPIRY_DATE}/{SOURCE_REPO}@{SOURCE_SHA}/{ARTIFACT_PATH} ``` - `latest` tags point at the most recent artifact for a branch: ```text