From 09b4edbdc54b98a748c2c7c0963a3d0f03b48438 Mon Sep 17 00:00:00 2001 From: Ivor Wanders Date: Tue, 19 Apr 2022 15:14:18 -0400 Subject: [PATCH 1/4] Add the build steps to the build//api/get-info endpoint. --- src/lib/Hydra/Controller/Build.pm | 40 +++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index 552f31af6..4c4d23fdb 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -6,6 +6,7 @@ use warnings; use base 'Hydra::Base::Controller::NixChannel'; use Hydra::Helper::Nix; use Hydra::Helper::CatalystUtils; +use Hydra::Controller::API; use File::Basename; use File::stat; use Data::Dump qw(dump); @@ -569,13 +570,52 @@ sub bump : Chained('buildChain') PathPart('bump') { } +sub buildStepToHash { + my ($buildstep) = @_; + return { + build => $buildstep->get_column('build'), + busy => $buildstep->busy, + drvpath => $buildstep->drvpath, + errormsg => $buildstep->errormsg, + isnondeterministic => $buildstep->isnondeterministic, + machine => $buildstep->machine, + overhead => $buildstep->overhead, + # The propagatedfrom field will hold a Build type if it was propagated, we'd like to display that info, so we + # convert the that record to a hash here and inline it, we already have the data on hand and it saves clients a + # request to obtain the actual reason why something happened. + propagatedfrom => defined($buildstep->propagatedfrom) ? Hydra::Controller::API::buildToHash($buildstep->propagatedfrom) : undef, + starttime => $buildstep->starttime, + status => $buildstep->status, + stepnr => $buildstep->stepnr, + stoptime => $buildstep->stoptime, + system => $buildstep->system, + timesbuilt => $buildstep->timesbuilt, + type => $buildstep->type, + haserrormsg => defined($buildstep->errormsg) && $buildstep->errormsg ne "" ? JSON::MaybeXS::true : JSON::MaybeXS::false + }; +} + + sub get_info : Chained('buildChain') PathPart('api/get-info') Args(0) { my ($self, $c) = @_; my $build = $c->stash->{build}; + + # Since this is the detailed info of the build, lets start with populating it with + # the info we can obtain form the build. + $c->stash->{json} = Hydra::Controller::API::buildToHash($build); + + # Provide the original get-info endpoint attributes. $c->stash->{json}->{buildId} = $build->id; $c->stash->{json}->{drvPath} = $build->drvpath; my $out = getMainOutput($build); $c->stash->{json}->{outPath} = $out->path if defined $out; + + # Finally, provide information about all the buildsteps that made up this build. + my @buildsteps = $build->buildsteps->search({}, {order_by => "stepnr asc"}); + my @buildsteplist; + push @buildsteplist, buildStepToHash($_) foreach @buildsteps; + $c->stash->{json}->{steps} = \@buildsteplist; + $c->forward('View::JSON'); } From 82a4f6518a1a53e72a7138c90dea5967e96e1e81 Mon Sep 17 00:00:00 2001 From: Ivor Wanders Date: Wed, 20 Apr 2022 10:26:51 -0400 Subject: [PATCH 2/4] Add the /api/get-info endpoint to the api spec. --- hydra-api.yaml | 213 +++++++++++++++++++++++++++--- src/lib/Hydra/Controller/API.pm | 4 +- src/lib/Hydra/Controller/Build.pm | 2 +- 3 files changed, 201 insertions(+), 18 deletions(-) diff --git a/hydra-api.yaml b/hydra-api.yaml index ce7e0f9a9..167578f37 100644 --- a/hydra-api.yaml +++ b/hydra-api.yaml @@ -507,6 +507,33 @@ paths: schema: $ref: '#/components/schemas/Error' + /build/{build-id}/api/get-info: + get: + summary: Retrieves info about this build id, providing information about the steps in it. + parameters: + - name: build-id + in: path + description: build identifier + required: true + schema: + type: integer + responses: + '200': + description: build + content: + application/json: + schema: + $ref: '#/components/schemas/BuildInfo' + examples: + build-success: + $ref: '#/components/examples/build-get-info-success' + '404': + description: build couldn't be found + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + /build/{build-id}/constituents: get: summary: Retrieves a build's constituent jobs @@ -673,7 +700,7 @@ components: nullable: true description: the path to the file to evaluate type: string - errormsg: + errormsg: &schema_errormsg nullable: true description: contains the stderr output of the nix-instantiate command type: string @@ -813,7 +840,7 @@ components: checkinterval: description: interval in seconds at which to check the jobset inputs type: integer - haserrormsg: + haserrormsg: &schema_haserrormsg description: true if the evaluation had errors type: boolean nrscheduled: @@ -896,18 +923,18 @@ components: description: The nix store path type: string - Build: + Build: type: object properties: id: type: integer - starttime: + starttime: &schema_starttime description: time when build started type: integer - stoptime: + stoptime: &schema_stoptime description: time when build ended type: integer - timestamp: + timestamp: &schema_timestamp description: time when the build was first created type: integer jobsetevals: @@ -915,47 +942,49 @@ components: type: array items: type: integer - finished: + finished: &schema_finished description: true when the build is finished type: boolean - nixname: + nixname: &schema_nixname description: name from the build's derivation type: string - buildstatus: + buildstatus: &schema_status nullable: true # should only be null if finished is false description: | Indicates the build status:
  • 0 : succeeded
  • 1 : failed
    • -
  • 2 : dependency failed
    • +
  • 2 : dependency failed (builds only)
  • 3 : aborted
  • 4 : canceled by the user
    • -
  • 6 : failed with output
    • +
  • 6 : failed with output (builds only)
  • 7 : timed out
    • +
  • 8 : cached failure (steps only)
  • 9 : aborted
  • 10 : log size limit exceeded
  • 11 : output size limit exceeded
    • +
  • 12 : not deterministic
  • * : failed
Note:buildstatus should only be `null` if `finished` is false. type: integer - jobset: + jobset: &schema_jobset description: jobset this build belongs to type: string priority: description: determines the priority with which this build will be executed (higher value means higher priority) type: integer - job: + job: &schema_job description: nix attribute from the nixexprpath type: string - drvpath: + drvpath: &schema_drvpath description: filename of the drv type: string - system: + system: &schema_system description: system this build was done for type: string - project: + project: &schema_project description: project this build belongs to type: string buildproducts: @@ -986,6 +1015,99 @@ components: unit: type: string description: unit of the measured build metric + BuildInfo: + properties: + id: + description: The build id. + type: integer + buildId: + description: Same as id, exists for backwards compatibility. + type: integer + finished: + <<: *schema_finished + steps: + description: List of steps that make up this build. + type: array + items: + type: object + additionalProperties: + $ref: '#/components/schemas/BuildStep' + job: + <<: *schema_job + system: + <<: *schema_system + buildstatus: + <<: *schema_status + jobset: + <<: *schema_jobset + project: + <<: *schema_project + timestamp: + <<: *schema_timestamp + nixname: + <<: *schema_nixname + drvPath: + description: Same as drvpath, exists for backwards compatibility. + type: string + outPath: + description: The nix store path for the output of this build. + type: string + + BuildStep: + type: object + properties: + stepnr: + description: The step number, this number can be used when retrieving logs for a particular step. + type: integer + starttime: + <<: *schema_starttime + stoptime: + <<: *schema_stoptime + status: + <<: *schema_status + system: + <<: *schema_system + propagatedfrom: + description: Null if the result was not propagated. + nullable: true + type: object + additionalProperties: + $ref: '#/components/schemas/Build' + errormsg: + <<: *schema_errormsg + haserrormsg: + <<: *schema_haserrormsg + machine: + description: Machine on which this step was executed. + type: string + busy: + description: | + Indicates the current stage this build step is in:
+
    +
  • 0 : not being worked on (status populated)
    • +
  • 1 : preparing
    • +
  • 10 : connecting
    • +
  • 20 : sending inputs
    • +
  • 30 : building
    • +
  • 40 : receiving outputs
    • +
  • 50 : post processing
    • +
  • * : unknown
    • +
+ type: integer + drvpath: + <<: *schema_drvpath + build: + description: Id of the build this step is a part of. + type: integer + timesbuilt: + nullable: true + type: integer + type: + nullable: true + overhead: + nullable: true + isnondeterministic: + nullable: true examples: projects-success: @@ -1110,3 +1232,62 @@ components: project: example-hello starttime: 1588365711 timestamp: 1588365711 + + build-get-info-success: + value: + id: 5 + buildstatus: 2 # actually a failed build, to show the propagated error. + nixname: hello-2.10 + finished: 1 + timestamp: 1588365711 + system: x86_64-linux + job: hello + jobset: hello + project: example-hello + steps: + - + timesbuilt: null + errormsg: null + haserrormsg: false + overhead: null + type: 0 + drvpath: /nix/store/ab9zv2y5gm8hr6g318p0s6kaclwj4slr-hello-2.10.drv + build: 1 + machine: localhost + busy: 1 + status: 4 + isnondeterministic: null + starttime: 1649950241 + propagatedfrom: null + system: x86_64-linux + stepnr: 1 + stoptime: 1649950294 + - + timesbuilt: null + errormsg: null + haserrormsg: false + overhead: null + type: 0 + drvpath: /nix/store/hhl1dql437h1glbqv43629f1pmnlcfoo-package.drv + build: 1 + machine: localhost + busy: 0 + status: 8 # Cached failure, will have propagated result. + isnondeterministic: null + starttime: 1649950241 + propagatedfrom: + jobset: hello + nixname: hello + system: x86_64-linux + job: sdk.x86_64-linux + project: hello + id: 3 # since this is a build, this denotes the build id the propagation result came from. + timestamp: 1649456722 + finished: 1 + + system: x86_64-linux + stepnr: 2 + stoptime: 1649950294 + buildId: 5 + drvPath: /nix/store/ab9zv2y5gm8hr6g318p0s6kaclwj4slr-hello-2.10.drv + outPath: /nix/store/y26qxcq1gg2hrqpxdc58b2fghv2bhxjg-hello-2.10 diff --git a/src/lib/Hydra/Controller/API.pm b/src/lib/Hydra/Controller/API.pm index 6f10ef575..3ae3bb4d2 100644 --- a/src/lib/Hydra/Controller/API.pm +++ b/src/lib/Hydra/Controller/API.pm @@ -30,7 +30,9 @@ sub buildToHash { system => $build->system, nixname => $build->nixname, finished => $build->finished, - timestamp => $build->timestamp + timestamp => $build->timestamp, + buildstatus => undef, + priority => undef }; if($build->finished) { diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index 4c4d23fdb..e5b9c43f3 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -580,7 +580,7 @@ sub buildStepToHash { isnondeterministic => $buildstep->isnondeterministic, machine => $buildstep->machine, overhead => $buildstep->overhead, - # The propagatedfrom field will hold a Build type if it was propagated, we'd like to display that info, so we + # The propagated from field will hold a Build type if it was propagated, we'd like to display that info, so we # convert the that record to a hash here and inline it, we already have the data on hand and it saves clients a # request to obtain the actual reason why something happened. propagatedfrom => defined($buildstep->propagatedfrom) ? Hydra::Controller::API::buildToHash($buildstep->propagatedfrom) : undef, From 8423bff765718c16534bf955dac3ee51eff91c91 Mon Sep 17 00:00:00 2001 From: Ivor Wanders Date: Fri, 12 Aug 2022 09:41:03 -0400 Subject: [PATCH 3/4] Write api reference differently. --- hydra-api.yaml | 45 +++++++++++++++------------------------------ 1 file changed, 15 insertions(+), 30 deletions(-) diff --git a/hydra-api.yaml b/hydra-api.yaml index 167578f37..9ab6ca8e4 100644 --- a/hydra-api.yaml +++ b/hydra-api.yaml @@ -1023,8 +1023,7 @@ components: buildId: description: Same as id, exists for backwards compatibility. type: integer - finished: - <<: *schema_finished + finished: *schema_finished steps: description: List of steps that make up this build. type: array @@ -1032,20 +1031,13 @@ components: type: object additionalProperties: $ref: '#/components/schemas/BuildStep' - job: - <<: *schema_job - system: - <<: *schema_system - buildstatus: - <<: *schema_status - jobset: - <<: *schema_jobset - project: - <<: *schema_project - timestamp: - <<: *schema_timestamp - nixname: - <<: *schema_nixname + job: *schema_job + system: *schema_system + buildstatus: *schema_status + jobset: *schema_jobset + project: *schema_project + timestamp: *schema_timestamp + nixname: *schema_nixname drvPath: description: Same as drvpath, exists for backwards compatibility. type: string @@ -1059,24 +1051,18 @@ components: stepnr: description: The step number, this number can be used when retrieving logs for a particular step. type: integer - starttime: - <<: *schema_starttime - stoptime: - <<: *schema_stoptime - status: - <<: *schema_status - system: - <<: *schema_system + starttime: *schema_starttime + stoptime: *schema_stoptime + status: *schema_status + system: *schema_system propagatedfrom: description: Null if the result was not propagated. nullable: true type: object additionalProperties: $ref: '#/components/schemas/Build' - errormsg: - <<: *schema_errormsg - haserrormsg: - <<: *schema_haserrormsg + errormsg: *schema_errormsg + haserrormsg: *schema_haserrormsg machine: description: Machine on which this step was executed. type: string @@ -1094,8 +1080,7 @@ components:
  • * : unknown
    • type: integer - drvpath: - <<: *schema_drvpath + drvpath: *schema_drvpath build: description: Id of the build this step is a part of. type: integer From 158e8561aec977ef89b230f98c12fbd598149605 Mon Sep 17 00:00:00 2001 From: Ivor Wanders Date: Fri, 12 Aug 2022 12:45:54 -0400 Subject: [PATCH 4/4] Add unit test for get-info build endpoint. --- src/lib/Hydra/Controller/Build.pm | 2 +- t/Hydra/Controller/Build/api.t | 43 +++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index e5b9c43f3..cc8f833e1 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -569,7 +569,7 @@ sub bump : Chained('buildChain') PathPart('bump') { $c->res->redirect($c->uri_for($self->action_for("build"), $c->req->captures)); } - +# This function is reused in the unit test. sub buildStepToHash { my ($buildstep) = @_; return { diff --git a/t/Hydra/Controller/Build/api.t b/t/Hydra/Controller/Build/api.t index 91a553df9..e66de8bcf 100644 --- a/t/Hydra/Controller/Build/api.t +++ b/t/Hydra/Controller/Build/api.t @@ -102,4 +102,47 @@ subtest "accessing the constituents API" => sub { is($buildB->{job}, "b"); }; +subtest "accessing the get-info API" => sub { + my $url = $build_url . "/api/get-info"; + + my $build_info = request(GET $url, + Accept => 'application/json', + ); + + ok($build_info->is_success, "Getting the build info"); + + my $data; + my $valid_json = lives { $data = decode_json($build_info->content); }; + ok($valid_json, "We get back valid JSON."); + if (!$valid_json) { + use Data::Dumper; + print STDERR Dumper $build_info->content; + } + + # Query the build steps from the aggregate build and create the expected list of results. + my @buildsteps = $aggregateBuild->buildsteps->search({}, {order_by => "stepnr asc"}); + my @buildsteplist; + push @buildsteplist, Hydra::Controller::Build::buildStepToHash($_) foreach @buildsteps; + + # Create the expected output with the build step information in it. + my $expected = { + project => "tests", + jobset => "aggregate", + outPath => $aggregateBuild->buildoutputs->find({ name => "out" })->path, + buildstatus => 0, + drvPath => $aggregateBuild->drvpath, + buildId => $aggregateBuild->id, + priority => undef, + finished => 1, + id => $aggregateBuild->id, + job => "aggregate", + nixname => "aggregate", + timestamp => $aggregateBuild->timestamp, + system => $aggregateBuild->system, + steps => \@buildsteplist, + }; + + is($data, $expected, "The build's info JSON matches our API."); +}; + done_testing;