Skip to content

Commit 68cc7bf

Browse files
sethaxensethaxen
authored
Move R-loo reference tests to their own workflow (#65)
* Move R tests to separate test file * Define custom test environment * Ignore all manifests * Move R reference tests to their own workflow
1 parent f92dace commit 68cc7bf

9 files changed

Lines changed: 155 additions & 117 deletions

File tree

‎.github/workflows/CI.yml‎

Lines changed: 2 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,13 @@ on:
55
- main
66
tags: ['*']
77
pull_request:
8+
89
concurrency:
910
# Skip intermediate builds: always.
1011
# Cancel intermediate builds: only if it is a pull request build.
1112
group: ${{ github.workflow }}-${{ github.ref }}
1213
cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
14+
1315
jobs:
1416
test:
1517
name: ${{ matrix.downgrade && 'Downgrade / ' || '' }}Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
@@ -34,23 +36,8 @@ jobs:
3436
arch: x64
3537
downgrade: true
3638

37-
env:
38-
install_r: ${{ matrix.os == 'ubuntu-latest' }}
39-
4039
steps:
4140
- uses: actions/checkout@v4
42-
- uses: r-lib/actions/setup-r@v2
43-
if: ${{ env.install_r }}
44-
- name: Install R loo
45-
uses: r-lib/actions/setup-r-dependencies@v2
46-
with:
47-
packages: |
48-
github::stan-dev/loo@v2.8.0
49-
dependencies: '"hard"'
50-
if: ${{ env.install_r }}
51-
- name: Set R lib path for RCall.jl
52-
run: echo "LD_LIBRARY_PATH=$(R RHOME)/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
53-
if: ${{ env.install_r }}
5441
- uses: julia-actions/setup-julia@v2
5542
id: setup-julia
5643
with:
@@ -64,13 +51,6 @@ jobs:
6451
if: ${{ matrix.downgrade }}
6552
name: Downgrade dependencies to oldest supported versions
6653
- uses: julia-actions/julia-buildpkg@v1
67-
- name: Build Conda
68-
run: |
69-
using Pkg; Pkg.activate(; temp=true);
70-
Pkg.add("Conda");
71-
Pkg.build("Conda");
72-
shell: julia --color=yes {0}
73-
if: ${{ env.install_r }}
7454
- uses: julia-actions/julia-runtest@v1
7555
- uses: julia-actions/julia-processcoverage@v1
7656
- uses: codecov/codecov-action@v5

‎.github/workflows/r_reference.yml‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
name: R-reference
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
tags: ['*']
8+
pull_request:
9+
10+
concurrency:
11+
# Skip intermediate builds: always.
12+
# Cancel intermediate builds: only if it is a pull request build.
13+
group: ${{ github.workflow }}-${{ github.ref }}
14+
cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
15+
16+
jobs:
17+
test:
18+
name: ${{ github.event_name }}
19+
runs-on: ubuntu-latest
20+
21+
steps:
22+
- uses: actions/checkout@v4
23+
- uses: r-lib/actions/setup-r@v2
24+
- name: Install R loo
25+
uses: r-lib/actions/setup-r-dependencies@v2
26+
with:
27+
packages: |
28+
github::stan-dev/loo@v2.8.0
29+
dependencies: '"hard"'
30+
- name: Set R lib path for RCall.jl
31+
run: echo "LD_LIBRARY_PATH=$(R RHOME)/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
32+
- uses: julia-actions/setup-julia@v2
33+
id: setup-julia
34+
with:
35+
version: '1'
36+
- uses: julia-actions/cache@v2
37+
- name: Build Conda
38+
run: |
39+
using Pkg; Pkg.activate(; temp=true);
40+
Pkg.add("Conda");
41+
Pkg.build("Conda");
42+
shell: julia --color=yes {0}
43+
- name: Install dependencies, build, and run tests
44+
run: |
45+
using Pkg
46+
Pkg.instantiate()
47+
include(joinpath(pwd(), "test", "reference", "runtests.jl"))
48+
shell: julia --color=yes --project=test/reference/ {0}

‎.gitignore‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
*.jl.*.cov
22
*.jl.cov
33
*.jl.mem
4-
/Manifest.toml
5-
/docs/Manifest.toml
4+
Manifest.toml
65
/docs/build/

‎Project.toml‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,6 @@ PDMats = "0.11.11"
5656
PSIS = "0.9.1"
5757
PrettyTables = "2.1"
5858
Printf = "1"
59-
RCall = "0.14.7"
6059
Random = "1"
6160
Roots = "1, 2"
6261
Setfield = "1"
@@ -75,8 +74,7 @@ FiniteDifferences = "26cc04aa-876d-5657-8c51-4c34ba976000"
7574
GLM = "38e38edf-8417-5370-95a0-9cbb8c7f171a"
7675
Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
7776
OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881"
78-
RCall = "6f49c342-dc21-5d91-9882-a32aef131414"
7977
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
8078

8179
[targets]
82-
test = ["ArviZExampleData", "DimensionalData", "FiniteDifferences", "GLM", "Logging", "OffsetArrays", "RCall", "Test"]
80+
test = ["ArviZExampleData", "DimensionalData", "FiniteDifferences", "GLM", "Logging", "OffsetArrays", "Test"]

‎test/helpers.jl‎

Lines changed: 0 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,49 +1,5 @@
11
using ArviZExampleData
22
using IntervalSets
3-
using RCall
4-
5-
r_loo_installed() = !isempty(rcopy(R"system.file(package='loo')"))
6-
7-
# R loo with our API
8-
function loo_r(log_likelihood; reff=nothing)
9-
R"require('loo')"
10-
if reff === nothing
11-
reff = rcopy(R"loo::relative_eff(exp($(log_likelihood)))")
12-
end
13-
result = R"loo::loo($log_likelihood, r_eff=$reff)"
14-
estimates = rcopy(R"$(result)$estimates")
15-
estimates = (
16-
elpd=estimates[1, 1],
17-
se_elpd=estimates[1, 2],
18-
p=estimates[2, 1],
19-
se_p=estimates[2, 2],
20-
)
21-
pointwise = rcopy(R"$(result)$pointwise")
22-
pointwise = (
23-
elpd=pointwise[:, 1],
24-
se_elpd=pointwise[:, 2],
25-
p=pointwise[:, 3],
26-
reff=reff,
27-
pareto_shape=pointwise[:, 5],
28-
)
29-
return (; estimates, pointwise)
30-
end
31-
32-
# R loo with our API
33-
function waic_r(log_likelihood)
34-
R"require('loo')"
35-
result = R"loo::waic($log_likelihood)"
36-
estimates = rcopy(R"$(result)$estimates")
37-
estimates = (
38-
elpd=estimates[1, 1],
39-
se_elpd=estimates[1, 2],
40-
p=estimates[2, 1],
41-
se_p=estimates[2, 2],
42-
)
43-
pointwise = rcopy(R"$(result)$pointwise")
44-
pointwise = (elpd=pointwise[:, 1], p=pointwise[:, 2])
45-
return (; estimates, pointwise)
46-
end
473

484
function log_likelihood_eight_schools(idata)
495
# convert to Array to keep compile times low

‎test/loo.jl‎

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -101,30 +101,4 @@ using Test
101101
(-Inf, 0.5] good 4 (50.0%) 270
102102
(0.5, 0.7] okay 4 (50.0%) 307"""
103103
end
104-
@testset "agrees with R loo" begin
105-
if r_loo_installed()
106-
models = eight_schools_data()
107-
@testset for name in keys(models)
108-
log_likelihood = log_likelihood_eight_schools(models[name])
109-
reff_rand = rand(size(log_likelihood, 3))
110-
@testset for reff in (nothing, reff_rand)
111-
result_r = loo_r(log_likelihood; reff)
112-
result = loo(log_likelihood; reff)
113-
@test result.estimates.elpd ≈ result_r.estimates.elpd
114-
@test result.estimates.se_elpd ≈ result_r.estimates.se_elpd
115-
@test result.estimates.p ≈ result_r.estimates.p
116-
@test result.estimates.se_p ≈ result_r.estimates.se_p
117-
@test result.pointwise.elpd ≈ result_r.pointwise.elpd
118-
# increased tolerance for se_elpd, since we use a different approach
119-
@test result.pointwise.se_elpd ≈ result_r.pointwise.se_elpd rtol = 0.01
120-
@test result.pointwise.p ≈ result_r.pointwise.p
121-
@test result.pointwise.reff ≈ result_r.pointwise.reff
122-
@test result.pointwise.pareto_shape ≈ result_r.pointwise.pareto_shape
123-
end
124-
end
125-
else
126-
@warn "Skipping consistency tests against R loo::loo, since loo is not installed."
127-
@test_broken false
128-
end
129-
end
130104
end

‎test/reference/Project.toml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
[deps]
2+
Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
3+
PosteriorStats = "7f36be82-ad55-44ba-a5c0-b8b5480d7aa5"
4+
RCall = "6f49c342-dc21-5d91-9882-a32aef131414"
5+
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
6+
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
7+
8+
[sources]
9+
PosteriorStats = {path = "../.."}
10+
11+
[compat]
12+
RCall = "0.14.7"

‎test/reference/runtests.jl‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
using Distributions
2+
using PosteriorStats
3+
using Random
4+
using RCall
5+
using Test
6+
7+
# R loo with our API
8+
function loo_r(log_likelihood; reff=nothing)
9+
R"require('loo')"
10+
if reff === nothing
11+
reff = rcopy(R"loo::relative_eff(exp($(log_likelihood)))")
12+
end
13+
result = R"loo::loo($log_likelihood, r_eff=$reff)"
14+
estimates = rcopy(R"$(result)$estimates")
15+
estimates = (
16+
elpd=estimates[1, 1],
17+
se_elpd=estimates[1, 2],
18+
p=estimates[2, 1],
19+
se_p=estimates[2, 2],
20+
)
21+
pointwise = rcopy(R"$(result)$pointwise")
22+
pointwise = (
23+
elpd=pointwise[:, 1],
24+
se_elpd=pointwise[:, 2],
25+
p=pointwise[:, 3],
26+
reff=reff,
27+
pareto_shape=pointwise[:, 5],
28+
)
29+
return (; estimates, pointwise)
30+
end
31+
32+
# R loo with our API
33+
function waic_r(log_likelihood)
34+
R"require('loo')"
35+
result = R"loo::waic($log_likelihood)"
36+
estimates = rcopy(R"$(result)$estimates")
37+
estimates = (
38+
elpd=estimates[1, 1],
39+
se_elpd=estimates[1, 2],
40+
p=estimates[2, 1],
41+
se_p=estimates[2, 2],
42+
)
43+
pointwise = rcopy(R"$(result)$pointwise")
44+
pointwise = (elpd=pointwise[:, 1], p=pointwise[:, 2])
45+
return (; estimates, pointwise)
46+
end
47+
48+
function generate_log_likelihoods(proposal, target, ndraws, nchains, nparams)
49+
draws = rand(proposal, ndraws, nchains, nparams)
50+
log_likelihood = loglikelihood.(target, draws)
51+
return log_likelihood
52+
end
53+
54+
Random.seed!(24)
55+
56+
@testset "Consistency with R loo" begin
57+
proposal = Normal()
58+
target = TDist(7)
59+
60+
@testset "loo" begin
61+
log_likelihood = generate_log_likelihoods(proposal, target, 1000, 4, 10)
62+
reff_rand = rand(size(log_likelihood, 3))
63+
@testset for reff in (nothing, reff_rand)
64+
result_r = loo_r(log_likelihood; reff)
65+
result = loo(log_likelihood; reff)
66+
@test result.estimates.elpd ≈ result_r.estimates.elpd
67+
@test result.estimates.se_elpd ≈ result_r.estimates.se_elpd
68+
@test result.estimates.p ≈ result_r.estimates.p
69+
@test result.estimates.se_p ≈ result_r.estimates.se_p
70+
@test result.pointwise.elpd ≈ result_r.pointwise.elpd
71+
# increased tolerance for se_elpd, since we use a different approach
72+
@test result.pointwise.se_elpd ≈ result_r.pointwise.se_elpd rtol = 0.01
73+
@test result.pointwise.p ≈ result_r.pointwise.p
74+
@test result.pointwise.reff ≈ result_r.pointwise.reff
75+
@test result.pointwise.pareto_shape ≈ result_r.pointwise.pareto_shape
76+
end
77+
end
78+
79+
@testset "waic" begin
80+
log_likelihood = generate_log_likelihoods(proposal, target, 1000, 4, 10)
81+
reff_rand = rand(size(log_likelihood, 3))
82+
result_r = waic_r(log_likelihood)
83+
result = waic(log_likelihood)
84+
@test result.estimates.elpd ≈ result_r.estimates.elpd
85+
@test result.estimates.se_elpd ≈ result_r.estimates.se_elpd
86+
@test result.estimates.p ≈ result_r.estimates.p
87+
@test result.estimates.se_p ≈ result_r.estimates.se_p
88+
@test result.pointwise.elpd ≈ result_r.pointwise.elpd
89+
@test result.pointwise.p ≈ result_r.pointwise.p
90+
end
91+
end

‎test/waic.jl‎

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -62,24 +62,4 @@ using Test
6262
elpd se_elpd p se_p
6363
-31 1.4 0.9 0.32"""
6464
end
65-
@testset "agrees with R waic" begin
66-
if r_loo_installed()
67-
models = eight_schools_data()
68-
@testset for name in keys(models)
69-
log_likelihood = log_likelihood_eight_schools(models[name])
70-
reff_rand = rand(size(log_likelihood, 3))
71-
result_r = waic_r(log_likelihood)
72-
result = waic(log_likelihood)
73-
@test result.estimates.elpd ≈ result_r.estimates.elpd
74-
@test result.estimates.se_elpd ≈ result_r.estimates.se_elpd
75-
@test result.estimates.p ≈ result_r.estimates.p
76-
@test result.estimates.se_p ≈ result_r.estimates.se_p
77-
@test result.pointwise.elpd ≈ result_r.pointwise.elpd
78-
@test result.pointwise.p ≈ result_r.pointwise.p
79-
end
80-
else
81-
@warn "Skipping consistency tests against R loo::waic, since loo is not installed."
82-
@test_broken false
83-
end
84-
end
8565
end

0 commit comments

Comments
 (0)