From 4c8a8401c558b933f0eeb170496687b30545549a Mon Sep 17 00:00:00 2001 From: Hank Donnay Date: Thu, 26 Mar 2026 12:13:07 -0500 Subject: [PATCH 1/4] units: helper for consistent unit usage This establishes a central package to keep metric units consistent. Signed-off-by: Hank Donnay Change-Id: I3821b242289d35e1ab4f8cf38f8063226a6a6964 JJ: See-Also: CLAIRDEV-NNNN JJ: Closes: #NNNN --- go.mod | 2 +- internal/units/doc.go | 25 ++ internal/units/histogram.go | 72 +++++ internal/units/histogram_test.go | 37 +++ internal/units/testdata/ucum-cs.units | 364 ++++++++++++++++++++++++++ internal/units/units.go | 18 ++ internal/units/units_test.go | 113 ++++++++ 7 files changed, 630 insertions(+), 1 deletion(-) create mode 100644 internal/units/doc.go create mode 100644 internal/units/histogram.go create mode 100644 internal/units/histogram_test.go create mode 100644 internal/units/testdata/ucum-cs.units create mode 100644 internal/units/units.go create mode 100644 internal/units/units_test.go diff --git a/go.mod b/go.mod index a91796a9c..cf97bee9f 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/spdx/tools-golang v0.5.7 github.com/ulikunitz/xz v0.5.15 go.opentelemetry.io/otel v1.42.0 + go.opentelemetry.io/otel/metric v1.42.0 go.opentelemetry.io/otel/trace v1.42.0 go.uber.org/mock v0.6.0 golang.org/x/crypto v0.48.0 @@ -52,7 +53,6 @@ require ( github.com/prometheus/procfs v0.16.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/otel/metric v1.42.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect golang.org/x/mod v0.33.0 // indirect google.golang.org/protobuf v1.36.8 // indirect diff --git a/internal/units/doc.go b/internal/units/doc.go new file mode 100644 index 000000000..c749771ed --- /dev/null +++ b/internal/units/doc.go @@ -0,0 +1,25 @@ +// Package units provides a common spot for [OTel units]. +// +// Units should follow the [Unified Code for Units of Measure] (UCUM). +// +// - Instruments for utilization metrics (that measure the fraction out of a +// total) are dimensionless and SHOULD use the default unit 1 (the unity). +// - All non-units that use curly braces to annotate a quantity need to match +// the grammatical number of the quantity it represent. For example, if +// measuring the number of individual requests to a process the unit would be +// "{request}", not "{requests}". +// - Instruments that measure an integer count of something SHOULD only use +// annotations with curly braces to give additional meaning without the +// leading default unit (1). For example, use "{packet}", "{error}", +// "{fault}", etc. +// - Instrument units other than 1 and those that use annotations SHOULD be +// specified using the UCUM case sensitive ("c/s") variant. For example, "Cel" +// for the unit with full name "degree Celsius". +// - Instruments SHOULD use non-prefixed units (i.e. "By" instead of "MiBy") +// unless there is good technical reason to not do so. +// - When instruments are measuring durations, seconds (i.e. "s") SHOULD be +// used. +// +// [Unified Code for Units of Measure]: https://ucum.org/ucum +// [OTel units]: https://opentelemetry.io/docs/specs/semconv/general/metrics/#instrument-units +package units diff --git a/internal/units/histogram.go b/internal/units/histogram.go new file mode 100644 index 000000000..601c2107a --- /dev/null +++ b/internal/units/histogram.go @@ -0,0 +1,72 @@ +package units + +import ( + "math/big" + "slices" + + "go.opentelemetry.io/otel/metric" +) + +// Buckets as suggested for [request durations]. +// +// [request durations]: https://opentelemetry.io/docs/specs/semconv/http/http-metrics/#metric-httpserverrequestduration +var Buckets metric.HistogramOption + +// LargeBuckets is a 10x multiple of [Buckets]. +var LargeBuckets metric.HistogramOption + +// VeryLargeBuckets is a 20x multiple [Buckets]. +var VeryLargeBuckets metric.HistogramOption + +func init() { + Buckets = metric.WithExplicitBucketBoundaries(BucketBoundaries(0.005, 14)...) + LargeBuckets = metric.WithExplicitBucketBoundaries(BucketBoundaries(0.05, 14)...) + VeryLargeBuckets = metric.WithExplicitBucketBoundaries(BucketBoundaries(0.1, 14)...) +} + +// BucketBoundaries returns "count" bucket boundaries in the same pattern as the +// semconv suggested bucket boundaries. +func BucketBoundaries(start float64, count int) []float64 { + // This uses [big.Rat], which is probably not strictly necessary, but avoids + // needing to do rounding shenanigans. The overhead of the math is also just + // paid once at setup. + ten := big.NewRat(10, 1) + rat := big.NewRat(1, 4) + steps := []*big.Rat{ + big.NewRat(1, 1), + big.NewRat(2, 1), + big.NewRat(3, 1), + } + + n := new(big.Rat).SetFloat64(start) + seq := func(yield func(float64) bool) { + // Yield wrapper: convert the [big.Rat] and check the number of values + // we're supposed to produce. + y := func(n *big.Rat) bool { + v, _ := n.Float64() + count-- + return yield(v) && count > 0 + } + if !y(n) { + return + } + n.Mul(n, big.NewRat(2, 1)) + v, incr := new(big.Rat), new(big.Rat) + for { + if !y(n) { + return + } + + n.Mul(n, ten) + incr.Mul(n, rat) + + for _, step := range steps { + v.Mul(incr, step) + if !y(v) { + return + } + } + } + } + return slices.Collect(seq) +} diff --git a/internal/units/histogram_test.go b/internal/units/histogram_test.go new file mode 100644 index 000000000..81a90bd65 --- /dev/null +++ b/internal/units/histogram_test.go @@ -0,0 +1,37 @@ +package units + +import ( + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestBucketSeq(t *testing.T) { + tt := []struct { + Name string + Want []float64 + }{ + { + "Buckets", + []float64{0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10}, + }, + { + "LargeBuckets", + []float64{0.05, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 25, 50, 75, 100}, + }, + { + "VeryLargeBuckets", + []float64{0.1, 0.2, 0.5, 1, 1.5, 2, 5, 10, 15, 20, 50, 100, 150, 200}, + }, + } + for _, tc := range tt { + t.Run(tc.Name, func(t *testing.T) { + want := tc.Want + got := BucketBoundaries(want[0], len(want)) + + if !cmp.Equal(got, want) { + t.Error(cmp.Diff(got, want)) + } + }) + } +} diff --git a/internal/units/testdata/ucum-cs.units b/internal/units/testdata/ucum-cs.units new file mode 100644 index 000000000..2d4a3d4f3 --- /dev/null +++ b/internal/units/testdata/ucum-cs.units @@ -0,0 +1,364 @@ +# +# Copyright (c) 1998-2024 Regenstrief Institute. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +# +####################################################################### +# +# The Unified Code for Units of Measure (UCUM) +# +# This file is generated automatically. Please refer to the original +# UCUM specification at +# +# https://ucum.org +# +case sensitive +prefix Y 1e24 +prefix Z 1e21 +prefix E 1e18 +prefix P 1e15 +prefix T 1e12 +prefix G 1e9 +prefix M 1e6 +prefix k 1e3 +prefix h 1e2 +prefix da 1e1 +prefix d 1e-1 +prefix c 1e-2 +prefix m 1e-3 +prefix u 1e-6 +prefix n 1e-9 +prefix p 1e-12 +prefix f 1e-15 +prefix a 1e-18 +prefix z 1e-21 +prefix y 1e-24 +prefix Ki 1024 +prefix Mi 1048576 +prefix Gi 1073741824 +prefix Ti 1099511627776 +dimensions 7 +base m +base s +base g +base rad +base K +base C +base cd +10* = 10 1 nonmetric # the number ten for arbitrary powers +10^ = 10 1 nonmetric # the number ten for arbitrary powers +[pi] = 3.1415926535897932384626433832795028841971693993751058209749445923 1 nonmetric # the number pi +% = 1 10*-2 nonmetric # percent +[ppth] = 1 10*-3 nonmetric # parts per thousand +[ppm] = 1 10*-6 nonmetric # parts per million +[ppb] = 1 10*-9 nonmetric # parts per billion +[pptr] = 1 10*-12 nonmetric # parts per trillion +mol = 6.02214076 10*23 metric # mole +sr = 1 rad2 metric # steradian +Hz = 1 s-1 metric # hertz +N = 1 kg.m/s2 metric # newton +Pa = 1 N/m2 metric # pascal +J = 1 N.m metric # joule +W = 1 J/s metric # watt +A = 1 C/s metric # ampère +V = 1 J/C metric # volt +F = 1 C/V metric # farad +Ohm = 1 V/A metric # ohm +S = 1 Ohm-1 metric # siemens +Wb = 1 V.s metric # weber +Cel = cel(1 K) metric # degree Celsius +T = 1 Wb/m2 metric # tesla +H = 1 Wb/A metric # henry +lm = 1 cd.sr metric # lumen +lx = 1 lm/m2 metric # lux +Bq = 1 s-1 metric # becquerel +Gy = 1 J/kg metric # gray +Sv = 1 J/kg metric # sievert +gon = 0.9 deg nonmetric # gon +deg = 2 [pi].rad/360 nonmetric # degree +' = 1 deg/60 nonmetric # minute +'' = 1 '/60 nonmetric # second +l = 1 dm3 metric # liter +L = 1 l metric # liter +ar = 100 m2 metric # are +min = 60 s nonmetric # minute +h = 60 min nonmetric # hour +d = 24 h nonmetric # day +a_t = 365.24219 d nonmetric # tropical year +a_j = 365.25 d nonmetric # mean Julian year +a_g = 365.2425 d nonmetric # mean Gregorian year +a = 1 a_j nonmetric # year +wk = 7 d nonmetric # week +mo_s = 29.53059 d nonmetric # synodal month +mo_j = 1 a_j/12 nonmetric # mean Julian month +mo_g = 1 a_g/12 nonmetric # mean Gregorian month +mo = 1 mo_j nonmetric # month +t = 1e3 kg metric # tonne +bar = 1e5 Pa metric # bar +u = 1.66053906660e-24 g metric # unified atomic mass unit +eV = 1 [e].V metric # electronvolt +AU = 149597.870691 Mm nonmetric # astronomic unit +pc = 3.085678e16 m metric # parsec +[c] = 299792458 m/s metric # velocity of light +[h] = 6.62607015e-34 J.s metric # Planck constant +[k] = 1.380649e-23 J/K metric # Boltzmann constant +[eps_0] = 8.854187817e-12 F/m metric # permittivity of vacuum +[mu_0] = 1 4.[pi].10*-7.N/A2 metric # permeability of vacuum +[e] = 1.602176634e-19 C metric # elementary charge +[m_e] = 9.1093837139e-31 kg metric # electron mass +[m_p] = 1.67262192595e-27 kg metric # proton mass +[G] = 6.67430e-11 m3.kg-1.s-2 metric # Newtonian constant of gravitation +[g] = 980665e-5 m/s2 metric # standard acceleration of free fall +atm = 101325 Pa nonmetric # standard atmosphere +[ly] = 1 [c].a_j metric # light-year +gf = 1 g.[g] metric # gram-force +[lbf_av] = 1 [lb_av].[g] nonmetric # pound force +Ky = 1 cm-1 metric # Kayser +Gal = 1 cm/s2 metric # Gal +dyn = 1 g.cm/s2 metric # dyne +erg = 1 dyn.cm metric # erg +P = 1 dyn.s/cm2 metric # Poise +Bi = 10 A metric # Biot +St = 1 cm2/s metric # Stokes +Mx = 1e-8 Wb metric # Maxwell +G = 1e-4 T metric # Gauss +Oe = 250 /[pi].A/m metric # Oersted +Gb = 1 Oe.cm metric # Gilbert +sb = 1 cd/cm2 metric # stilb +Lmb = 1 cd/cm2/[pi] metric # Lambert +ph = 1e-4 lx metric # phot +Ci = 37e9 Bq metric # Curie +R = 2.58e-4 C/kg metric # Roentgen +RAD = 100 erg/g metric # radiation absorbed dose +REM = 1 RAD metric # radiation equivalent man +[in_i] = 254e-2 cm nonmetric # inch +[ft_i] = 12 [in_i] nonmetric # foot +[yd_i] = 3 [ft_i] nonmetric # yard +[mi_i] = 5280 [ft_i] nonmetric # mile +[fth_i] = 6 [ft_i] nonmetric # fathom +[nmi_i] = 1852 m nonmetric # nautical mile +[kn_i] = 1 [nmi_i]/h nonmetric # knot +[sin_i] = 1 [in_i]2 nonmetric # square inch +[sft_i] = 1 [ft_i]2 nonmetric # square foot +[syd_i] = 1 [yd_i]2 nonmetric # square yard +[cin_i] = 1 [in_i]3 nonmetric # cubic inch +[cft_i] = 1 [ft_i]3 nonmetric # cubic foot +[cyd_i] = 1 [yd_i]3 nonmetric # cubic yard +[bf_i] = 144 [in_i]3 nonmetric # board foot +[cr_i] = 128 [ft_i]3 nonmetric # cord +[mil_i] = 1e-3 [in_i] nonmetric # mil +[cml_i] = 1 [pi]/4.[mil_i]2 nonmetric # circular mil +[hd_i] = 4 [in_i] nonmetric # hand +[ft_us] = 1200 m/3937 nonmetric # foot +[yd_us] = 3 [ft_us] nonmetric # yard +[in_us] = 1 [ft_us]/12 nonmetric # inch +[rd_us] = 16.5 [ft_us] nonmetric # rod +[ch_us] = 4 [rd_us] nonmetric # Gunter's chain +[lk_us] = 1 [ch_us]/100 nonmetric # link for Gunter's chain +[rch_us] = 100 [ft_us] nonmetric # Ramden's chain +[rlk_us] = 1 [rch_us]/100 nonmetric # link for Ramden's chain +[fth_us] = 6 [ft_us] nonmetric # fathom +[fur_us] = 40 [rd_us] nonmetric # furlong +[mi_us] = 8 [fur_us] nonmetric # mile +[acr_us] = 160 [rd_us]2 nonmetric # acre +[srd_us] = 1 [rd_us]2 nonmetric # square rod +[smi_us] = 1 [mi_us]2 nonmetric # square mile +[sct] = 1 [mi_us]2 nonmetric # section +[twp] = 36 [sct] nonmetric # township +[mil_us] = 1e-3 [in_us] nonmetric # mil +[in_br] = 2.539998 cm nonmetric # inch +[ft_br] = 12 [in_br] nonmetric # foot +[rd_br] = 16.5 [ft_br] nonmetric # rod +[ch_br] = 4 [rd_br] nonmetric # Gunter's chain +[lk_br] = 1 [ch_br]/100 nonmetric # link for Gunter's chain +[fth_br] = 6 [ft_br] nonmetric # fathom +[pc_br] = 2.5 [ft_br] nonmetric # pace +[yd_br] = 3 [ft_br] nonmetric # yard +[mi_br] = 5280 [ft_br] nonmetric # mile +[nmi_br] = 6080 [ft_br] nonmetric # nautical mile +[kn_br] = 1 [nmi_br]/h nonmetric # knot +[acr_br] = 4840 [yd_br]2 nonmetric # acre +[gal_us] = 231 [in_i]3 nonmetric # Queen Anne's wine gallon +[bbl_us] = 42 [gal_us] nonmetric # barrel +[qt_us] = 1 [gal_us]/4 nonmetric # quart +[pt_us] = 1 [qt_us]/2 nonmetric # pint +[gil_us] = 1 [pt_us]/4 nonmetric # gill +[foz_us] = 1 [gil_us]/4 nonmetric # fluid ounce +[fdr_us] = 1 [foz_us]/8 nonmetric # fluid dram +[min_us] = 1 [fdr_us]/60 nonmetric # minim +[crd_us] = 128 [ft_i]3 nonmetric # cord +[bu_us] = 2150.42 [in_i]3 nonmetric # bushel +[gal_wi] = 1 [bu_us]/8 nonmetric # historical winchester gallon +[pk_us] = 1 [bu_us]/4 nonmetric # peck +[dqt_us] = 1 [pk_us]/8 nonmetric # dry quart +[dpt_us] = 1 [dqt_us]/2 nonmetric # dry pint +[tbs_us] = 1 [foz_us]/2 nonmetric # tablespoon +[tsp_us] = 1 [tbs_us]/3 nonmetric # teaspoon +[cup_us] = 16 [tbs_us] nonmetric # cup +[foz_m] = 30 mL nonmetric # metric fluid ounce +[cup_m] = 240 mL nonmetric # metric cup +[tsp_m] = 5 mL nonmetric # metric teaspoon +[tbs_m] = 15 mL nonmetric # metric tablespoon +[gal_br] = 4.54609 l nonmetric # gallon +[pk_br] = 2 [gal_br] nonmetric # peck +[bu_br] = 4 [pk_br] nonmetric # bushel +[qt_br] = 1 [gal_br]/4 nonmetric # quart +[pt_br] = 1 [qt_br]/2 nonmetric # pint +[gil_br] = 1 [pt_br]/4 nonmetric # gill +[foz_br] = 1 [gil_br]/5 nonmetric # fluid ounce +[fdr_br] = 1 [foz_br]/8 nonmetric # fluid dram +[min_br] = 1 [fdr_br]/60 nonmetric # minim +[gr] = 64.79891 mg nonmetric # grain +[lb_av] = 7000 [gr] nonmetric # pound +[oz_av] = 1 [lb_av]/16 nonmetric # ounce +[dr_av] = 1 [oz_av]/16 nonmetric # dram +[scwt_av] = 100 [lb_av] nonmetric # short hundredweight +[lcwt_av] = 112 [lb_av] nonmetric # long hundredweight +[ston_av] = 20 [scwt_av] nonmetric # short ton +[lton_av] = 20 [lcwt_av] nonmetric # long ton +[stone_av] = 14 [lb_av] nonmetric # stone +[pwt_tr] = 24 [gr] nonmetric # pennyweight +[oz_tr] = 20 [pwt_tr] nonmetric # ounce +[lb_tr] = 12 [oz_tr] nonmetric # pound +[sc_ap] = 20 [gr] nonmetric # scruple +[dr_ap] = 3 [sc_ap] nonmetric # dram +[oz_ap] = 8 [dr_ap] nonmetric # ounce +[lb_ap] = 12 [oz_ap] nonmetric # pound +[oz_m] = 28 g nonmetric # metric ounce +[lne] = 1 [in_i]/12 nonmetric # line +[pnt] = 1 [lne]/6 nonmetric # point +[pca] = 12 [pnt] nonmetric # pica +[pnt_pr] = 0.013837 [in_i] nonmetric # Printer's point +[pca_pr] = 12 [pnt_pr] nonmetric # Printer's pica +[pied] = 32.48 cm nonmetric # pied +[pouce] = 1 [pied]/12 nonmetric # pouce +[ligne] = 1 [pouce]/12 nonmetric # ligne +[didot] = 1 [ligne]/6 nonmetric # didot +[cicero] = 12 [didot] nonmetric # cicero +[degF] = degf(5 K/9) nonmetric # degree Fahrenheit +[degR] = 5 K/9 nonmetric # degree Rankine +[degRe] = degre(5 K/4) nonmetric # degree Réaumur +cal_[15] = 4.18580 J metric # calorie at 15 °C +cal_[20] = 4.18190 J metric # calorie at 20 °C +cal_m = 4.19002 J metric # mean calorie +cal_IT = 4.1868 J metric # international table calorie +cal_th = 4.184 J metric # thermochemical calorie +cal = 1 cal_th metric # calorie +[Cal] = 1 kcal_th nonmetric # nutrition label Calories +[Btu_39] = 1.05967 kJ nonmetric # British thermal unit at 39 °F +[Btu_59] = 1.05480 kJ nonmetric # British thermal unit at 59 °F +[Btu_60] = 1.05468 kJ nonmetric # British thermal unit at 60 °F +[Btu_m] = 1.05587 kJ nonmetric # mean British thermal unit +[Btu_IT] = 1.05505585262 kJ nonmetric # international table British thermal unit +[Btu_th] = 1.054350 kJ nonmetric # thermochemical British thermal unit +[Btu] = 1 [Btu_th] nonmetric # British thermal unit +[HP] = 550 [ft_i].[lbf_av]/s nonmetric # horsepower +tex = 1 g/km metric # tex +[den] = 1 g/9/km nonmetric # Denier +m[H2O] = 980665e-5 kPa metric # meter of water column +m[Hg] = 133.3220 kPa metric # meter of mercury column +[in_i'H2O] = 1 m[H2O].[in_i]/m nonmetric # inch of water column +[in_i'Hg] = 1 m[Hg].[in_i]/m nonmetric # inch of mercury column +[PRU] = 1 mm[Hg].s/ml nonmetric # peripheral vascular resistance unit +[wood'U] = 1 mm[Hg].min/L nonmetric # Wood unit +[diop] = 1 /m nonmetric # diopter +[p'diop] = 100tan(1 rad) nonmetric # prism diopter +%[slope] = 100tan(1 rad) nonmetric # percent of slope +[mesh_i] = 1 /[in_i] nonmetric # mesh +[Ch] = 1 mm/3 nonmetric # Charrière +[drp] = 1 ml/20 nonmetric # drop +[hnsf'U] = 1 1 nonmetric # Hounsfield unit +[MET] = 3.5 mL/min/kg nonmetric # metabolic equivalent +[hp'_X] = hpX(1 1) nonmetric # homeopathic potency of decimal series (retired) +[hp'_C] = hpC(1 1) nonmetric # homeopathic potency of centesimal series (retired) +[hp'_M] = hpM(1 1) nonmetric # homeopathic potency of millesimal series (retired) +[hp'_Q] = hpQ(1 1) nonmetric # homeopathic potency of quintamillesimal series (retired) +[hp_X] = 1 1 nonmetric # arbitrary homeopathic potency of decimal hahnemannian series +[hp_C] = 1 1 nonmetric # arbitrary homeopathic potency of centesimal hahnemannian series +[hp_M] = 1 1 nonmetric # arbitrary homeopathic potency of millesimal hahnemannian series +[hp_Q] = 1 1 nonmetric # arbitrary homeopathic potency of quintamillesimal hahnemannian series +[kp_X] = 1 1 nonmetric # arbitrary homeopathic potency of decimal korsakovian series +[kp_C] = 1 1 nonmetric # arbitrary homeopathic potency of centesimal korsakovian series +[kp_M] = 1 1 nonmetric # arbitrary homeopathic potency of millesimal korsakovian series +[kp_Q] = 1 1 nonmetric # arbitrary homeopathic potency of quintamillesimal korsakovian series +eq = 1 mol metric # equivalents +osm = 1 mol metric # osmole +[pH] = pH(1 mol/l) nonmetric # pH +g% = 1 g/dl metric # gram percent +[S] = 1 10*-13.s nonmetric # Svedberg unit +[HPF] = 1 1 nonmetric # high power field +[LPF] = 100 1 nonmetric # low power field +kat = 1 mol/s metric # katal +U = 1 umol/min metric # Unit +[iU] = 1 1 metric # arbitrary international unit +[IU] = 1 [iU] metric # arbitrary international unit +[arb'U] = 1 1 nonmetric # arbitrary arbitrary unit +[USP'U] = 1 1 nonmetric # arbitrary United States Pharmacopeia unit +[GPL'U] = 1 1 nonmetric # arbitrary GPL unit +[MPL'U] = 1 1 nonmetric # arbitrary MPL unit +[APL'U] = 1 1 nonmetric # arbitrary APL unit +[beth'U] = 1 1 nonmetric # arbitrary Bethesda unit +[anti'Xa'U] = 1 1 nonmetric # arbitrary anti factor Xa unit +[todd'U] = 1 1 nonmetric # arbitrary Todd unit +[dye'U] = 1 1 nonmetric # arbitrary Dye unit +[smgy'U] = 1 1 nonmetric # arbitrary Somogyi unit +[bdsk'U] = 1 1 nonmetric # arbitrary Bodansky unit +[ka'U] = 1 1 nonmetric # arbitrary King-Armstrong unit +[knk'U] = 1 1 nonmetric # arbitrary Kunkel unit +[mclg'U] = 1 1 nonmetric # arbitrary Mac Lagan unit +[tb'U] = 1 1 nonmetric # arbitrary tuberculin unit +[CCID_50] = 1 1 nonmetric # arbitrary 50% cell culture infectious dose +[TCID_50] = 1 1 nonmetric # arbitrary 50% tissue culture infectious dose +[EID_50] = 1 1 nonmetric # arbitrary 50% embryo infectious dose +[PFU] = 1 1 nonmetric # arbitrary plaque forming units +[FFU] = 1 1 nonmetric # arbitrary focus forming units +[CFU] = 1 1 nonmetric # arbitrary colony forming units +[IR] = 1 1 nonmetric # arbitrary index of reactivity +[BAU] = 1 1 nonmetric # arbitrary bioequivalent allergen unit +[AU] = 1 1 nonmetric # arbitrary allergen unit +[Amb'a'1'U] = 1 1 nonmetric # arbitrary allergen unit for Ambrosia artemisiifolia +[PNU] = 1 1 nonmetric # arbitrary protein nitrogen unit +[Lf] = 1 1 nonmetric # arbitrary Limit of flocculation +[D'ag'U] = 1 1 nonmetric # arbitrary D-antigen unit +[FEU] = 1 1 nonmetric # arbitrary fibrinogen equivalent unit +[ELU] = 1 1 nonmetric # arbitrary ELISA unit +[EU] = 1 1 nonmetric # arbitrary Ehrlich unit +Np = ln(1 1) metric # neper +B = lg(1 1) metric # bel +B[SPL] = 2lg(2 10*-5.Pa) metric # bel sound pressure +B[V] = 2lg(1 V) metric # bel volt +B[mV] = 2lg(1 mV) metric # bel millivolt +B[uV] = 2lg(1 uV) metric # bel microvolt +B[10.nV] = 2lg(10 nV) metric # bel 10 nanovolt +B[W] = lg(1 W) metric # bel watt +B[kW] = lg(1 kW) metric # bel kilowatt +st = 1 m3 metric # stere +Ao = 0.1 nm nonmetric # Ångström +b = 100 fm2 nonmetric # barn +att = 1 kgf/cm2 nonmetric # technical atmosphere +mho = 1 S metric # mho +[psi] = 1 [lbf_av]/[in_i]2 nonmetric # pound per square inch +circ = 2 [pi].rad nonmetric # circle +sph = 4 [pi].sr nonmetric # sphere +[car_m] = 2e-1 g nonmetric # metric carat +[car_Au] = 1 /24 nonmetric # carat of gold alloys +[smoot] = 67 [in_i] nonmetric # Smoot +[m/s2/Hz^(1/2)] = sqrt(1 m2/s4/Hz) nonmetric # meter per square seconds per square root of hertz +[NTU] = 1 1 nonmetric # Nephelometric Turbidity Unit +[FNU] = 1 1 nonmetric # Formazin Nephelometric Unit +bit_s = ld(1 1) nonmetric # bit +bit = 1 1 metric # bit +By = 8 bit metric # byte +Bd = 1 /s metric # baud diff --git a/internal/units/units.go b/internal/units/units.go new file mode 100644 index 000000000..044dfe1aa --- /dev/null +++ b/internal/units/units.go @@ -0,0 +1,18 @@ +package units + +import ( + "go.opentelemetry.io/otel/metric" +) + +// Various common units. +var ( + Byte = metric.WithUnit("By") + Second = metric.WithUnit("s") +) + +// Count returns a [metric.InstrumentOption] for integer counts of something. +// +// The passed string should be singular and not include braces. +func Count(singular string) metric.InstrumentOption { + return metric.WithUnit("{" + singular + "}") +} diff --git a/internal/units/units_test.go b/internal/units/units_test.go new file mode 100644 index 000000000..8dfc5fa40 --- /dev/null +++ b/internal/units/units_test.go @@ -0,0 +1,113 @@ +package units + +import ( + "bufio" + "go/ast" + "go/parser" + "go/token" + "maps" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +// TestUnits ensures that top-level variables in "units.go" declared with +// [metric.ByUnit] are valid UCUM units. +func TestUnits(t *testing.T) { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "units.go", nil, 0) + if err != nil { + t.Fatal(err) + } + + stack := make([]ast.Node, 0, 1) + units := loadUnits(t) + ast.PreorderStack(f, stack, func(n ast.Node, _ []ast.Node) (descend bool) { + if _, ok := n.(*ast.File); ok { + return true + } + decl, ok := n.(*ast.GenDecl) + if !ok { + return + } + if decl.Tok != token.VAR { + return + } + for _, spec := range decl.Specs { + spec := spec.(*ast.ValueSpec) + if spec.Type != nil { + continue + } + for i, name := range spec.Names { + if !name.IsExported() { + continue + } + v := spec.Values[i] + c, ok := v.(*ast.CallExpr) + if !ok || len(c.Args) != 1 { + continue + } + sel, ok := c.Fun.(*ast.SelectorExpr) + if !ok || sel.X.(*ast.Ident).Name != "metric" || sel.Sel.Name != "WithUnit" { + continue + } + lit, ok := c.Args[0].(*ast.BasicLit) + if !ok { + continue + } + u, err := strconv.Unquote(lit.Value) + if err != nil { + t.Errorf("%v: %v", fset.Position(spec.Pos()), err) + continue + } + if _, ok := units[u]; !ok { + t.Errorf("%s:\tunknown unit %q @ %v", name.Name, u, fset.Position(spec.Pos())) + } else { + t.Logf("%s:\tknown unit %q", name.Name, u) + } + } + } + return + }) +} + +func loadUnits(t testing.TB) map[string]struct{} { + t.Helper() + + f, err := os.Open(filepath.Join("testdata", "ucum-cs.units")) + if err != nil { + t.Fatal(err) + } + s := bufio.NewScanner(f) + defer func() { + if err := s.Err(); err != nil { + t.Errorf("scanner error: %v", err) + } + if err := f.Close(); err != nil { + t.Errorf("close error: %v", err) + } + }() + + seq := func(yield func(string, struct{}) bool) { + for s.Scan() { + l, _, _ := strings.Cut(s.Text(), "#") + fs := strings.Fields(l) + switch { + case len(fs) == 2 && fs[0] == "base": + l = fs[1] + case len(fs) == 5 && fs[4] == "metric": + l = fs[0] + default: + continue + } + + if !yield(l, struct{}{}) { + return + } + } + } + + return maps.Collect(seq) +} From 016598e985ee39ddf88d6defdc42b0b2c3af470d Mon Sep 17 00:00:00 2001 From: Hank Donnay Date: Thu, 26 Mar 2026 12:20:53 -0500 Subject: [PATCH 2/4] cache: add `Len` method to `Live` This will allow for users to to report metrics on cache size. Signed-off-by: Hank Donnay Change-Id: I8acbc67d48a4c5ef5fdcd077c7ba7bf66a6a6964 --- internal/cache/live.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/internal/cache/live.go b/internal/cache/live.go index d5a1e6b95..b5882ed94 100644 --- a/internal/cache/live.go +++ b/internal/cache/live.go @@ -94,3 +94,18 @@ func (c *Live[K, V]) Get(ctx context.Context, key K, create CreateFunc[K, V]) (* // No additional calls are made for individual values; the cache simply drops // any references it has. func (c *Live[K, V]) Clear() { c.m.Clear() } + +// Len reports the approximate number of entries in the cache. +// +// The count is approximate because concurrent removals and additions may not be +// seen. If a caller wants an accurate count, it must arrange to prevent +// concurrent modifications. +func (c *Live[K, V]) Len() (n int) { + c.m.Range(func(_, value any) bool { + if v := value.(weak.Pointer[V]).Value(); v != nil { + n++ + } + return true + }) + return n +} From ec7aedf7c9f25b80cab8313166be7ed2b84f380e Mon Sep 17 00:00:00 2001 From: Hank Donnay Date: Thu, 26 Mar 2026 12:06:54 -0500 Subject: [PATCH 3/4] libindex: fetcher metrics This adds OpenTelemetry metrics to `RemoteFetchArena` instances. Signed-off-by: Hank Donnay Change-Id: Iccaceed2a9099c23737892b0bea071506a6a6964 --- libindex/fetcher.go | 66 ++++++++++++---- libindex/instrumentation.go | 145 ++++++++++++++++++++++++++++++++++++ libindex/metrics.go | 15 ---- 3 files changed, 197 insertions(+), 29 deletions(-) create mode 100644 libindex/instrumentation.go delete mode 100644 libindex/metrics.go diff --git a/libindex/fetcher.go b/libindex/fetcher.go index 41ca4dc06..a92523771 100644 --- a/libindex/fetcher.go +++ b/libindex/fetcher.go @@ -37,7 +37,8 @@ var ( // RemoteFetchArena uses disk space to track fetched layers, removing them once // all users are done with the layers. type RemoteFetchArena struct { - wc *http.Client + metrics arenaMetrics + wc *http.Client // The string is a layer digest. files cache.Live[string, os.File] @@ -46,6 +47,21 @@ type RemoteFetchArena struct { // NewRemoteFetchArena returns an initialized RemoteFetchArena. // +// Deprecated: Use [CreateRemoteFetchArena]. +// +//go:fix inline +func NewRemoteFetchArena(wc *http.Client, root string) *RemoteFetchArena { + a, err := CreateRemoteFetchArena(wc, root) + if err != nil { + // Backed ourselves into a corner on this API 🙃 + panic(err) + } + return a +} + +// CreateRemoteFetchArena returns an initialized RemoteFetchArena or an error +// encountered while creating it. +// // If the "root" parameter is "", the advice in [file-hierarchy(7)] and ["Using // /tmp/ and /var/tmp/ Safely"] is followed. Specifically, "/var/tmp" is used // unless "TMPDIR" is set in the environment, in which case the contents of that @@ -70,18 +86,21 @@ type RemoteFetchArena struct { // ["Using /tmp/ and /var/tmp/ Safely"]: https://systemd.io/TEMPORARY_DIRECTORIES/ // [O_TMPFILE]: https://man7.org/linux/man-pages/man2/open.2.html // [systemd-tmpfiles(8)]: https://www.freedesktop.org/software/systemd/man/latest/systemd-tmpfiles.html -func NewRemoteFetchArena(wc *http.Client, root string) *RemoteFetchArena { +func CreateRemoteFetchArena(wc *http.Client, root string) (*RemoteFetchArena, error) { name := fixTemp(root) dir, err := os.OpenRoot(name) if err != nil { - // Backed ourselves into a corner on this API 🙃 - panic(fmt.Errorf("fetcher: unable to OpenRoot(%q): %w", root, err)) + return nil, fmt.Errorf("fetcher: unable to OpenRoot(%q): %w", root, err) } a := &RemoteFetchArena{ wc: wc, root: dir, } - return a + if err := a.setupMetrics(name); err != nil { + return nil, fmt.Errorf("fetcher: unable to construct metrics: %w", err) + } + + return a, nil } // FetchInto populates "l" and "cl" via a cache. @@ -91,8 +110,10 @@ func (a *RemoteFetchArena) fetchInto(ctx context.Context, l *claircore.Layer, cl key := desc.Digest return func() (err error) { - ctx, span := tracer.Start(ctx, "RemoteFetchArena.fetchInto", trace.WithAttributes(attribute.String("key", key))) - defer span.End() + cacheHit := true + ctx, span := tracer.Start(ctx, "RemoteFetchArena.fetchInto", + trace.WithAttributes(attribute.String("key", key))) + req := a.metrics.Request() defer func() { span.RecordError(err) if err == nil { @@ -100,6 +121,8 @@ func (a *RemoteFetchArena) fetchInto(ctx context.Context, l *claircore.Layer, cl } else { span.SetStatus(codes.Error, "fetchInto error") } + req(ctx, err != nil, cacheHit) + span.End() }() // NB This is not closed on purpose. The [io.Closer] populated by this @@ -111,6 +134,7 @@ func (a *RemoteFetchArena) fetchInto(ctx context.Context, l *claircore.Layer, cl // [reopen] helper. var spool *os.File spool, err = a.files.Get(ctx, key, func(ctx context.Context, _ string) (*os.File, error) { + cacheHit = false return a.fetchFileForCache(ctx, desc) }) if err != nil { @@ -155,7 +179,7 @@ func (f closeFunc) Close() error { // we can be a bit more lax. func (a *RemoteFetchArena) fetchFileForCache(ctx context.Context, desc *claircore.LayerDescription) (*os.File, error) { log := slog.With("arena", a.root.Name(), "layer", desc.Digest, "uri", desc.URI) - ctx, span := tracer.Start(ctx, "RemoteFetchArena.fetchUnlinkedFile") + ctx, span := tracer.Start(ctx, "RemoteFetchArena.fetchFileForCache") defer span.End() span.SetStatus(codes.Error, "") log.DebugContext(ctx, "layer fetch start") @@ -196,7 +220,8 @@ func (a *RemoteFetchArena) fetchFileForCache(ctx context.Context, desc *claircor if err != nil { return nil, err } - tr := io.TeeReader(resp.Body, vh) + var cmpSz writeCounter + tr := io.TeeReader(resp.Body, io.MultiWriter(&cmpSz, vh)) // TODO(hank) All this decompression code could go away, but that would mean // that a buffer file would have to be allocated later, adding additional @@ -212,7 +237,7 @@ func (a *RemoteFetchArena) fetchFileForCache(ctx context.Context, desc *claircor // Look at the content-type and optionally fix it up. ct := resp.Header.Get("content-type") log.DebugContext(ctx, "reported content-type", "content-type", ct) - span.SetAttributes(attribute.String("payload.content-type", ct), attribute.Stringer("payload.compression.detected", kind)) + span.SetAttributes(payloadType(ct), payloadCompression(kind)) if ct == "" || ct == "text/plain" || ct == "binary/octet-stream" || ct == "application/octet-stream" { switch kind { case zreader.KindGzip: @@ -225,7 +250,7 @@ func (a *RemoteFetchArena) fetchFileForCache(ctx context.Context, desc *claircor return nil, fmt.Errorf("fetcher: disallowed compression kind: %q", kind.String()) } log.DebugContext(ctx, "fixed content-type", "content-type", ct) - span.SetAttributes(attribute.String("payload.content-type.detected", ct)) + span.AddEvent("FixedContentType", trace.WithAttributes(payloadTypeFixed(ct))) } var wantZ zreader.Compression @@ -273,10 +298,11 @@ func (a *RemoteFetchArena) fetchFileForCache(ctx context.Context, desc *claircor buf := bufio.NewWriter(f) n, err := io.Copy(buf, zr) log.DebugContext(ctx, "wrote file", "size", n, "big", n >= bigLayerSize, "copy_error", err) - // TODO(hank) Add a metric for "big files" and a histogram for size. if err != nil { return nil, err } + a.metrics.CompressedSize(ctx, int64(cmpSz)) + a.metrics.UncompressedSize(ctx, n) if err := buf.Flush(); err != nil { return nil, err } @@ -295,13 +321,25 @@ func (a *RemoteFetchArena) fetchFileForCache(ctx context.Context, desc *claircor const bigLayerSize = 1024 * 1024 * 1024 // 1 GiB +// WriteCounter is a helper that just counts the bytes passed to it. +// +// Useful in an [io.MultiWriter]. +type writeCounter int64 + +func (c *writeCounter) Write(p []byte) (int, error) { + n := len(p) + *(*int64)(c) += int64(n) + return n, nil +} + // Close forgets all references in the arena. // // Any outstanding Layers may cause keys to be forgotten at unpredictable times. func (a *RemoteFetchArena) Close(_ context.Context) error { a.files.Clear() - if err := a.root.Close(); err != nil { - return fmt.Errorf("fetcher: RemoteFetchArena: unable to close os.Root: %w", err) + err := errors.Join(a.metrics.entries.Unregister(), a.root.Close()) + if err != nil { + return fmt.Errorf("fetcher: RemoteFetchArena: closing: %w", err) } return nil } diff --git a/libindex/instrumentation.go b/libindex/instrumentation.go new file mode 100644 index 000000000..7c155bca8 --- /dev/null +++ b/libindex/instrumentation.go @@ -0,0 +1,145 @@ +package libindex + +import ( + "context" + "fmt" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" + + "github.com/quay/claircore/internal/units" +) + +const i14nName = "github.com/quay/claircore/libindex" + +var ( + tracer trace.Tracer + meter metric.Meter +) + +func init() { + tracer = otel.Tracer(i14nName) + meter = otel.Meter(i14nName) +} + +var ( + pathKey = attribute.Key("path") + compressedKey = attribute.Key("compressed") + failedKey = attribute.Key("failed") + cacheHitKey = attribute.Key("cache_hit") + payloadTypeKey = attribute.Key("payload.content_type.header") + payloadTypeFixedKey = attribute.Key("payload.content_type.detected") + payloadCompressionKey = attribute.Key("payload.compression.detected") +) + +func pathAttr(p string) attribute.KeyValue { + return pathKey.String(p) +} + +func compressedAttr(b bool) attribute.KeyValue { + return compressedKey.Bool(b) +} + +func failedAttr(b bool) attribute.KeyValue { + return failedKey.Bool(b) +} + +func cacheHitAttr(b bool) attribute.KeyValue { + return cacheHitKey.Bool(b) +} + +func payloadType(ct string) attribute.KeyValue { + return payloadTypeKey.String(ct) +} + +func payloadTypeFixed(ct string) attribute.KeyValue { + return payloadTypeFixedKey.String(ct) +} + +func payloadCompression(c fmt.Stringer) attribute.KeyValue { + return payloadCompressionKey.String(c.String()) +} + +// ArenaMetrics holds metrics for a single [RemoteFetchArena]. +// +// The resulting metrics are scoped for the arena. +type arenaMetrics struct { + sizes metric.Int64Histogram + duration metric.Float64Histogram + requests metric.Int64Counter + entries metric.Registration +} + +// SetupMetrics does setup for metrics in the receiver. +func (a *RemoteFetchArena) setupMetrics(dir string) (err error) { + meter := otel.Meter(i14nName, + metric.WithInstrumentationAttributes( + pathAttr(dir), + ), + ) + + a.metrics.sizes, err = meter.Int64Histogram("fetcher.layer_size", + metric.WithDescription("Size of container layers."), + units.Byte, + ) + if err != nil { + return err + } + a.metrics.duration, err = meter.Float64Histogram("fetcher.duration", + metric.WithDescription("Duration of fetcher requests."), + units.Second, + units.LargeBuckets, + ) + if err != nil { + return err + } + a.metrics.requests, err = meter.Int64Counter("fetcher.requests", + metric.WithDescription("Number of fetcher requests."), + units.Count("request"), + ) + if err != nil { + return err + } + + entries, err := meter.Int64ObservableGauge("fetcher.cache.entries", + metric.WithDescription("Number of live entries in the layer cache."), + units.Count("entry"), + ) + if err != nil { + return err + } + a.metrics.entries, err = meter.RegisterCallback(func(_ context.Context, o metric.Observer) error { + o.ObserveInt64(entries, int64(a.files.Len())) + return nil + }, entries) + if err != nil { + return err + } + + return nil +} + +// CompressedSize records a layer's compressed size. +func (m *arenaMetrics) CompressedSize(ctx context.Context, sz int64) { + m.sizes.Record(ctx, sz, metric.WithAttributes(compressedAttr(true))) +} + +// UncompressedSize records a layer's uncompressed (on-disk) size. +func (m *arenaMetrics) UncompressedSize(ctx context.Context, sz int64) { + m.sizes.Record(ctx, sz, metric.WithAttributes(compressedAttr(false))) +} + +// Request returns a function to record one fetch request. +// +// This method counts total requests and durations. +func (m *arenaMetrics) Request() func(ctx context.Context, failed, cacheHit bool) { + start := time.Now() + return func(ctx context.Context, failed, cacheHit bool) { + s := attribute.NewSet(failedAttr(failed), cacheHitAttr(cacheHit)) + m.duration.Record(ctx, time.Since(start).Seconds(), metric.WithAttributeSet(s)) + m.requests.Add(ctx, 1, metric.WithAttributeSet(s)) + } +} diff --git a/libindex/metrics.go b/libindex/metrics.go deleted file mode 100644 index 7beb4fe2d..000000000 --- a/libindex/metrics.go +++ /dev/null @@ -1,15 +0,0 @@ -package libindex - -import ( - "go.opentelemetry.io/otel" - semconv "go.opentelemetry.io/otel/semconv/v1.21.0" - "go.opentelemetry.io/otel/trace" -) - -var tracer trace.Tracer - -func init() { - tracer = otel.Tracer("github.com/quay/claircore/libindex", - trace.WithSchemaURL(semconv.SchemaURL), - ) -} From 757e6649cb5bc3dfb2b5e4a3e41c292f5b09d064 Mon Sep 17 00:00:00 2001 From: Hank Donnay Date: Thu, 26 Mar 2026 13:37:59 -0500 Subject: [PATCH 4/4] libindex: various lints Signed-off-by: Hank Donnay Change-Id: Ie36484d4b76a2dba60cebeb260c3c22e6a6a6964 --- libindex/doc.go | 5 +++++ libindex/fetcher_test.go | 14 +++++++------- libindex/libindex.go | 4 ++-- libindex/libindex_integration_test.go | 8 ++++++-- libindex/options.go | 6 +++++- 5 files changed, 25 insertions(+), 12 deletions(-) create mode 100644 libindex/doc.go diff --git a/libindex/doc.go b/libindex/doc.go new file mode 100644 index 000000000..4e0c4fa33 --- /dev/null +++ b/libindex/doc.go @@ -0,0 +1,5 @@ +// Package libindex is the claircore Indexer. +// +// The Indexer is the component that analyzes container images (or +// container-like filesystems) and produces IndexReports. +package libindex diff --git a/libindex/fetcher_test.go b/libindex/fetcher_test.go index b5ef8a6ca..ff9508baa 100644 --- a/libindex/fetcher_test.go +++ b/libindex/fetcher_test.go @@ -231,18 +231,18 @@ func commonLayerServer(t testing.TB, ct int) ([]claircore.LayerDescription, http t.Cleanup(func() { // We know we're doing 2 sets of fetches. - max := ct * 2 * runtime.GOMAXPROCS(0) + limit := ct * 2 * runtime.GOMAXPROCS(0) var total int for _, v := range fetch { total += int(*v) } switch { - case total > max: - t.Errorf("more fetches than should be possible: %d > %d", total, max) - case total == max: - t.Errorf("prevented no fetches: %d == %d", total, max) - case total < max: - t.Logf("prevented %[3]d fetches: %[1]d < %d", total, max, max-total) + case total > limit: + t.Errorf("more fetches than should be possible: %d > %d", total, limit) + case total == limit: + t.Errorf("prevented no fetches: %d == %d", total, limit) + case total < limit: + t.Logf("prevented %[3]d fetches: %[1]d < %d", total, limit, limit-total) } }) inner := http.FileServer(http.Dir(dir)) diff --git a/libindex/libindex.go b/libindex/libindex.go index a97f9d1dc..75f93d83c 100644 --- a/libindex/libindex.go +++ b/libindex/libindex.go @@ -197,7 +197,7 @@ func (l *Libindex) Index(ctx context.Context, manifest *claircore.Manifest) (*cl // // If the identifier has changed, clients should arrange for layers to be // re-indexed. -func (l *Libindex) State(ctx context.Context) (string, error) { +func (l *Libindex) State(_ context.Context) (string, error) { return l.state, nil } @@ -207,7 +207,7 @@ func (l *Libindex) State(ctx context.Context) (string, error) { // Indexers running different scanner versions will produce different state strings. // Thus this state value can be used as a cue for clients to re-index their manifests // and obtain a new IndexReport. -func (l *Libindex) setState(ctx context.Context, vscnrs indexer.VersionedScanners) error { +func (l *Libindex) setState(_ context.Context, vscnrs indexer.VersionedScanners) error { h := md5.New() var ns []string m := make(map[string][]byte) diff --git a/libindex/libindex_integration_test.go b/libindex/libindex_integration_test.go index 99230be17..302da34d7 100644 --- a/libindex/libindex_integration_test.go +++ b/libindex/libindex_integration_test.go @@ -104,10 +104,14 @@ func (tc testcase) RunInner(ctx context.Context, t *testing.T, pool *pgxpool.Poo } // create libindex instance + a, err := CreateRemoteFetchArena(c, t.TempDir()) + if err != nil { + t.Fatalf("failed to create RemoteFetchArena: %v", err) + } opts := &Options{ Store: store, Locker: ctxLocker, - FetchArena: NewRemoteFetchArena(c, t.TempDir()), + FetchArena: a, ScanLockRetry: 2 * time.Second, LayerScanConcurrency: 1, Ecosystems: []*indexer.Ecosystem{ @@ -125,7 +129,7 @@ func (tc testcase) RunInner(ctx context.Context, t *testing.T, pool *pgxpool.Poo RepositoryScanners: func(_ context.Context) ([]indexer.RepositoryScanner, error) { return nil, nil }, - FileScanners: func(ctx context.Context) ([]indexer.FileScanner, error) { + FileScanners: func(_ context.Context) ([]indexer.FileScanner, error) { return []indexer.FileScanner{&whiteout.Scanner{}}, nil }, Coalescer: func(_ context.Context) (indexer.Coalescer, error) { diff --git a/libindex/options.go b/libindex/options.go index 0e7a90c88..d1c97f7bb 100644 --- a/libindex/options.go +++ b/libindex/options.go @@ -7,7 +7,11 @@ import ( ) const ( - DefaultScanLockRetry = 5 * time.Second + // DefaultScanLockRetry is the default time to wait between attempting to + // acquire the "scan lock". + DefaultScanLockRetry = 5 * time.Second + // DefaultLayerScanConcurrency is the default number of concurrent layer + // "scans". DefaultLayerScanConcurrency = 10 )