diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..12fd39f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,47 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + branches: ['**'] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + + - name: Check formatting + run: cargo fmt --all -- --check + + - name: Build + run: cargo build --lib --bins --verbose + + - name: Test + run: cargo test --verbose + + - name: Clippy + run: cargo clippy --lib --bins --tests --verbose + + # Guards the rust-version in Cargo.toml so the declared MSRV can't silently + # drift above the floor. The toolchain action pins 1.85 as this job's + # default, so a bare cargo invocation builds against it. + msrv: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@master + with: + toolchain: "1.85" + + - name: Build with MSRV + run: cargo build --lib --bins --verbose + + - name: Test with MSRV + run: cargo test --verbose diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index ccd6140..0000000 --- a/.travis.yml +++ /dev/null @@ -1,43 +0,0 @@ -language: rust -rust: - - stable - - beta - - nightly - -cache: cargo - -os: - - linux - - osx - -sudo: false - -env: - global: - - RUSTFLAGS="-C link-dead-code" - -addons: - apt: - packages: - - libcurl4-openssl-dev - - libelf-dev - - libdw-dev - - cmake - - gcc - - binutils-dev - - libiberty-dev - -after_success: | - wget https://github.com/SimonKagstrom/kcov/archive/master.tar.gz && - tar xzf master.tar.gz && - cd kcov-master && - mkdir build && - cd build && - cmake .. && - make && - make install DESTDIR=../../kcov-build && - cd ../.. && - rm -rf kcov-master && - for file in target/debug/hornet-*[^\.d]; do mkdir -p "target/cov/$(basename $file)"; ./kcov-build/usr/local/bin/kcov --exclude-pattern=/.cargo,/usr/lib --verify "target/cov/$(basename $file)" "$file"; done && - bash <(curl -s https://codecov.io/bash) && - echo "Uploaded code coverage" diff --git a/Cargo.toml b/Cargo.toml index d4b477c..a93fe8d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,8 @@ name = "hornet" version = "0.1.0" authors = ["Saurav Sachidanand "] +edition = "2021" +rust-version = "1.84" license = "MIT OR Apache-2.0" description = "A Rust implementation of the PCP instrumentation API" repository = "https://github.com/performancecopilot/hornet" @@ -9,14 +11,10 @@ readme="README.md" keywords = ["performance", "instrumentation", "metric", "pcp", "mmv"] [dependencies] -bitflags = "0.9.1" -hdrsample = "4.0.0" -lazy_static = "0.2.8" +bitflags = "2" +hdrhistogram = "=7.5.4" memmap2 = "0.9" [dev-dependencies] -rand = "0.3.15" -hyper = "0.11.2" -futures = "0.1.14" -curl = "0.4.8" -iron = "0.5.1" +rand = "0.8" +tempfile = "3.16" diff --git a/README.md b/README.md index 44cfe59..3d48281 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# hornet [![crates.io badge](https://img.shields.io/crates/v/hornet.svg)](https://crates.io/crates/hornet) [![docs.rs badge](https://docs.rs/hornet/badge.svg)](https://docs.rs/hornet/0.1.0/hornet/) [![Travis CI Build Status](https://travis-ci.org/performancecopilot/hornet.svg?branch=master)](https://travis-ci.org/performancecopilot/hornet) [![AppVeyor Build Status](https://ci.appveyor.com/api/projects/status/ccvbo3chne8046vn/branch/master?svg=true)](https://ci.appveyor.com/project/saurvs/hornet-2qtki/branch/master) [![codecov](https://codecov.io/gh/performancecopilot/hornet/branch/master/graph/badge.svg)](https://codecov.io/gh/performancecopilot/hornet) +# hornet [![crates.io badge](https://img.shields.io/crates/v/hornet.svg)](https://crates.io/crates/hornet) [![docs.rs badge](https://docs.rs/hornet/badge.svg)](https://docs.rs/hornet/0.1.0/hornet/) [![CI](https://github.com/performancecopilot/hornet/actions/workflows/ci.yml/badge.svg)](https://github.com/performancecopilot/hornet/actions/workflows/ci.yml) `hornet` is a Performance Co-Pilot (PCP) Memory Mapped Values (MMV) instrumentation library written in Rust. diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index a91bc79..0000000 --- a/appveyor.yml +++ /dev/null @@ -1,70 +0,0 @@ -environment: - matrix: - # i686-pc-windows-gnu - - TARGET: i686-pc-windows-gnu - CHANNEL: stable - MINGW_URL: https://s3.amazonaws.com/rust-lang-ci - MINGW_ARCHIVE: i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z - MINGW_DIR: mingw32 - - TARGET: i686-pc-windows-gnu - CHANNEL: beta - MINGW_URL: https://s3.amazonaws.com/rust-lang-ci - MINGW_ARCHIVE: i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z - MINGW_DIR: mingw32 - - TARGET: i686-pc-windows-gnu - CHANNEL: nightly - MINGW_URL: https://s3.amazonaws.com/rust-lang-ci - MINGW_ARCHIVE: i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z - MINGW_DIR: mingw32 - - # x86_64-pc-windows-gnu - - TARGET: x86_64-pc-windows-gnu - CHANNEL: stable - MSYS_BITS: 64 - - TARGET: x86_64-pc-windows-gnu - CHANNEL: beta - MSYS_BITS: 64 - - TARGET: x86_64-pc-windows-gnu - CHANNEL: nightly - MSYS_BITS: 64 - - # i686-pc-windows-msvc - - TARGET: i686-pc-windows-msvc - CHANNEL: stable - - TARGET: i686-pc-windows-msvc - CHANNEL: beta - - TARGET: i686-pc-windows-msvc - CHANNEL: nightly - - # x86_64-pc-windows-msvc - - TARGET: x86_64-pc-windows-msvc - CHANNEL: stable - - TARGET: x86_64-pc-windows-msvc - CHANNEL: beta - - TARGET: x86_64-pc-windows-msvc - CHANNEL: nightly - -install: - - appveyor-retry appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe - - rustup-init.exe --default-host %TARGET% --default-toolchain %CHANNEL% -y - - set PATH=%PATH%;C:\Users\appveyor\.cargo\bin - - if defined MSYS_BITS set PATH=C:\msys64\mingw%MSYS_BITS%\bin;C:\msys64\usr\bin;%PATH% - - if defined MINGW_URL appveyor DownloadFile %MINGW_URL%/%MINGW_ARCHIVE% - - if defined MINGW_URL 7z x -y %MINGW_ARCHIVE% > nul - - if defined MINGW_URL set PATH=C:\Python27;%CD%\%MINGW_DIR%\bin;C:\msys64\usr\bin;%PATH% - - rustc -V - - cargo -V - -build: false - -test_script: - - cargo test --verbose - -cache: - - target - - C:\Users\appveyor\.cargo\registry - -matrix: - allow_failures: - - channel: beta - - channel: nightly diff --git a/examples/acme.rs b/examples/acme.rs index 7bf6cc1..8625292 100644 --- a/examples/acme.rs +++ b/examples/acme.rs @@ -1,9 +1,6 @@ -extern crate hornet; -extern crate rand; - use hornet::client::metric::*; use hornet::client::Client; -use rand::random; +use rand::Rng; use std::thread; use std::time::Duration; @@ -60,9 +57,9 @@ fn main() { /* update metrics */ loop { - let rnd_idx = random::() % products.len(); + let rnd_idx = rand::thread_rng().gen_range(0..products.len()); let product = products[rnd_idx]; - let working_time = random::() % 3; + let working_time = rand::thread_rng().gen_range(0..3); thread::sleep(Duration::from_secs(working_time)); let count = *counts.val(product).unwrap(); diff --git a/examples/growth.rs b/examples/growth.rs index 8ccbf90..cf5d04d 100644 --- a/examples/growth.rs +++ b/examples/growth.rs @@ -1,6 +1,3 @@ -extern crate hornet; -extern crate rand; - use hornet::client::metric::*; use hornet::client::Client; use std::thread; diff --git a/examples/histogram.rs b/examples/histogram.rs index 61401b0..d4e1122 100644 --- a/examples/histogram.rs +++ b/examples/histogram.rs @@ -1,14 +1,10 @@ -extern crate hornet; -extern crate rand; - use hornet::client::metric::*; use hornet::client::Client; -use rand::distributions::{IndependentSample, Range}; -use rand::thread_rng; +use rand::Rng; /* For detailed usage and behaviour of the underlying HDR histogram object, - check out jonhoo's hdrsample crate at https://github.com/jonhoo/hdrsample + check out the hdrhistogram crate at https://github.com/HdrHistogram/HdrHistogram_rust */ fn main() { @@ -42,15 +38,13 @@ fn main() { /* record 100 random values */ - let range = Range::new(low, high); - let mut thread_rng = thread_rng(); + let mut rng = rand::thread_rng(); for _ in 0..100 { - hist.record(range.ind_sample(&mut thread_rng)).unwrap(); + hist.record(rng.gen_range(low..high)).unwrap(); } /* record a single random value 100 times */ - hist.record_n(range.ind_sample(&mut thread_rng), 100) - .unwrap(); + hist.record_n(rng.gen_range(low..high), 100).unwrap(); } diff --git a/examples/http_download.rs b/examples/http_download.rs deleted file mode 100644 index e96fb16..0000000 --- a/examples/http_download.rs +++ /dev/null @@ -1,47 +0,0 @@ -extern crate curl; -extern crate hornet; - -use curl::easy::Easy; -use hornet::client::metric::*; -use hornet::client::Client; - -/* - this example uses the Timer metric to measure time spent - downloading the zipped linux kernel source using libcurl bindings -*/ - -const URL: &'static str = "https://codeload.github.com/torvalds/linux/zip/master"; - -fn main() { - let mut timer = Timer::new("time", Time::Sec, "Time elapsed downloading", "").unwrap(); - - let mut bytes = Metric::new( - "bytes", - 0, - Semantics::Discrete, - Unit::new().space(Space::Byte, 1).unwrap(), - "Bytes downloaded so far", - "", - ) - .unwrap(); - - let client = Client::new("download").unwrap(); - client.export(&mut [&mut timer, &mut bytes]).unwrap(); - - let mut easy = Easy::new(); - easy.url(URL).unwrap(); - - easy.progress(true).unwrap(); - easy.progress_function(move |_, bytes_downloaded, _, _| { - timer.stop().ok(); - timer.start().ok(); - bytes.set_val(bytes_downloaded as u64).unwrap(); - true - }) - .unwrap(); - - println!("Downloading from {}", URL); - println!("Progress mapped at {}", client.mmv_path().to_str().unwrap()); - - easy.perform().unwrap(); -} diff --git a/examples/http_server.rs b/examples/http_server.rs deleted file mode 100644 index 5c4d5b5..0000000 --- a/examples/http_server.rs +++ /dev/null @@ -1,87 +0,0 @@ -extern crate futures; -extern crate hornet; -extern crate hyper; - -use futures::future::FutureResult; -use hornet::client::metric::*; -use hornet::client::Client; -use hyper::header::{ContentLength, ContentType}; -use hyper::server::{Http, Request, Response, Service}; -use hyper::{Get, StatusCode}; -use std::sync::{Arc, Mutex}; - -/* - records count of HTTP GET requests on localhost:8000 - - this example also shows how to safely read and update - a metric concurrently -*/ - -static URL: &'static str = "127.0.0.1:8000"; - -struct HTTPCounterService { - arc: Arc>, -} - -impl Service for HTTPCounterService { - type Request = Request; - type Response = Response; - type Error = hyper::Error; - type Future = FutureResult; - - fn call(&self, req: Request) -> Self::Future { - futures::future::ok(match (req.method(), req.path()) { - (&Get, "/") => { - let mut counter = self.arc.lock().unwrap(); - - /* increase the counter value by one */ - counter.up().unwrap(); - - let body = format!("HTTP GET count = {}", counter.val()); - Response::new() - .with_header(ContentLength(body.len() as u64)) - .with_header(ContentType::plaintext()) - .with_body(body) - } - _ => Response::new().with_status(StatusCode::NotFound), - }) - } -} - -fn main() { - /* create a counter metric */ - - let mut counter = Counter::new( - "get", - 0, // initial value - "GET request count", // short description - &format!("Count of GET requests on http://{}/", URL), // long description - ) - .unwrap(); - - /* export it to an mmv */ - - let client = Client::new("localhost.http").unwrap(); - client.export(&mut [&mut counter]).unwrap(); - - /* - since the counter could be updated concurrently, wrap it - in a mutex. to have shared ownership of the mutex itself, - wrap it in an atomic reference counting pointer - */ - - let mutex = Mutex::new(counter); - let arc = Arc::new(mutex); - - /* create and run the server */ - - let addr = URL.parse().unwrap(); - let server = Http::new() - .bind(&addr, move || Ok(HTTPCounterService { arc: arc.clone() })) - .unwrap(); - - println!("Listening on http://{}", server.local_addr().unwrap()); - println!("Counter mapped at {}", client.mmv_path().to_str().unwrap()); - - server.run().unwrap(); -} diff --git a/examples/iron_middleware.rs b/examples/iron_middleware.rs deleted file mode 100644 index 06e3c52..0000000 --- a/examples/iron_middleware.rs +++ /dev/null @@ -1,86 +0,0 @@ -extern crate hornet; -extern crate iron; - -use hornet::client::metric::*; -use hornet::client::Client; -use iron::method::Method; -use iron::middleware::BeforeMiddleware; -use iron::prelude::*; -use iron::status; -use std::sync::Mutex; - -/* - this examples demonstrates usage of CountVector metric - embedded in Iron BeforeMiddleware -*/ - -static URL: &'static str = "127.0.0.1:8000"; - -fn method_str(method: &Method) -> String { - format!("{}", method) -} - -struct MethodCounter { - pub metric: Mutex, -} - -impl MethodCounter { - fn new() -> Self { - let metric = CountVector::new( - "methods_count", - 0, - &[ - &method_str(&Method::Options), - &method_str(&Method::Get), - &method_str(&Method::Post), - &method_str(&Method::Put), - &method_str(&Method::Delete), - &method_str(&Method::Head), - &method_str(&Method::Trace), - &method_str(&Method::Connect), - ], - "Counts of recieved HTTP request methods", - "", - ) - .unwrap(); - - MethodCounter { - metric: Mutex::new(metric), - } - } -} - -impl BeforeMiddleware for MethodCounter { - fn before(&self, req: &mut Request) -> IronResult<()> { - match &req.method { - &Method::Extension(_) => {} - _ => { - let mut counter = self.metric.lock().unwrap(); - counter.up(&method_str(&req.method)).unwrap().unwrap(); - } - } - Ok(()) - } - - fn catch(&self, _: &mut Request, _: IronError) -> IronResult<()> { - Ok(()) - } -} - -fn main() { - let method_counter = MethodCounter::new(); - - let client = Client::new("localhost.methods").unwrap(); - { - let mut metric = method_counter.metric.lock().unwrap(); - client.export(&mut [&mut *metric]).unwrap(); - } - - let mut chain = Chain::new(|_: &mut Request| Ok(Response::with((status::Ok, "Hello World!")))); - chain.link_before(method_counter); - - println!("Listening on http://{}", URL); - println!("Counter mapped at {}", client.mmv_path().to_str().unwrap()); - - Iron::new(chain).http(URL).unwrap(); -} diff --git a/examples/physical.rs b/examples/physical.rs index c28bc11..a8215ba 100644 --- a/examples/physical.rs +++ b/examples/physical.rs @@ -1,9 +1,6 @@ -extern crate hornet; -extern crate rand; - use hornet::client::metric::*; use hornet::client::Client; -use rand::{thread_rng, Rng}; +use rand::Rng; use std::thread; use std::time::Duration; @@ -22,18 +19,18 @@ fn main() { let hz = Unit::new().time(Time::Sec, -1).unwrap(); let mut freq = Metric::new( - "frequency", // name (max 63 bytes) - thread_rng().gen::(), // initial value - Semantics::Instant, // semantics - hz, // unit - "", // optional short description (max 255 bytes) - "", // optional long description (max 255 bytes) + "frequency", // name (max 63 bytes) + rand::thread_rng().gen::(), // initial value + Semantics::Instant, // semantics + hz, // unit + "", // optional short description (max 255 bytes) + "", // optional long description (max 255 bytes) ) .unwrap(); let mut photons = Metric::new( "photons", - thread_rng().gen::(), + rand::thread_rng().gen::(), Semantics::Counter, Unit::new().count(Count::One, 1).unwrap(), "No. of photons", @@ -53,8 +50,8 @@ fn main() { color.set_val(String::from("magenta")).unwrap(); loop { - freq.set_val(thread_rng().gen::()).unwrap(); - photons.set_val(thread_rng().gen::()).unwrap(); + freq.set_val(rand::thread_rng().gen::()).unwrap(); + photons.set_val(rand::thread_rng().gen::()).unwrap(); thread::sleep(Duration::from_secs(1)); } diff --git a/src/bin/mmvdump.rs b/src/bin/mmvdump.rs index 9d74cc0..8f35fa2 100644 --- a/src/bin/mmvdump.rs +++ b/src/bin/mmvdump.rs @@ -1,5 +1,3 @@ -extern crate hornet; - use hornet::mmv; use std::env; use std::path::Path; diff --git a/src/client/metric/histogram.rs b/src/client/metric/histogram.rs index 3a8e521..7a09257 100644 --- a/src/client/metric/histogram.rs +++ b/src/client/metric/histogram.rs @@ -1,10 +1,10 @@ use super::*; -use hdrsample; -use hdrsample::Histogram as HdrHist; +use hdrhistogram; +use hdrhistogram::Histogram as HdrHist; /// A histogram metric that records data and reports statistics /// -/// Internally backed by a [HDR Histogram](https://github.com/jonhoo/hdrsample), +/// Internally backed by a [HDR Histogram](https://github.com/HdrHistogram/HdrHistogram_rust), /// much of API and documentation being borrowed from it. /// /// Exports the `max`, `min`, `mean` and `stdev` statistics to an MMV @@ -28,7 +28,7 @@ pub enum CreationError { /// Instance error Instance(String), /// HDR Histogram creation error - HdrHist(hdrsample::CreationError), + HdrHist(hdrhistogram::CreationError), } impl From for CreationError { @@ -37,8 +37,8 @@ impl From for CreationError { } } -impl From for CreationError { - fn from(err: hdrsample::CreationError) -> CreationError { +impl From for CreationError { + fn from(err: hdrhistogram::CreationError) -> CreationError { CreationError::HdrHist(err) } } @@ -49,7 +49,7 @@ pub enum RecordError { /// IO error Io(io::Error), /// HDR histogram record error - HdrHist(hdrsample::RecordError), + HdrHist(hdrhistogram::RecordError), } impl From for RecordError { @@ -58,8 +58,8 @@ impl From for RecordError { } } -impl From for RecordError { - fn from(err: hdrsample::RecordError) -> RecordError { +impl From for RecordError { + fn from(err: hdrhistogram::RecordError) -> RecordError { RecordError::HdrHist(err) } } @@ -145,11 +145,11 @@ impl Histogram { } /// Total number of samples recorded so far pub fn count(&self) -> u64 { - self.histogram.count() + self.histogram.len() } /// Number of distinct values that can currently be represented pub fn len(&self) -> usize { - self.histogram.len() + self.histogram.distinct_values() } /// Lowest recorded value @@ -222,8 +222,7 @@ impl MMVWriter for Histogram { #[test] pub fn test() { use super::super::Client; - use rand::distributions::{IndependentSample, Range}; - use rand::{thread_rng, Rng}; + use rand::Rng; let low = 1; let high = 60 * 60 * 1000; @@ -236,14 +235,13 @@ pub fn test() { .export(&mut [&mut hist]) .unwrap(); - let val_range = Range::new(low, high); - let mut rng = thread_rng(); + let mut rng = rand::thread_rng(); - let n = thread_rng().gen::() % 100; + let n = rng.gen::() % 100; for _ in 0..n { - hist.record(val_range.ind_sample(&mut rng)).unwrap(); + hist.record(rng.gen_range(low..high)).unwrap(); } - hist.record_n(val_range.ind_sample(&mut rng), n).unwrap(); + hist.record_n(rng.gen_range(low..high), n).unwrap(); assert_eq!(*hist.im.val(MIN_INST).unwrap(), hist.histogram.min() as f64); diff --git a/src/client/metric/mod.rs b/src/client/metric/mod.rs index 4cab41d..2452515 100644 --- a/src/client/metric/mod.rs +++ b/src/client/metric/mod.rs @@ -8,6 +8,7 @@ use std::io; use std::io::{Cursor, Write}; use std::mem; use std::str; +use std::sync::LazyLock; use super::super::mmv::{MTCode, Version}; use super::super::{ @@ -222,7 +223,7 @@ pub(super) use self::private::MetricType; pub(super) use self::private::{MMVWriter, MMVWriterState, MmapView}; macro_rules! impl_metric_type_for ( - ($typ:tt, $base_typ:tt, $type_code:expr) => ( + ($typ:tt, $base_typ:tt, $type_code:expr_2021) => ( impl MetricType for $typ { private_impl!{} @@ -405,7 +406,7 @@ const COUNT_SCALE_LSB: u8 = 8; const LS_FOUR_BIT_MASK: u32 = 0xF; macro_rules! check_dim ( - ($dim:expr) => ( + ($dim:expr_2021) => ( if $dim > 7 || $dim < -8 { return Err(format!("Unit dimension {} is out of range [-8, 7]", $dim)) } @@ -491,7 +492,7 @@ impl Unit { } macro_rules! write_dim ( - ($dim:expr, $scale:expr, $scale_type:tt, $f:expr) => ( + ($dim:expr_2021, $scale:expr_2021, $scale_type:tt, $f:expr_2021) => ( if let Some(dim_scale) = $scale_type::from_u8($scale) { write!($f, "{}", dim_scale)?; if $dim.abs() > 1 { @@ -584,9 +585,8 @@ pub struct Metric { mmap_view: MmapView, } -lazy_static! { - static ref SCRATCH_VIEW: MmapView = MmapView::anonymous(STRING_BLOCK_LEN as usize).unwrap(); -} +static SCRATCH_VIEW: LazyLock = + LazyLock::new(|| MmapView::anonymous(STRING_BLOCK_LEN as usize).unwrap()); impl Metric { /// Creates a new PCP MMV Metric @@ -1285,17 +1285,19 @@ fn test_units() { assert!(Unit::new().time(Time::Sec, -9).is_err()); } +#[cfg(test)] +fn random_ascii(len: usize) -> String { + use rand::distributions::{Alphanumeric, DistString}; + + Alphanumeric.sample_string(&mut rand::thread_rng(), len) +} + #[test] fn test_invalid_strings() { - use rand::{thread_rng, Rng}; - let sem = Semantics::Discrete; let unit = Unit::new(); - let invalid_string: String = thread_rng() - .gen_ascii_chars() - .take(STRING_BLOCK_LEN as usize) - .collect(); + let invalid_string = random_ascii(STRING_BLOCK_LEN as usize); assert!(Metric::new(&invalid_string, 0, sem, unit, "", "").is_err()); assert!(Metric::new("", 0, sem, unit, &invalid_string, "").is_err()); @@ -1313,19 +1315,11 @@ fn test_invalid_strings() { #[test] fn test_mmv2_string_check() { - use rand::{thread_rng, Rng}; - let sem = Semantics::Discrete; let unit = Unit::new(); - let mmv1_string: String = thread_rng() - .gen_ascii_chars() - .take((MMV1_NAME_MAX_LEN - 1) as usize) - .collect(); - let mmv2_string: String = thread_rng() - .gen_ascii_chars() - .take((STRING_BLOCK_LEN - 1) as usize) - .collect(); + let mmv1_string = random_ascii((MMV1_NAME_MAX_LEN - 1) as usize); + let mmv2_string = random_ascii((STRING_BLOCK_LEN - 1) as usize); let mmv1_metric = Metric::new(&mmv1_string, 0, sem, unit, "", "").unwrap(); assert_eq!(mmv1_metric.has_mmv2_string(), false); @@ -1351,15 +1345,11 @@ fn test_mmv2_string_check() { fn test_mmv2_string_blocks() { use super::super::mmv::*; use super::Client; - use rand::{thread_rng, Rng}; let sem = Semantics::Discrete; let unit = Unit::new(); - let mmv2_string: String = thread_rng() - .gen_ascii_chars() - .take((STRING_BLOCK_LEN - 1) as usize) - .collect(); + let mmv2_string = random_ascii((STRING_BLOCK_LEN - 1) as usize); let mut metric = Metric::new(&mmv2_string, 0, sem, unit, "", "").unwrap(); let indom = Indom::new(&[&mmv2_string], "", "").unwrap(); @@ -1399,31 +1389,19 @@ fn test_mmv2_string_blocks() { fn test_random_numeric_metrics() { use super::Client; use crate::byteio::ReadBytesExt; - use rand::{thread_rng, Rng}; + use rand::Rng; let mut metrics = Vec::new(); let mut new_vals = Vec::new(); - let n_metrics = thread_rng().gen::() % 20; + let n_metrics = rand::thread_rng().gen::() % 20; let client = Client::new("numeric_metrics").unwrap(); for _ in 1..n_metrics { - let rnd_name: String = thread_rng() - .gen_ascii_chars() - .take(MMV1_NAME_MAX_LEN as usize - 1) - .collect(); - - let rnd_shorthelp: String = thread_rng() - .gen_ascii_chars() - .take(STRING_BLOCK_LEN as usize - 1) - .collect(); - - let rnd_longhelp: String = thread_rng() - .gen_ascii_chars() - .take(STRING_BLOCK_LEN as usize - 1) - .collect(); - - let rnd_val1 = thread_rng().gen::(); + let rnd_name = random_ascii(MMV1_NAME_MAX_LEN as usize - 1); + let rnd_shorthelp = random_ascii(STRING_BLOCK_LEN as usize - 1); + let rnd_longhelp = random_ascii(STRING_BLOCK_LEN as usize - 1); + let rnd_val1 = rand::thread_rng().gen::(); let mut metric = Metric::new( &rnd_name, @@ -1437,20 +1415,22 @@ fn test_random_numeric_metrics() { assert_eq!(*metric.val(), rnd_val1); - let rnd_val2 = thread_rng().gen::(); + let rnd_val2 = rand::thread_rng().gen::(); assert!(metric.set_val(rnd_val2).is_ok()); assert_eq!(*metric.val(), rnd_val2); metrics.push(metric); - new_vals.push(thread_rng().gen::()); + new_vals.push(rand::thread_rng().gen::()); } { // mmv_writers needs to go out of scope before we can mutate // the metrics after exporting. The type annotation is needed // because type inference fails. - let mut mmv_writers: Vec<&mut MMVWriter> = - metrics.iter_mut().map(|m| m as &mut MMVWriter).collect(); + let mut mmv_writers: Vec<&mut dyn MMVWriter> = metrics + .iter_mut() + .map(|m| m as &mut dyn MMVWriter) + .collect(); client.export(&mut mmv_writers).unwrap(); } @@ -1469,7 +1449,7 @@ fn test_random_numeric_metrics() { fn test_simple_metrics() { use super::Client; use crate::byteio::ReadBytesExt; - use rand::{thread_rng, Rng}; + use rand::Rng; use std::ffi::CStr; use std::mem::transmute; @@ -1477,7 +1457,7 @@ fn test_simple_metrics() { let hz = Unit::new().time(Time::Sec, -1).unwrap(); let mut freq = Metric::new( "frequency", - thread_rng().gen::(), + rand::thread_rng().gen::(), Semantics::Instant, hz, "", @@ -1499,7 +1479,7 @@ fn test_simple_metrics() { // u32 metric let mut photons = Metric::new( "photons", - thread_rng().gen::(), + rand::thread_rng().gen::(), Semantics::Counter, Unit::new().count(Count::One, 1).unwrap(), "No. of photons", @@ -1512,13 +1492,13 @@ fn test_simple_metrics() { .export(&mut [&mut freq, &mut color, &mut photons]) .unwrap(); - let new_freq = thread_rng().gen::(); + let new_freq = rand::thread_rng().gen::(); assert!(freq.set_val(new_freq).is_ok()); let new_color = String::from("magenta"); assert!(color.set_val(new_color.clone()).is_ok()); - let new_photon_count = thread_rng().gen::(); + let new_photon_count = rand::thread_rng().gen::(); assert!(photons.set_val(new_photon_count).is_ok()); let freq_bytes = freq.mmap_view.to_vec(); diff --git a/src/client/mod.rs b/src/client/mod.rs index 383b335..3311f1d 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -1,4 +1,5 @@ use crate::byteio::WriteBytesExt; +use bitflags::bitflags; use memmap2::MmapMut; use std::env; use std::ffi::{OsStr, OsString}; @@ -54,22 +55,27 @@ fn init_pcp_conf(pcp_root: &Path) -> io::Result<()> { /* attempt to load variables from pcp_root/etc/pcp.conf into environment. if pcp_root/etc/pcp.conf is not a file, can't be read, or parsing it fails, we *don't* return the error */ - parse_pcp_conf(pcp_root.join("etc").join("pcp.conf")).ok(); + if let Ok(values) = parse_pcp_conf(pcp_root.join("etc").join("pcp.conf")) { + apply_pcp_conf(values); + } /* attempt to load variables from pcp_root/$PCP_CONF into environment. if pcp_root/$PCP_CONF is not a file, can't be read, or parsing it fails, we *do* return the error */ let pcp_conf = pcp_root.join(env::var_os("PCP_CONF").unwrap_or(OsString::new())); - parse_pcp_conf(pcp_conf) + let values = parse_pcp_conf(pcp_conf)?; + apply_pcp_conf(values); + Ok(()) } /// Parses one `PCP_VARIABLE_NAME=value` line, per the syntax in /// `man 5 pcp.conf`: no space around the `=`, and values are unquoted and /// may contain spaces. /// -/// Returns `None` for anything else, including an unterminated last line. +/// Returns `None` for anything else. fn parse_pcp_conf_line(line: &[u8]) -> Option<(&[u8], &[u8])> { - let line = line.strip_suffix(b"\n")?; + let line = line.strip_suffix(b"\n").unwrap_or(line); + let line = line.strip_suffix(b"\r").unwrap_or(line); let eq = line.iter().position(|&b| b == b'=')?; let (key, val) = (&line[..eq], &line[eq + 1..]); @@ -82,32 +88,43 @@ fn parse_pcp_conf_line(line: &[u8]) -> Option<(&[u8], &[u8])> { return None; } - if val.len() < 2 { - return None; - } let is_quote = |b: u8| b == b'"' || b == b'\''; - if is_quote(val[0]) || is_quote(val[val.len() - 1]) { + if val.first().map_or(false, |&b| is_quote(b)) { return None; } Some((key, val)) } -fn parse_pcp_conf>(conf_path: P) -> io::Result<()> { +fn parse_pcp_conf>(conf_path: P) -> io::Result> { let pcp_conf = File::open(conf_path)?; let mut buf_reader = BufReader::new(pcp_conf); + let mut values = Vec::new(); let mut line = Vec::new(); while buf_reader.read_until(b'\n', &mut line)? > 0 { if let Some((key, val)) = parse_pcp_conf_line(&line) { if let (Some(key), Some(val)) = (osstr_from_bytes(key), osstr_from_bytes(val)) { - env::set_var(key, val); + values.push((key.to_os_string(), val.to_os_string())); } } line.clear(); } - Ok(()) + Ok(values) +} + +fn unset_pcp_conf_values( + values: impl IntoIterator, + is_set: impl Fn(&OsStr) -> bool, +) -> Vec<(OsString, OsString)> { + values.into_iter().filter(|(key, _)| !is_set(key)).collect() +} + +fn apply_pcp_conf(values: Vec<(OsString, OsString)>) { + for (key, val) in unset_pcp_conf_values(values, |key| env::var_os(key).is_some()) { + env::set_var(key, val); + } } fn get_mmv_dir() -> io::Result { @@ -141,6 +158,7 @@ fn get_mmv_dir() -> io::Result { bitflags! { /// Flags used to modify how a client exports metrics + #[derive(Clone, Copy)] pub struct MMVFlags: u32 { /// Metric names aren't prefixed with MMV filename const NOPREFIX = 1; @@ -155,12 +173,12 @@ impl fmt::Display for MMVFlags { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let mut prev_flag = false; - if self.contains(NOPREFIX) { + if self.contains(MMVFlags::NOPREFIX) { write!(f, "no prefix")?; prev_flag = true; } - if self.contains(PROCESS) { + if self.contains(MMVFlags::PROCESS) { if prev_flag { write!(f, ",")?; } @@ -168,7 +186,7 @@ impl fmt::Display for MMVFlags { prev_flag = true; } - if self.contains(SENTINEL) { + if self.contains(MMVFlags::SENTINEL) { if prev_flag { write!(f, ",")?; } @@ -194,7 +212,7 @@ pub struct Client { impl Client { /// Creates a new client with `PROCESS` flag and `0` cluster ID pub fn new(name: &str) -> io::Result { - Client::new_custom(name, PROCESS, 0) + Client::new_custom(name, MMVFlags::PROCESS, 0) } /// Creates a new client with custom flags and cluster ID @@ -216,7 +234,7 @@ impl Client { /// /// If an MMV file is already present at `mmv_path`, it's overwritten /// with the newer metrics. - pub fn export(&self, metrics: &mut [&mut MMVWriter]) -> io::Result<()> { + pub fn export(&self, metrics: &mut [&mut dyn MMVWriter]) -> io::Result<()> { let mut ws = MMVWriterState::new(); let mut mmv_ver = Version::V1; @@ -386,10 +404,10 @@ fn write_toc_block( #[test] fn test_mmv_header() { use crate::byteio::ReadBytesExt; - use rand::{thread_rng, Rng}; + use rand::Rng; - let cluster_id = thread_rng().gen::(); - let flags = PROCESS | SENTINEL; + let cluster_id = rand::thread_rng().gen::(); + let flags = MMVFlags::PROCESS | MMVFlags::SENTINEL; let client = Client::new_custom("mmv_header_test", flags, cluster_id).unwrap(); client.export(&mut []).unwrap(); @@ -431,47 +449,39 @@ fn test_mmv_dir() { } #[test] -fn test_init_pcp_conf() { - let conf_keys = vec![ - "PCP_VERSION", - "PCP_USER", - "PCP_GROUP", - "PCP_PLATFORM", - "PCP_PLATFORM_PATHS", - "PCP_ETC_DIR", - "PCP_SYSCONF_DIR", - "PCP_SYSCONFIG_DIR", - "PCP_RC_DIR", - "PCP_BIN_DIR", - "PCP_BINADM_DIR", - "PCP_LIB_DIR", - "PCP_LIB32_DIR", - "PCP_SHARE_DIR", - "PCP_INC_DIR", - "PCP_MAN_DIR", - "PCP_PMCDCONF_PATH", - "PCP_PMCDOPTIONS_PATH", - "PCP_PMCDRCLOCAL_PATH", - "PCP_PMPROXYOPTIONS_PATH", - "PCP_PMWEBDOPTIONS_PATH", - "PCP_PMMGROPTIONS_PATH", - "PCP_PMIECONTROL_PATH", - "PCP_PMSNAPCONTROL_PATH", - "PCP_PMLOGGERCONTROL_PATH", - "PCP_PMDAS_DIR", - "PCP_RUN_DIR", - "PCP_PMDAS_DIR", - "PCP_LOG_DIR", - "PCP_TMP_DIR", - "PCP_TMPFILE_DIR", - "PCP_DOC_DIR", - "PCP_DEMOS_DIR", - ]; - - let pcp_root = get_pcp_root(); - if init_pcp_conf(&pcp_root).is_ok() { - for key in conf_keys.iter() { - env::var(key).expect(&format!("{} not set", key)); - } - } +fn test_parse_pcp_conf_fixture() { + let tmp = tempfile::tempdir().unwrap(); + let conf = tmp.path().join("pcp.conf"); + fs::write( + &conf, + b"# comments and blank lines are ignored\r\n\ + PCP_TMP_DIR=/tmp/from-file\r\n\ + PCP_USER=hornet\r\n\ + PCP_EMPTY=\r\n\ + PCP_QUOTED=\"not accepted\"\r\n\ + NOT_PCP=ignored\r\n", + ) + .unwrap(); + + let values = parse_pcp_conf(&conf).unwrap(); + assert_eq!( + values, + vec![ + ( + OsString::from("PCP_TMP_DIR"), + OsString::from("/tmp/from-file"), + ), + (OsString::from("PCP_USER"), OsString::from("hornet")), + (OsString::from("PCP_EMPTY"), OsString::new()), + ] + ); + + let values_to_set = unset_pcp_conf_values(values, |key| key == OsStr::new(PCP_TMP_DIR_KEY)); + assert_eq!( + values_to_set, + vec![ + (OsString::from("PCP_USER"), OsString::from("hornet")), + (OsString::from("PCP_EMPTY"), OsString::new()), + ] + ); } diff --git a/src/lib.rs b/src/lib.rs index b6ec9e0..0db960d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,12 +1,3 @@ -extern crate hdrsample; -extern crate memmap2; -#[macro_use] -extern crate bitflags; -#[macro_use] -extern crate lazy_static; -#[cfg(test)] -extern crate rand; - const CLUSTER_ID_BIT_LEN: usize = 12; const ITEM_BIT_LEN: usize = 10; const INDOM_BIT_LEN: usize = 22; diff --git a/src/mmv/mod.rs b/src/mmv/mod.rs index 3b0fe7b..4e3a063 100644 --- a/src/mmv/mod.rs +++ b/src/mmv/mod.rs @@ -111,7 +111,7 @@ impl From for MMVDumpError { } macro_rules! return_mmvdumperror ( - ($err:expr, $val:expr) => ( + ($err:expr_2021, $val:expr_2021) => ( let mut err_str = $err.to_owned(); err_str.push_str(&format!(": {}", $val)); return Err(MMVDumpError::InvalidMMV(err_str)); @@ -676,7 +676,7 @@ impl StringBlk { } macro_rules! blks_from_toc ( - ($toc:expr, $blk_typ:tt, $cursor:expr) => { + ($toc:expr_2021, $blk_typ:tt, $cursor:expr_2021) => { if let Some(ref toc) = $toc { let mut blks = BTreeMap::new(); @@ -692,7 +692,7 @@ macro_rules! blks_from_toc ( BTreeMap::new() } }; - ($toc:expr, $blk_typ:tt, $mmv_ver:expr, $cursor:expr) => { + ($toc:expr_2021, $blk_typ:tt, $mmv_ver:expr_2021, $cursor:expr_2021) => { if let Some(ref toc) = $toc { let mut blks = BTreeMap::new(); diff --git a/src/private.rs b/src/private.rs index a48dad0..18c3a59 100644 --- a/src/private.rs +++ b/src/private.rs @@ -15,14 +15,14 @@ macro_rules! private_decl { /// This trait is private; this method exists to make it /// impossible to implement outside the crate. #[doc(hidden)] - fn __rayon_private__(&self) -> ::private::PrivateMarker; + fn __rayon_private__(&self) -> $crate::private::PrivateMarker; }; } macro_rules! private_impl { () => { - fn __rayon_private__(&self) -> ::private::PrivateMarker { - ::private::PrivateMarker + fn __rayon_private__(&self) -> $crate::private::PrivateMarker { + $crate::private::PrivateMarker } }; } diff --git a/tests/mmvfmt.rs b/tests/mmvfmt.rs index 285f965..dbdd496 100644 --- a/tests/mmvfmt.rs +++ b/tests/mmvfmt.rs @@ -1,5 +1,3 @@ -extern crate hornet; - use hornet::mmv; use std::fs; use std::fs::File;