├── .gitignore ├── src └── lib.rs ├── .rustfmt.toml ├── impl ├── Cargo.toml └── src │ └── lib.rs ├── README.md ├── Cargo.toml ├── .github └── workflows │ ├── release.yml │ └── ci.yml ├── LICENSE.md └── tests └── tests.rs /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | pub use flaky_test_impl::flaky_test; 2 | 3 | #[doc(hidden)] 4 | pub use ::futures_util; 5 | -------------------------------------------------------------------------------- /.rustfmt.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2018-2021 the Deno authors. All rights reserved. MIT license. 2 | max_width = 80 3 | tab_spaces = 2 4 | edition = "2018" 5 | -------------------------------------------------------------------------------- /impl/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "flaky_test_impl" 3 | version = "0.2.2" 4 | authors = ["the Deno authors"] 5 | edition = "2021" 6 | license = "MIT" 7 | repository = "https://github.com/denoland/flaky_test" 8 | description = "implementation detail of the `flaky_test` macro" 9 | 10 | [lib] 11 | proc-macro = true 12 | 13 | [dependencies] 14 | proc-macro2 = "1" 15 | quote = "1" 16 | syn = { version = "1", features = ["full"] } 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # flaky_test 2 | 3 | [![crates](https://img.shields.io/crates/v/flaky_test.svg)](https://crates.io/crates/flaky_test) 4 | [![docs](https://docs.rs/flaky_test/badge.svg)](https://docs.rs/flaky_test) 5 | 6 | This attribute macro will register and run a test 3 times, erroring only if all 7 | three times fail. Useful for situations when a test is flaky. 8 | 9 | ```rust 10 | #[flaky_test::flaky_test] 11 | fn my_test() { 12 | assert_eq!(1, 2); 13 | } 14 | ``` 15 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "flaky_test" 3 | version = "0.2.2" 4 | authors = ["the Deno authors"] 5 | edition = "2021" 6 | license = "MIT" 7 | repository = "https://github.com/denoland/flaky_test" 8 | description = "atttribute macro for running a flaky test multiple times" 9 | 10 | [workspace] 11 | members = ["impl"] 12 | 13 | [dependencies] 14 | flaky_test_impl = { version = "0.2.2", path = "impl" } 15 | futures-util = { version = "0.3", default-features = false, features = ["std"] } 16 | 17 | [dev-dependencies] 18 | tokio = { version = "1", default-features = false, features = ["rt", "rt-multi-thread", "macros"] } 19 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: release 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | releaseKind: 7 | description: 'Kind of release' 8 | type: choice 9 | options: 10 | - patch 11 | - minor 12 | - major 13 | required: true 14 | 15 | jobs: 16 | rust: 17 | name: release 18 | runs-on: ubuntu-latest 19 | timeout-minutes: 30 20 | 21 | steps: 22 | - name: Clone repository 23 | uses: actions/checkout@v4 24 | with: 25 | token: ${{ secrets.DENOBOT_PAT }} 26 | 27 | - uses: denoland/setup-deno@v1 28 | - uses: dtolnay/rust-toolchain@stable 29 | 30 | - name: Tag and release 31 | env: 32 | GITHUB_TOKEN: ${{ secrets.DENOBOT_PAT }} 33 | GH_WORKFLOW_ACTOR: ${{ github.actor }} 34 | run: | 35 | git config user.email "denobot@users.noreply.github.com" 36 | git config user.name "denobot" 37 | deno run -A https://raw.githubusercontent.com/denoland/automation/0.20.0/tasks/publish_release.ts --${{github.event.inputs.releaseKind}} flaky_test 38 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright 2018-2021 the Deno authors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | 3 | on: 4 | pull_request: 5 | branches: [main] 6 | push: 7 | branches: [main] 8 | tags: 9 | - '*' 10 | 11 | jobs: 12 | rust: 13 | runs-on: ubuntu-latest 14 | timeout-minutes: 30 15 | 16 | env: 17 | GH_ACTIONS: 1 18 | RUST_BACKTRACE: full 19 | RUSTFLAGS: -D warnings 20 | 21 | steps: 22 | - name: Clone repository 23 | uses: actions/checkout@v4 24 | 25 | - name: Install rust 26 | uses: dtolnay/rust-toolchain@stable 27 | with: 28 | toolchain: stable 29 | components: clippy,rustfmt 30 | 31 | - name: Format 32 | run: cargo fmt --all --check 33 | 34 | - name: Lint 35 | run: cargo clippy --all-targets --all-features --release 36 | 37 | - name: Test 38 | run: cargo test --release --all-targets --all-features 39 | 40 | - name: Build 41 | run: cargo build --release --all-targets --all-features 42 | 43 | - name: Publish 44 | if: | 45 | github.repository == 'denoland/flaky_test' && 46 | startsWith(github.ref, 'refs/tags/') 47 | env: 48 | CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} 49 | run: | 50 | cargo publish --package flaky_test_impl 51 | cargo publish 52 | -------------------------------------------------------------------------------- /tests/tests.rs: -------------------------------------------------------------------------------- 1 | use flaky_test::flaky_test; 2 | use std::sync::atomic::AtomicUsize; 3 | use std::sync::atomic::Ordering; 4 | 5 | #[flaky_test] 6 | fn assert_true() { 7 | println!("should pass"); 8 | } 9 | 10 | #[flaky_test] 11 | fn fail_first_two_times() { 12 | static C: AtomicUsize = AtomicUsize::new(0); 13 | if C.fetch_add(1, Ordering::SeqCst) < 2 { 14 | panic!("flaky"); 15 | } 16 | assert_eq!(3, C.load(Ordering::SeqCst)); 17 | } 18 | 19 | #[flaky_test(times = 10)] 20 | fn fail_first_nine_times() { 21 | static C: AtomicUsize = AtomicUsize::new(0); 22 | if C.fetch_add(1, Ordering::SeqCst) < 9 { 23 | panic!("flaky"); 24 | } 25 | assert_eq!(10, C.load(Ordering::SeqCst)); 26 | } 27 | 28 | #[flaky_test] 29 | #[should_panic] 30 | fn fail_three_times() { 31 | panic!("should panic"); 32 | } 33 | 34 | #[flaky_test(times = 10)] 35 | #[should_panic] 36 | fn fail_ten_times() { 37 | panic!("should panic"); 38 | } 39 | 40 | #[flaky_test(tokio)] 41 | async fn tokio_basic() { 42 | let fut = std::future::ready(42); 43 | assert_eq!(fut.await, 42); 44 | } 45 | 46 | #[flaky_test(tokio(flavor = "multi_thread", worker_threads = 2))] 47 | async fn tokio_complex() { 48 | let fut = std::future::ready(42); 49 | assert_eq!(fut.await, 42); 50 | } 51 | 52 | #[flaky_test(tokio, times = 5)] 53 | async fn tokio_with_times() { 54 | let fut = std::future::ready(42); 55 | assert_eq!(fut.await, 42); 56 | } 57 | 58 | #[flaky_test(tokio)] 59 | #[should_panic] 60 | async fn tokio_with_should_panic() { 61 | let fut = std::future::ready(0); 62 | assert_eq!(fut.await, 42); 63 | } 64 | -------------------------------------------------------------------------------- /impl/src/lib.rs: -------------------------------------------------------------------------------- 1 | use proc_macro::TokenStream; 2 | use quote::quote; 3 | use syn::parse::Parser as _; 4 | use syn::punctuated::Punctuated; 5 | use syn::Attribute; 6 | use syn::ItemFn; 7 | use syn::Lit; 8 | use syn::Meta; 9 | use syn::MetaList; 10 | use syn::MetaNameValue; 11 | use syn::NestedMeta; 12 | use syn::Token; 13 | 14 | struct FlakyTestArgs { 15 | times: usize, 16 | runtime: Runtime, 17 | } 18 | 19 | enum Runtime { 20 | Sync, 21 | Tokio(Option>), 22 | } 23 | 24 | impl Default for FlakyTestArgs { 25 | fn default() -> Self { 26 | FlakyTestArgs { 27 | times: 3, 28 | runtime: Runtime::Sync, 29 | } 30 | } 31 | } 32 | 33 | fn parse_attr(attr: proc_macro2::TokenStream) -> syn::Result { 34 | let parser = Punctuated::::parse_terminated; 35 | let punctuated = parser.parse2(attr)?; 36 | 37 | let mut ret = FlakyTestArgs::default(); 38 | 39 | for meta in punctuated { 40 | match meta { 41 | Meta::Path(path) => { 42 | if path.is_ident("tokio") { 43 | ret.runtime = Runtime::Tokio(None); 44 | } else { 45 | return Err(syn::Error::new_spanned(path, "expected `tokio`")); 46 | } 47 | } 48 | Meta::NameValue(MetaNameValue { 49 | path, 50 | lit: Lit::Int(lit_int), 51 | .. 52 | }) => { 53 | if path.is_ident("times") { 54 | ret.times = lit_int.base10_parse::()?; 55 | } else { 56 | return Err(syn::Error::new_spanned( 57 | path, 58 | "expected `times = `", 59 | )); 60 | } 61 | } 62 | Meta::List(MetaList { path, nested, .. }) => { 63 | if path.is_ident("tokio") { 64 | ret.runtime = Runtime::Tokio(Some(nested)); 65 | } else { 66 | return Err(syn::Error::new_spanned(path, "expected `tokio`")); 67 | } 68 | } 69 | _ => { 70 | return Err(syn::Error::new_spanned( 71 | meta, 72 | "expected `times = ` or `tokio`", 73 | )); 74 | } 75 | } 76 | } 77 | 78 | Ok(ret) 79 | } 80 | 81 | /// A flaky test will be run multiple times until it passes. 82 | /// 83 | /// # Example 84 | /// 85 | /// ```rust 86 | /// use flaky_test::flaky_test; 87 | /// 88 | /// // By default it will be retried up to 3 times. 89 | /// #[flaky_test] 90 | /// fn test_default() { 91 | /// println!("should pass"); 92 | /// } 93 | /// 94 | /// // The number of max attempts can be adjusted via `times`. 95 | /// #[flaky_test(times = 5)] 96 | /// fn usage_with_named_args() { 97 | /// println!("should pass"); 98 | /// } 99 | /// 100 | /// # use std::convert::Infallible; 101 | /// # async fn async_operation() -> Result { 102 | /// # Ok(42) 103 | /// # } 104 | /// // Async tests can be run by passing `tokio`. 105 | /// // Make sure `tokio` is added in your `Cargo.toml`. 106 | /// #[flaky_test(tokio)] 107 | /// async fn async_test() { 108 | /// let res = async_operation().await.unwrap(); 109 | /// assert_eq!(res, 42); 110 | /// } 111 | /// 112 | /// // `tokio` and `times` can be combined. 113 | /// #[flaky_test(tokio, times = 5)] 114 | /// async fn async_test_five_times() { 115 | /// let res = async_operation().await.unwrap(); 116 | /// assert_eq!(res, 42); 117 | /// } 118 | /// 119 | /// // Any arguments that `#[tokio::test]` supports can be specified. 120 | /// #[flaky_test(tokio(flavor = "multi_thraed", worker_threads = 2))] 121 | /// async fn async_test_complex() { 122 | /// let res = async_operation().await.unwrap(); 123 | /// assert_eq!(res, 42); 124 | /// } 125 | /// ``` 126 | #[proc_macro_attribute] 127 | pub fn flaky_test(attr: TokenStream, input: TokenStream) -> TokenStream { 128 | let attr = proc_macro2::TokenStream::from(attr); 129 | let mut input = proc_macro2::TokenStream::from(input); 130 | 131 | match inner(attr, input.clone()) { 132 | Err(e) => { 133 | input.extend(e.into_compile_error()); 134 | input.into() 135 | } 136 | Ok(t) => t.into(), 137 | } 138 | } 139 | 140 | fn inner( 141 | attr: proc_macro2::TokenStream, 142 | input: proc_macro2::TokenStream, 143 | ) -> syn::Result { 144 | let args = parse_attr(attr)?; 145 | let input_fn: ItemFn = syn::parse2(input)?; 146 | let attrs = input_fn.attrs.clone(); 147 | 148 | match args.runtime { 149 | Runtime::Sync => sync(input_fn, attrs, args.times), 150 | Runtime::Tokio(tokio_args) => { 151 | tokio(input_fn, attrs, args.times, tokio_args) 152 | } 153 | } 154 | } 155 | 156 | fn sync( 157 | input_fn: ItemFn, 158 | attrs: Vec, 159 | times: usize, 160 | ) -> syn::Result { 161 | let fn_name = input_fn.sig.ident.clone(); 162 | 163 | Ok(quote! { 164 | #[test] 165 | #(#attrs)* 166 | fn #fn_name() { 167 | #input_fn 168 | 169 | for i in 0..#times { 170 | println!("flaky_test retry {}", i); 171 | let r = ::std::panic::catch_unwind(|| { 172 | #fn_name(); 173 | }); 174 | if r.is_ok() { 175 | return; 176 | } 177 | if i == #times - 1 { 178 | ::std::panic::resume_unwind(r.unwrap_err()); 179 | } 180 | } 181 | } 182 | }) 183 | } 184 | 185 | fn tokio( 186 | input_fn: ItemFn, 187 | attrs: Vec, 188 | times: usize, 189 | tokio_args: Option>, 190 | ) -> syn::Result { 191 | if input_fn.sig.asyncness.is_none() { 192 | return Err(syn::Error::new_spanned(input_fn.sig, "must be `async fn`")); 193 | } 194 | 195 | let fn_name = input_fn.sig.ident.clone(); 196 | let tokio_macro = match tokio_args { 197 | Some(args) => quote! { #[::tokio::test(#args)] }, 198 | None => quote! { #[::tokio::test] }, 199 | }; 200 | 201 | Ok(quote! { 202 | #tokio_macro 203 | #(#attrs)* 204 | async fn #fn_name() { 205 | #input_fn 206 | 207 | for i in 0..#times { 208 | println!("flaky_test retry {}", i); 209 | let fut = ::std::panic::AssertUnwindSafe(#fn_name()); 210 | let r = <_ as ::flaky_test::futures_util::future::FutureExt>::catch_unwind(fut).await; 211 | if r.is_ok() { 212 | return; 213 | } 214 | if i == #times - 1 { 215 | ::std::panic::resume_unwind(r.unwrap_err()); 216 | } 217 | } 218 | } 219 | }) 220 | } 221 | --------------------------------------------------------------------------------