From fe3b2122cdfa82813d2b40193ce6e33a88b2c1ef Mon Sep 17 00:00:00 2001 From: Julio Biason Date: Tue, 2 Jan 2024 14:30:34 -0300 Subject: [PATCH] =?UTF-8?q?Tokio=20+=20Pest=20actually=20works=20?= =?UTF-8?q?=F0=9F=A4=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pesttokiotest/Cargo.lock | 311 +++++++++++++++++++++++++++++++++++++ pesttokiotest/Cargo.toml | 12 ++ pesttokiotest/README.md | 4 + pesttokiotest/example.csv | 1 + pesttokiotest/src/csv.pest | 7 + pesttokiotest/src/main.rs | 52 +++++++ 6 files changed, 387 insertions(+) create mode 100644 pesttokiotest/Cargo.lock create mode 100644 pesttokiotest/Cargo.toml create mode 100644 pesttokiotest/README.md create mode 100644 pesttokiotest/example.csv create mode 100644 pesttokiotest/src/csv.pest create mode 100644 pesttokiotest/src/main.rs diff --git a/pesttokiotest/Cargo.lock b/pesttokiotest/Cargo.lock new file mode 100644 index 0000000..0e85ea4 --- /dev/null +++ b/pesttokiotest/Cargo.lock @@ -0,0 +1,311 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cc" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cpufeatures" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + +[[package]] +name = "libc" +version = "0.2.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" + +[[package]] +name = "memchr" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" + +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + +[[package]] +name = "object" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "pest" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae9cee2a55a544be8b89dc6848072af97a20f2422603c10865be2a42b580fff5" +dependencies = [ + "memchr", + "thiserror", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81d78524685f5ef2a3b3bd1cafbc9fcabb036253d9b1463e726a91cd16e2dfc2" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68bd1206e71118b5356dae5ddc61c8b11e28b09ef6a31acbd15ea48a28e0c227" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pest_meta" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c747191d4ad9e4a4ab9c8798f1e82a39affe7ef9648390b7e5548d18e099de6" +dependencies = [ + "once_cell", + "pest", + "sha2", +] + +[[package]] +name = "pesttokiotest" +version = "0.1.0" +dependencies = [ + "pest", + "pest_derive", + "tokio", + "tokio-macros", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "proc-macro2" +version = "1.0.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2de98502f212cfcea8d0bb305bd0f49d7ebdd75b64ba0a68f937d888f4e0d6db" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "syn" +version = "2.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89456b690ff72fddcecf231caedbe615c59480c93358a93dfae7fc29e3ebbf0e" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio" +version = "1.35.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" +dependencies = [ + "backtrace", + "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "ucd-trie" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" diff --git a/pesttokiotest/Cargo.toml b/pesttokiotest/Cargo.toml new file mode 100644 index 0000000..02586b8 --- /dev/null +++ b/pesttokiotest/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "pesttokiotest" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +pest = "2.7.5" +pest_derive = "2.7.5" +tokio = { version = "1.35.1", features = ["rt", "macros", "fs"] } +tokio-macros = "2.2.0" diff --git a/pesttokiotest/README.md b/pesttokiotest/README.md new file mode 100644 index 0000000..df58134 --- /dev/null +++ b/pesttokiotest/README.md @@ -0,0 +1,4 @@ +# PestTokioTest + +Testing Pest with Tokio ('cause Pest structures are not `Send`, and I'm +wondering if there is a way around it). diff --git a/pesttokiotest/example.csv b/pesttokiotest/example.csv new file mode 100644 index 0000000..d56ef44 --- /dev/null +++ b/pesttokiotest/example.csv @@ -0,0 +1 @@ +123 123 123 ( 1.2 2.3 4.5 ) ( 123 (2 3 4)) diff --git a/pesttokiotest/src/csv.pest b/pesttokiotest/src/csv.pest new file mode 100644 index 0000000..875e5ba --- /dev/null +++ b/pesttokiotest/src/csv.pest @@ -0,0 +1,7 @@ +csv = { SOI ~ line ~ EOI } + +line = _{ (value | group)+ } +value = @{ (ASCII_ALPHANUMERIC | ".")+ } +group = { "(" ~ line ~ ")" } + +WHITESPACE = _{ (" " | "\t") } diff --git a/pesttokiotest/src/main.rs b/pesttokiotest/src/main.rs new file mode 100644 index 0000000..a63cbd3 --- /dev/null +++ b/pesttokiotest/src/main.rs @@ -0,0 +1,52 @@ +use std::path::Path; + +use pest::Parser; +use pest_derive::Parser; + +#[derive(Parser)] +#[grammar = "csv.pest"] +struct CsvParser; + +#[tokio::main(flavor = "current_thread")] +async fn main() { + let mut args = std::env::args(); + let _ = args.next().unwrap(); + let filename = args.next().unwrap(); + + tokio::spawn(async move { + let content = get(&Path::new(&filename)).await; + println!("{:?}", content); + let parsed = parse(&content); + println!("{:?}", parsed); + }) + .await + .unwrap(); +} + +async fn get(filename: &Path) -> String { + let bytes = tokio::fs::read(filename).await.unwrap(); + String::from_utf8_lossy(&bytes).trim().to_string() +} + +fn parse(content: &str) -> Vec> { + let mut parser = CsvParser::parse(Rule::csv, content).unwrap(); + let root = parser.next().unwrap(); + root.into_inner() + .filter_map(|pair| extract(pair)) + .collect::>() +} + +fn extract(pair: pest::iterators::Pair<'_, Rule>) -> Option> { + match pair.as_rule() { + Rule::value => Some(vec![pair.as_str()]), + Rule::group => { + let local = pair + .into_inner() + .filter_map(|pair| extract(pair)) + .flatten() + .collect::>(); + Some(local) + } + _ => None, + } +}