From bad3e4fc12f521efb8d225684eb48392fb85ee60 Mon Sep 17 00:00:00 2001 From: Julio Biason Date: Sat, 18 Apr 2020 15:13:37 -0300 Subject: [PATCH] Push updated to Joplin, if set up --- Cargo.lock | 24 +++--- Cargo.toml | 4 +- src/main.rs | 20 +++-- src/storage/attachment.rs | 11 +-- src/storage/data.rs | 4 +- src/storage/filesystem.rs | 5 +- src/storage/joplin.rs | 173 ++++++++++++++++++++++++++++---------- src/storage/mod.rs | 3 +- 8 files changed, 171 insertions(+), 73 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3c9aa20..b110ecc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -90,7 +90,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)", - "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -505,7 +505,7 @@ dependencies = [ "http 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", "http-body 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", - "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", @@ -577,11 +577,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "iovec" -version = "0.1.2" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -720,12 +719,13 @@ dependencies = [ [[package]] name = "mio" -version = "0.6.19" +version = "0.6.21" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1493,7 +1493,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", - "mio 0.6.19 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-current-thread 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-executor 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1551,7 +1551,7 @@ dependencies = [ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "mio 0.6.19 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1576,8 +1576,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", - "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "mio 0.6.19 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-reactor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1863,7 +1863,7 @@ dependencies = [ "checksum idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" "checksum idna 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" "checksum indexmap 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a61202fbe46c4a951e9404a720a0180bcf3212c750d735cb5c4ba4dc551299f3" -"checksum iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe6e417e7d0975db6512b90796e8ce223145ac4e33c377e4a42882a0e88bb08" +"checksum iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" "checksum isolang 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "265ef164908329e47e753c769b14cbb27434abf0c41984dca201484022f09ce5" "checksum itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "501266b7edd0174f8530248f87f99c88fbe60ca4ef3dd486835b8d8d53136f7f" "checksum jni 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "294eca097d1dc0bf59de5ab9f7eafa5f77129e9f6464c957ed3ddeb705fb4292" @@ -1882,7 +1882,7 @@ dependencies = [ "checksum mime 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "dd1d63acd1b78403cc0c325605908475dd9b9a3acbf65ed8bcab97e27014afcf" "checksum mime_guess 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1a0ed03949aef72dbdf3116a383d7b38b4768e6f960528cd6a6044aa9ed68599" "checksum miniz_oxide 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7108aff85b876d06f22503dcce091e29f76733b2bfdd91eebce81f5e68203a10" -"checksum mio 0.6.19 (registry+https://github.com/rust-lang/crates.io-index)" = "83f51996a3ed004ef184e16818edc51fadffe8e7ca68be67f9dee67d84d0ff23" +"checksum mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)" = "302dec22bcf6bae6dfb69c647187f4b4d0fb6f535521f7bc022430ce8e12008f" "checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" "checksum native-tls 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "4b2df1a4c22fd44a62147fd8f13dd0f95c9d8ca7b2610299b2a2f9cf8964274e" "checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" diff --git a/Cargo.toml b/Cargo.toml index 23e8896..d283b7d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,8 +6,8 @@ edition = "2018" [dependencies] elefren = { version = "0.20", features = ["toml"] } -html2md = "0.2.9" -reqwest = "0.9.17" +html2md = "0.2" +reqwest = "0.9" serde = "*" serde_derive = "*" toml = "0.5" diff --git a/src/main.rs b/src/main.rs index e65978e..b4e61dd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,11 +1,13 @@ use std::io; -use crate::storage::data::Data; -use crate::storage::filesystem::Filesystem; use elefren::helpers::cli; use elefren::helpers::toml as elefren_toml; use elefren::prelude::*; +use crate::storage::data::Data; +use crate::storage::filesystem::Filesystem; +use crate::storage::joplin::Joplin; + mod config; mod storage; @@ -13,8 +15,12 @@ fn main() { let config = dbg!(config::Config::get()); let client = dbg!(get_mastodon_connection()); let top = dbg!(config.last_favorite.to_string()); - // let _joplin = crate::storage::joplin::validate(&config); - let save_to = Filesystem::new(); + let joplin_storage = if let Some(joplin_config) = &config.joplin { + Some(Joplin::new_from_config(&joplin_config)) + } else { + None + }; + let fs_storage = Filesystem::new(); let most_recent_favourite = client .favourites() @@ -23,7 +29,11 @@ fn main() { .take_while(|record| dbg!(record).id != top) .map(|record| { let conversion = dbg!(Data::from(dbg!(&record))); - conversion.save(&save_to); + if let Some(joplin) = &joplin_storage { + conversion.save(joplin); + } else { + conversion.save(&fs_storage); + } record }) .fold(None, { diff --git a/src/storage/attachment.rs b/src/storage/attachment.rs index 8b738f4..b2b594d 100644 --- a/src/storage/attachment.rs +++ b/src/storage/attachment.rs @@ -1,8 +1,8 @@ use std::convert::From; -use std::fs::File; -use std::path::Path; use std::time::Duration; +use reqwest::Response; + #[derive(Debug)] pub struct Attachment { url: String, @@ -18,7 +18,7 @@ impl From<&elefren::entities::attachment::Attachment> for Attachment { } impl Attachment { - pub fn get_filename(&self) -> String { + pub fn filename(&self) -> String { let mut frags = self.url.rsplitn(2, '/'); if let Some(path_part) = frags.next() { @@ -29,8 +29,7 @@ impl Attachment { } } - pub fn download(&self, local_filename: &Path) { - let mut fp = File::create(local_filename).expect("Failed to create file"); + pub fn download(&self) -> Response { reqwest::Client::builder() .timeout(Duration::from_secs(600)) .build() @@ -38,7 +37,5 @@ impl Attachment { .get(&self.url) .send() .unwrap() - .copy_to(&mut fp) - .unwrap(); } } diff --git a/src/storage/data.rs b/src/storage/data.rs index 6f5e964..a65eaea 100644 --- a/src/storage/data.rs +++ b/src/storage/data.rs @@ -12,6 +12,7 @@ pub struct Data { pub account: String, pub text: String, pub attachments: Vec, + pub source: String, } /// Convert the incoming Status from Elefren to ours. @@ -28,6 +29,7 @@ impl From<&Status> for Data { .iter() .map(|attachment| Attachment::from(attachment)) .collect(), + source: origin.url.as_ref().unwrap_or(&String::new()).to_string(), } } } @@ -52,7 +54,7 @@ fn build_text(status: &Status) -> String { result.push_str(&html2md::parse_html(&base_content)); if let Some(url) = source { - result.push_str("\n"); + result.push_str("\n\n"); result.push_str(&url); } diff --git a/src/storage/filesystem.rs b/src/storage/filesystem.rs index e20cc2d..952b43c 100644 --- a/src/storage/filesystem.rs +++ b/src/storage/filesystem.rs @@ -42,8 +42,9 @@ impl Filesystem { /// Save the attachments. fn save_attachments(&self, data: &Data) { data.attachments.iter().for_each(|attachment| { - let filename = self.dir(data).join(attachment.get_filename()); - attachment.download(filename.as_path()); + let filename = self.dir(data).join(attachment.filename()); + let mut fp = File::create(filename).expect("Failed to create file"); + attachment.download().copy_to(&mut fp).unwrap(); }) } } diff --git a/src/storage/joplin.rs b/src/storage/joplin.rs index 71739af..da780e5 100644 --- a/src/storage/joplin.rs +++ b/src/storage/joplin.rs @@ -1,72 +1,159 @@ -use crate::config::Config; -use crate::config::JoplinConfig; +use std::collections::HashMap; +use reqwest::multipart::Form; +use reqwest::multipart::Part; use reqwest::Error; -use reqwest::Url; use serde_derive::Deserialize; +use crate::config::JoplinConfig; +use crate::storage::data::Data; +use crate::storage::storage::Storage; + /// This is the folder structured returned by Joplin. It is here so Reqwests can /// unjson the data (there are more fields, but these are the only ones we need /// right now). #[allow(dead_code)] -#[derive(Deserialize)] +#[derive(Deserialize, Debug)] struct Folder { id: String, title: String, } +#[allow(dead_code)] +#[derive(Deserialize, Debug)] +struct Resource { + id: String, + filename: String, +} + /// Connection to Joplin. -pub struct JoplinConnection { +pub struct Joplin { port: u32, token: String, folder_id: String, + client: reqwest::Client, } -pub fn validate(config: &Config) -> Option { - if let Some(joplin_config) = &config.joplin { - let folder_id = dbg!(get_folder_id(&joplin_config)); +impl Storage for Joplin { + fn save(&self, record: &Data) { + let resources = dbg!(self.save_attachments(&record)); + let mut text = record.text.to_string(); + let title = format!("{}/{}", record.account, record.id); + Joplin::add_resources_to_text(&mut text, &resources); + dbg!(self.save_content(&title, &text, &record.source)); + } +} - if let Some(folder) = folder_id { - Some(JoplinConnection { - port: joplin_config.port, - token: joplin_config.token.to_string(), - folder_id: folder, - }) +impl Joplin { + pub fn new_from_config(config: &JoplinConfig) -> Joplin { + if let Some(folder_id) = Joplin::find_folder(config) { + Joplin { + port: config.port, + token: config.token.to_string(), + folder_id: folder_id, + client: reqwest::Client::new(), + } } else { - println!("No folder named {}", joplin_config.folder); - None + println!("The notebook {} does not exist", &config.folder); + panic!("The specified notebook does not exist"); } - } else { - println!("Joplin not set up"); - None } -} - -fn build_url(config: &JoplinConfig, resource: &String) -> Url { - let base_url = format!( - "http://localhost:{port}/{resource}?token={token}", - port = config.port, - resource = resource, - token = config.token - ); - let url = Url::parse(&base_url); - url.unwrap() -} -fn get_folder_id(config: &JoplinConfig) -> Option { - let request = get_folder_list(config); - if let Ok(folders) = request { - for folder in folders { - if folder.title == *config.folder { - return Some(folder.id); + fn find_folder(config: &JoplinConfig) -> Option { + if let Ok(folders) = dbg!(Joplin::get_folder_list(config)) { + for folder in folders { + if folder.title == *config.folder { + return Some(folder.id); + } } + None + } else { + println!("Failed to retrieve the notebook list"); + panic!("Failed to retrieve Joplin notebook list"); } } - None -} -fn get_folder_list(config: &JoplinConfig) -> Result, Error> { - let folders: Vec = - reqwest::get(&build_url(config, &String::from("folders")).into_string())?.json()?; - Ok(folders) + fn get_folder_list(config: &JoplinConfig) -> Result, Error> { + let base_url = format!( + "http://localhost:{port}/folders?token={token}", + port = config.port, + token = config.token + ); + let folders: Vec = reqwest::get(&base_url)?.json()?; + Ok(folders) + } + + fn add_resources_to_text(text: &mut String, resources: &Vec) { + resources.iter().for_each(|resource| { + let link = format!( + "![{filename}](:/{resource})", + filename = resource.filename, + resource = resource.id + ); + text.push_str("\n\n"); + text.push_str(&link); + }); + } + + fn save_attachments(&self, record: &Data) -> Vec { + record + .attachments + .iter() + .map(|attachment| { + let mut buffer: Vec = vec![]; + attachment.download().copy_to(&mut buffer).unwrap(); + let resource_id = + dbg!(self.upload_resource(attachment.filename().to_string(), buffer)); + + Resource { + id: resource_id, + filename: attachment.filename().to_string(), + } + }) + .collect() + } + + fn base_url(&self, resource: &str) -> String { + format!( + "http://localhost:{port}/{resource}?token={token}", + port = self.port, + token = self.token, + resource = resource + ) + } + + fn upload_resource(&self, filename: String, content: Vec) -> String { + let props = format!( + "{{\"title\": \"{filename}\", \"filename\": \"{filename}\"}}", + filename = &filename, + ); + let data_part = Part::bytes(content).file_name(filename); + let props_part = Part::text(props); + let form = Form::new() + .part("data", data_part) + .part("props", props_part); + let resource: Resource = self + .client + .post(&self.base_url("resources")) + .multipart(form) + .send() + .unwrap() + .json() + .unwrap(); + resource.id + } + + fn save_content(&self, title: &String, text: &String, source: &String) { + let mut request = HashMap::new(); + request.insert("parent_id", &self.folder_id); + request.insert("title", &title); + request.insert("body", &text); + request.insert("source_url", &source); + + self.client + .post(&self.base_url("notes")) + .json(&request) + .send() + .unwrap(); + } } diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 64620c1..cf6ce4a 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -1,4 +1,5 @@ pub mod attachment; +pub mod data; pub mod filesystem; +pub mod joplin; pub mod storage; -pub mod data;