From e869ddf7ee8a19f1c6c06ea84358908a99d1e901 Mon Sep 17 00:00:00 2001 From: Julio Biason Date: Sun, 12 Apr 2020 11:19:02 -0300 Subject: [PATCH] Generic'fied the storage, so we can make the Joplin storage work --- .gitignore | 2 + Cargo.lock | 2 +- Cargo.toml | 2 +- src/config/mod.rs | 60 ++++++++++++++++ src/main.rs | 142 ++++---------------------------------- src/storage/attachment.rs | 44 ++++++++++++ src/storage/filesystem.rs | 79 +++++++++++++++++++++ src/storage/mod.rs | 3 + src/storage/storage.rs | 12 ++++ 9 files changed, 215 insertions(+), 131 deletions(-) create mode 100644 src/config/mod.rs create mode 100644 src/storage/attachment.rs create mode 100644 src/storage/filesystem.rs create mode 100644 src/storage/mod.rs create mode 100644 src/storage/storage.rs diff --git a/.gitignore b/.gitignore index 503771c..17864f2 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ mastodon.toml downfav.toml data +functional.cafe +sinblr.com diff --git a/Cargo.lock b/Cargo.lock index a203fb1..a7b36c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -260,7 +260,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "downfav" -version = "0.2.0" +version = "0.3.0" dependencies = [ "elefren 0.19.4 (registry+https://github.com/rust-lang/crates.io-index)", "html2md 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index e4d7b62..224f91f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "downfav" -version = "0.2.0" +version = "0.3.0" authors = ["Julio Biason "] edition = "2018" diff --git a/src/config/mod.rs b/src/config/mod.rs new file mode 100644 index 0000000..f7f1bb9 --- /dev/null +++ b/src/config/mod.rs @@ -0,0 +1,60 @@ +use std::fs::File; +use std::io::prelude::*; + +use serde_derive::Deserialize; +use serde_derive::Serialize; + +use toml; + +#[derive(Serialize, Deserialize, Debug)] +pub struct JoplinConfig { + port: u32, + folder: String, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct Config { + pub last_favorite: String, + pub joplin: Option, +} + +impl Config { + pub fn get() -> Config { + if let Ok(mut fp) = File::open("downfav.toml") { + let mut contents = String::new(); + fp.read_to_string(&mut contents).unwrap(); + + let config: Config = toml::from_str(&contents).unwrap_or(Config { + last_favorite: "".to_string(), + joplin: None, + }); + config + } else { + Config { + last_favorite: "".to_string(), + joplin: None, + } + } + } + + + pub fn save(&self, most_recent_favourite: Option) -> () { + if let Some(id) = most_recent_favourite { + let new_configuration = Config { + last_favorite: id, + joplin: match &self.joplin { + None => None, + Some(x) => Some(JoplinConfig { + folder: x.folder.to_string(), + port: x.port, + }), + }, + }; + let content = toml::to_string(&new_configuration).unwrap(); + + if let Ok(mut fp) = File::create("downfav.toml") { + fp.write_all(content.as_bytes()).unwrap(); + } + } + } +} diff --git a/src/main.rs b/src/main.rs index 90bb13d..853f5c4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,38 +1,17 @@ -use std::fs::File; use std::io; -use std::io::prelude::*; -use std::path::Path; -use std::path::PathBuf; -use std::time::Duration; -use elefren::entities::attachment::Attachment; -use elefren::entities::status::Status; +use crate::storage::storage::Storage; use elefren::helpers::cli; use elefren::helpers::toml as elefren_toml; use elefren::prelude::*; -use reqwest; - -use serde_derive::Deserialize; -use serde_derive::Serialize; -use toml; - -#[derive(Serialize, Deserialize, Debug)] -struct JoplinConfig { - port: u32, - folder: String, -} - -#[derive(Serialize, Deserialize, Debug)] -struct Config { - last_favorite: String, - joplin: Option, -} +mod config; +mod storage; fn main() { - let config = dbg!(get_config()); - let client = get_mastodon_connection(); - let top = config.last_favorite.to_string(); + let config = dbg!(config::Config::get()); + let client = dbg!(get_mastodon_connection()); + let top = dbg!(config.last_favorite.to_string()); let most_recent_favourite = client .favourites() @@ -40,7 +19,9 @@ fn main() { .items_iter() .take_while(|record| record.id != top) .map(|record| { - dump_record(&record); + let storage = dbg!(storage::filesystem::Filesystem::from(&record)); + storage.open(); + storage.save(); record }) .fold(None, { @@ -53,34 +34,16 @@ fn main() { } }); - save_config(&config, most_recent_favourite) -} - -fn save_config(config: &Config, most_recent_favourite: Option) -> () { - if let Some(id) = most_recent_favourite { - let new_configuration = Config { - last_favorite: id, - joplin: match &config.joplin { - None => None, - Some(x) => Some(JoplinConfig { - folder: x.folder.to_string(), - port: x.port, - }), - }, - }; - let content = toml::to_string(&new_configuration).unwrap(); - - if let Ok(mut fp) = File::create("downfav.toml") { - fp.write_all(content.as_bytes()).unwrap(); - } - } + config.save(most_recent_favourite); } +/// Get a connection with Mastodon; if there is no set up with any account yet, +/// requests one. fn get_mastodon_connection() -> Mastodon { if let Ok(data) = elefren_toml::from_file("mastodon.toml") { Mastodon::from(data) } else { - print!("Your server URL: "); + println!("Your server URL: "); let mut server = String::new(); io::stdin() .read_line(&mut server) @@ -95,82 +58,3 @@ fn get_mastodon_connection() -> Mastodon { mastodon } } - -fn get_config() -> Config { - if let Ok(mut fp) = File::open("downfav.toml") { - let mut contents = String::new(); - fp.read_to_string(&mut contents).unwrap(); - - let config: Config = toml::from_str(&contents).unwrap_or(Config { - last_favorite: "".to_string(), - joplin: None, - }); - config - } else { - Config { - last_favorite: "".to_string(), - joplin: None, - } - } -} - -fn dump_record(record: &Status) { - println!("Downloading {}/{}", &record.account.acct, &record.id); - create_structure(dbg!(&record)); - save_content(&record); - save_attachments(&record); -} - -fn toot_dir(record: &Status) -> PathBuf { - Path::new("data") - .join(&record.account.acct) - .join(&record.id) -} - -fn create_structure(record: &Status) { - println!("Current ID: {}", record.id); - std::fs::create_dir_all(toot_dir(record)).expect("Failed to create the storage path"); -} - -fn save_content(record: &Status) { - if let Ok(mut fp) = File::create(toot_dir(&record).join("toot.md")) { - fp.write_all(html2md::parse_html(&record.content).as_bytes()) - .expect("Failed to save content"); - } -} - -fn save_attachments(record: &Status) { - let base_path = toot_dir(&record); - record - .media_attachments - .iter() - .for_each(move |x| save_attachment(dbg!(&x), &base_path)); -} - -fn save_attachment(attachment: &Attachment, base_path: &PathBuf) { - let filename = get_attachment_filename(dbg!(&attachment.url)); - println!("\tAttachment: {:?}", &filename); - let saving_target = dbg!(base_path.join(filename)); - if let Ok(mut fp) = File::create(saving_target) { - let client = reqwest::Client::builder() - .timeout(Duration::from_secs(600)) - .build() - .unwrap(); - client - .get(&attachment.url) - .send() - .expect("Failed to connect to server") - .copy_to(&mut fp) - .expect("Failed to save attachment"); - } -} - -fn get_attachment_filename(url: &str) -> String { - let mut frags = url.rsplitn(2, '/'); - if let Some(path_part) = frags.next() { - dbg!(path_part.split('?').next().unwrap_or(url).to_string()) - } else { - // this is, most of the time, bad (due special characters -- like '?' -- and path) - dbg!(url.to_string()) - } -} diff --git a/src/storage/attachment.rs b/src/storage/attachment.rs new file mode 100644 index 0000000..8b738f4 --- /dev/null +++ b/src/storage/attachment.rs @@ -0,0 +1,44 @@ +use std::convert::From; +use std::fs::File; +use std::path::Path; +use std::time::Duration; + +#[derive(Debug)] +pub struct Attachment { + url: String, +} + +impl From<&elefren::entities::attachment::Attachment> for Attachment { + fn from(origin: &elefren::entities::attachment::Attachment) -> Self { + println!("Found attachment: {}", origin.url); + Self { + url: origin.url.to_string(), + } + } +} + +impl Attachment { + pub fn get_filename(&self) -> String { + let mut frags = self.url.rsplitn(2, '/'); + + if let Some(path_part) = frags.next() { + dbg!(path_part.split('?').next().unwrap_or(&self.url).to_string()) + } else { + // this is, most of the time, bad (due special characters -- like '?' -- and path) + dbg!(self.url.to_string()) + } + } + + pub fn download(&self, local_filename: &Path) { + let mut fp = File::create(local_filename).expect("Failed to create file"); + reqwest::Client::builder() + .timeout(Duration::from_secs(600)) + .build() + .unwrap() + .get(&self.url) + .send() + .unwrap() + .copy_to(&mut fp) + .unwrap(); + } +} diff --git a/src/storage/filesystem.rs b/src/storage/filesystem.rs new file mode 100644 index 0000000..e351354 --- /dev/null +++ b/src/storage/filesystem.rs @@ -0,0 +1,79 @@ +use elefren::entities::status::Status; +use html2md; +use std::convert::From; +use std::fs::File; +use std::io::Write; +use std::path::Path; +use std::path::PathBuf; + +use crate::storage::attachment::Attachment; +use crate::storage::storage::Storage; + +#[derive(Debug)] +pub struct Filesystem { + id: String, + account: String, + text: String, + attachments: Vec, +} + +impl Filesystem { + /// The directory in which the data from this toot will be saved. + fn dir(&self) -> PathBuf { + Path::new("data").join(&self.account).join(&self.id) + } + + /// Make sure the path structure exists for saving the data. + fn create_dirs(&self) { + std::fs::create_dir_all(self.dir()).expect("Failed to create storage directory"); + } + + /// Save the content in the directory. + fn save_content(&self) { + let filename = self.dir().join("toot.md"); + let mut fp = File::create(filename).expect("Failed to create file"); + fp.write_all(self.text.as_bytes()) + .expect("Failed to save content"); + } + + /// Save the attachments. + fn save_attachments(&self) { + self.attachments.iter().for_each(|attachment| { + let filename = self.dir().join(attachment.get_filename()); + attachment.download(filename.as_path()); + }) + } +} + +impl Storage for Filesystem { + fn open(&self) { + dbg!(self.create_dirs()); + } + + fn get_id(&self) -> &String { + &self.id + } + + fn save(&self) { + self.save_content(); + self.save_attachments(); + } +} + +impl From<&Status> for Filesystem { + fn from(origin: &Status) -> Self { + println!("Downloading ID: {}", origin.id); + + Self { + id: origin.id.to_string(), + account: origin.account.acct.to_string(), + text: html2md::parse_html(&origin.content), + // on save, we download those URLs + attachments: origin + .media_attachments + .iter() + .map(|attachment| Attachment::from(attachment)) + .collect(), + } + } +} diff --git a/src/storage/mod.rs b/src/storage/mod.rs new file mode 100644 index 0000000..5ec7031 --- /dev/null +++ b/src/storage/mod.rs @@ -0,0 +1,3 @@ +pub mod filesystem; +pub mod storage; +pub mod attachment; diff --git a/src/storage/storage.rs b/src/storage/storage.rs new file mode 100644 index 0000000..bc3773c --- /dev/null +++ b/src/storage/storage.rs @@ -0,0 +1,12 @@ +/// Trait for storing favorites on a storage. +pub trait Storage { + /// Initization. Any required pre-storage functions must be added here. + fn open(&self); + + /// Save the favourite in the storage. + fn save(&self); + + /// Return the original favourite identification. + fn get_id(&self) -> &String; +} +