Browse Source

Push updated to Joplin, if set up

master
Julio Biason 5 years ago
parent
commit
bad3e4fc12
  1. 24
      Cargo.lock
  2. 4
      Cargo.toml
  3. 20
      src/main.rs
  4. 11
      src/storage/attachment.rs
  5. 4
      src/storage/data.rs
  6. 5
      src/storage/filesystem.rs
  7. 173
      src/storage/joplin.rs
  8. 3
      src/storage/mod.rs

24
Cargo.lock generated

@ -90,7 +90,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
"either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
"iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -505,7 +505,7 @@ dependencies = [
"http 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)",
"http-body 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
"iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
"itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
"net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)",
@ -577,11 +577,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "iovec"
version = "0.1.2"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -720,12 +719,13 @@ dependencies = [
[[package]]
name = "mio"
version = "0.6.19"
version = "0.6.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
"fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
"fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
"iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
"kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1493,7 +1493,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
"futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
"mio 0.6.19 (registry+https://github.com/rust-lang/crates.io-index)",
"mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)",
"num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-current-thread 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-executor 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1551,7 +1551,7 @@ dependencies = [
"futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
"mio 0.6.19 (registry+https://github.com/rust-lang/crates.io-index)",
"mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)",
"num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
"parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
"slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1576,8 +1576,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
"futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
"iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"mio 0.6.19 (registry+https://github.com/rust-lang/crates.io-index)",
"iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
"mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-reactor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -1863,7 +1863,7 @@ dependencies = [
"checksum idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e"
"checksum idna 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9"
"checksum indexmap 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a61202fbe46c4a951e9404a720a0180bcf3212c750d735cb5c4ba4dc551299f3"
"checksum iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe6e417e7d0975db6512b90796e8ce223145ac4e33c377e4a42882a0e88bb08"
"checksum iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e"
"checksum isolang 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "265ef164908329e47e753c769b14cbb27434abf0c41984dca201484022f09ce5"
"checksum itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "501266b7edd0174f8530248f87f99c88fbe60ca4ef3dd486835b8d8d53136f7f"
"checksum jni 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "294eca097d1dc0bf59de5ab9f7eafa5f77129e9f6464c957ed3ddeb705fb4292"
@ -1882,7 +1882,7 @@ dependencies = [
"checksum mime 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "dd1d63acd1b78403cc0c325605908475dd9b9a3acbf65ed8bcab97e27014afcf"
"checksum mime_guess 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1a0ed03949aef72dbdf3116a383d7b38b4768e6f960528cd6a6044aa9ed68599"
"checksum miniz_oxide 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7108aff85b876d06f22503dcce091e29f76733b2bfdd91eebce81f5e68203a10"
"checksum mio 0.6.19 (registry+https://github.com/rust-lang/crates.io-index)" = "83f51996a3ed004ef184e16818edc51fadffe8e7ca68be67f9dee67d84d0ff23"
"checksum mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)" = "302dec22bcf6bae6dfb69c647187f4b4d0fb6f535521f7bc022430ce8e12008f"
"checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919"
"checksum native-tls 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "4b2df1a4c22fd44a62147fd8f13dd0f95c9d8ca7b2610299b2a2f9cf8964274e"
"checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88"

4
Cargo.toml

@ -6,8 +6,8 @@ edition = "2018"
[dependencies]
elefren = { version = "0.20", features = ["toml"] }
html2md = "0.2.9"
reqwest = "0.9.17"
html2md = "0.2"
reqwest = "0.9"
serde = "*"
serde_derive = "*"
toml = "0.5"

20
src/main.rs

@ -1,11 +1,13 @@
use std::io;
use crate::storage::data::Data;
use crate::storage::filesystem::Filesystem;
use elefren::helpers::cli;
use elefren::helpers::toml as elefren_toml;
use elefren::prelude::*;
use crate::storage::data::Data;
use crate::storage::filesystem::Filesystem;
use crate::storage::joplin::Joplin;
mod config;
mod storage;
@ -13,8 +15,12 @@ fn main() {
let config = dbg!(config::Config::get());
let client = dbg!(get_mastodon_connection());
let top = dbg!(config.last_favorite.to_string());
// let _joplin = crate::storage::joplin::validate(&config);
let save_to = Filesystem::new();
let joplin_storage = if let Some(joplin_config) = &config.joplin {
Some(Joplin::new_from_config(&joplin_config))
} else {
None
};
let fs_storage = Filesystem::new();
let most_recent_favourite = client
.favourites()
@ -23,7 +29,11 @@ fn main() {
.take_while(|record| dbg!(record).id != top)
.map(|record| {
let conversion = dbg!(Data::from(dbg!(&record)));
conversion.save(&save_to);
if let Some(joplin) = &joplin_storage {
conversion.save(joplin);
} else {
conversion.save(&fs_storage);
}
record
})
.fold(None, {

11
src/storage/attachment.rs

@ -1,8 +1,8 @@
use std::convert::From;
use std::fs::File;
use std::path::Path;
use std::time::Duration;
use reqwest::Response;
#[derive(Debug)]
pub struct Attachment {
url: String,
@ -18,7 +18,7 @@ impl From<&elefren::entities::attachment::Attachment> for Attachment {
}
impl Attachment {
pub fn get_filename(&self) -> String {
pub fn filename(&self) -> String {
let mut frags = self.url.rsplitn(2, '/');
if let Some(path_part) = frags.next() {
@ -29,8 +29,7 @@ impl Attachment {
}
}
pub fn download(&self, local_filename: &Path) {
let mut fp = File::create(local_filename).expect("Failed to create file");
pub fn download(&self) -> Response {
reqwest::Client::builder()
.timeout(Duration::from_secs(600))
.build()
@ -38,7 +37,5 @@ impl Attachment {
.get(&self.url)
.send()
.unwrap()
.copy_to(&mut fp)
.unwrap();
}
}

4
src/storage/data.rs

@ -12,6 +12,7 @@ pub struct Data {
pub account: String,
pub text: String,
pub attachments: Vec<Attachment>,
pub source: String,
}
/// Convert the incoming Status from Elefren to ours.
@ -28,6 +29,7 @@ impl From<&Status> for Data {
.iter()
.map(|attachment| Attachment::from(attachment))
.collect(),
source: origin.url.as_ref().unwrap_or(&String::new()).to_string(),
}
}
}
@ -52,7 +54,7 @@ fn build_text(status: &Status) -> String {
result.push_str(&html2md::parse_html(&base_content));
if let Some(url) = source {
result.push_str("\n");
result.push_str("\n\n");
result.push_str(&url);
}

5
src/storage/filesystem.rs

@ -42,8 +42,9 @@ impl Filesystem {
/// Save the attachments.
fn save_attachments(&self, data: &Data) {
data.attachments.iter().for_each(|attachment| {
let filename = self.dir(data).join(attachment.get_filename());
attachment.download(filename.as_path());
let filename = self.dir(data).join(attachment.filename());
let mut fp = File::create(filename).expect("Failed to create file");
attachment.download().copy_to(&mut fp).unwrap();
})
}
}

173
src/storage/joplin.rs

@ -1,72 +1,159 @@
use crate::config::Config;
use crate::config::JoplinConfig;
use std::collections::HashMap;
use reqwest::multipart::Form;
use reqwest::multipart::Part;
use reqwest::Error;
use reqwest::Url;
use serde_derive::Deserialize;
use crate::config::JoplinConfig;
use crate::storage::data::Data;
use crate::storage::storage::Storage;
/// This is the folder structured returned by Joplin. It is here so Reqwests can
/// unjson the data (there are more fields, but these are the only ones we need
/// right now).
#[allow(dead_code)]
#[derive(Deserialize)]
#[derive(Deserialize, Debug)]
struct Folder {
id: String,
title: String,
}
#[allow(dead_code)]
#[derive(Deserialize, Debug)]
struct Resource {
id: String,
filename: String,
}
/// Connection to Joplin.
pub struct JoplinConnection {
pub struct Joplin {
port: u32,
token: String,
folder_id: String,
client: reqwest::Client,
}
pub fn validate(config: &Config) -> Option<JoplinConnection> {
if let Some(joplin_config) = &config.joplin {
let folder_id = dbg!(get_folder_id(&joplin_config));
impl Storage for Joplin {
fn save(&self, record: &Data) {
let resources = dbg!(self.save_attachments(&record));
let mut text = record.text.to_string();
let title = format!("{}/{}", record.account, record.id);
Joplin::add_resources_to_text(&mut text, &resources);
dbg!(self.save_content(&title, &text, &record.source));
}
}
if let Some(folder) = folder_id {
Some(JoplinConnection {
port: joplin_config.port,
token: joplin_config.token.to_string(),
folder_id: folder,
})
impl Joplin {
pub fn new_from_config(config: &JoplinConfig) -> Joplin {
if let Some(folder_id) = Joplin::find_folder(config) {
Joplin {
port: config.port,
token: config.token.to_string(),
folder_id: folder_id,
client: reqwest::Client::new(),
}
} else {
println!("No folder named {}", joplin_config.folder);
None
println!("The notebook {} does not exist", &config.folder);
panic!("The specified notebook does not exist");
}
} else {
println!("Joplin not set up");
None
}
}
fn build_url(config: &JoplinConfig, resource: &String) -> Url {
let base_url = format!(
"http://localhost:{port}/{resource}?token={token}",
port = config.port,
resource = resource,
token = config.token
);
let url = Url::parse(&base_url);
url.unwrap()
}
fn get_folder_id(config: &JoplinConfig) -> Option<String> {
let request = get_folder_list(config);
if let Ok(folders) = request {
for folder in folders {
if folder.title == *config.folder {
return Some(folder.id);
fn find_folder(config: &JoplinConfig) -> Option<String> {
if let Ok(folders) = dbg!(Joplin::get_folder_list(config)) {
for folder in folders {
if folder.title == *config.folder {
return Some(folder.id);
}
}
None
} else {
println!("Failed to retrieve the notebook list");
panic!("Failed to retrieve Joplin notebook list");
}
}
None
}
fn get_folder_list(config: &JoplinConfig) -> Result<Vec<Folder>, Error> {
let folders: Vec<Folder> =
reqwest::get(&build_url(config, &String::from("folders")).into_string())?.json()?;
Ok(folders)
fn get_folder_list(config: &JoplinConfig) -> Result<Vec<Folder>, Error> {
let base_url = format!(
"http://localhost:{port}/folders?token={token}",
port = config.port,
token = config.token
);
let folders: Vec<Folder> = reqwest::get(&base_url)?.json()?;
Ok(folders)
}
fn add_resources_to_text(text: &mut String, resources: &Vec<Resource>) {
resources.iter().for_each(|resource| {
let link = format!(
"![{filename}](:/{resource})",
filename = resource.filename,
resource = resource.id
);
text.push_str("\n\n");
text.push_str(&link);
});
}
fn save_attachments(&self, record: &Data) -> Vec<Resource> {
record
.attachments
.iter()
.map(|attachment| {
let mut buffer: Vec<u8> = vec![];
attachment.download().copy_to(&mut buffer).unwrap();
let resource_id =
dbg!(self.upload_resource(attachment.filename().to_string(), buffer));
Resource {
id: resource_id,
filename: attachment.filename().to_string(),
}
})
.collect()
}
fn base_url(&self, resource: &str) -> String {
format!(
"http://localhost:{port}/{resource}?token={token}",
port = self.port,
token = self.token,
resource = resource
)
}
fn upload_resource(&self, filename: String, content: Vec<u8>) -> String {
let props = format!(
"{{\"title\": \"{filename}\", \"filename\": \"{filename}\"}}",
filename = &filename,
);
let data_part = Part::bytes(content).file_name(filename);
let props_part = Part::text(props);
let form = Form::new()
.part("data", data_part)
.part("props", props_part);
let resource: Resource = self
.client
.post(&self.base_url("resources"))
.multipart(form)
.send()
.unwrap()
.json()
.unwrap();
resource.id
}
fn save_content(&self, title: &String, text: &String, source: &String) {
let mut request = HashMap::new();
request.insert("parent_id", &self.folder_id);
request.insert("title", &title);
request.insert("body", &text);
request.insert("source_url", &source);
self.client
.post(&self.base_url("notes"))
.json(&request)
.send()
.unwrap();
}
}

3
src/storage/mod.rs

@ -1,4 +1,5 @@
pub mod attachment;
pub mod data;
pub mod filesystem;
pub mod joplin;
pub mod storage;
pub mod data;

Loading…
Cancel
Save