From 0dcc518d2c877ec573aebec3a315347984eff624 Mon Sep 17 00:00:00 2001 From: Julio Biason Date: Mon, 14 Oct 2024 15:18:32 -0300 Subject: [PATCH] vault backup: 2024-10-14 15:18:32 --- .obsidian/workspace.json | 23 +- Links.md | 3 +- ...e’s kingdoms – How To Market A Game.md | 196 +++++++ ...imate Guide to Error Handling in Python.md | 333 ++++++++++++ Pages/Tracking the music I listen to.md | 502 ++++++++++++++++++ 5 files changed, 1046 insertions(+), 11 deletions(-) create mode 100644 Pages/Don’t build your castle in other people’s kingdoms – How To Market A Game.md create mode 100644 Pages/The Ultimate Guide to Error Handling in Python.md create mode 100644 Pages/Tracking the music I listen to.md diff --git a/.obsidian/workspace.json b/.obsidian/workspace.json index f5ab307..aa408da 100644 --- a/.obsidian/workspace.json +++ b/.obsidian/workspace.json @@ -13,7 +13,7 @@ "state": { "type": "markdown", "state": { - "file": "GW2/GW2 Future Character Names.md", + "file": "Pages/Tracking the music I listen to.md", "mode": "source", "source": false } @@ -85,7 +85,7 @@ "state": { "type": "backlink", "state": { - "file": "GW2/GW2 Future Character Names.md", + "file": "Pages/Tracking the music I listen to.md", "collapseAll": false, "extraContext": false, "sortOrder": "alphabetical", @@ -102,7 +102,7 @@ "state": { "type": "outgoing-link", "state": { - "file": "GW2/GW2 Future Character Names.md", + "file": "Pages/Tracking the music I listen to.md", "linksCollapsed": false, "unlinkedCollapsed": true } @@ -125,7 +125,7 @@ "state": { "type": "outline", "state": { - "file": "GW2/GW2 Future Character Names.md" + "file": "Pages/Tracking the music I listen to.md" } } } @@ -150,12 +150,18 @@ }, "active": "95791f89821765eb", "lastOpenFiles": [ - "Motorbike/Trips.md", + "Pages/The Ultimate Guide to Error Handling in Python.md", + "Pages/Tracking the music I listen to.md", + "Pages/Server Setup Basics.md", + "Pages/8 versions of UUID and when to use them.md", + "Pages/Don’t build your castle in other people’s kingdoms – How To Market A Game.md", "Links.md", - "Endereços/Twinkle Cintilação.md", - "Endereços", "Motorbike/Rota Biker.md", "Motorbike/Pasted image 20241002101959.png", + "GW2/GW2 Future Character Names.md", + "Motorbike/Trips.md", + "Endereços/Twinkle Cintilação.md", + "Endereços", "Motorbike/Cities.md", "Rota Biker.kml", "PythonSul 2025/0. Definição do evento.md", @@ -163,7 +169,6 @@ "Motorbike/Trips.md~", "Motorbike/Cities.md~", "GW2", - "GW2/GW2 Future Character Names.md", "Motorbike", "PythonSul 2025/Z. Kanban.md", "PythonSul 2025", @@ -175,8 +180,6 @@ "Images/Do Not Connect To The Internet.md", "Images", "Pages", - "Pages/Server Setup Basics.md", - "Pages/8 versions of UUID and when to use them.md", "Welcome.md" ] } \ No newline at end of file diff --git a/Links.md b/Links.md index fb6dbbf..7391d3b 100644 --- a/Links.md +++ b/Links.md @@ -2,4 +2,5 @@ https://github.com/elastio/bon: `bon` is a Rust crate for generating compile-tim https://github.com/spring-rs/spring-rs: **spring-rs** is a microservice framework written in Rust, similar to SpringBoot in java. **spring-rs** provides an easily extensible plug-in system for integrating excellent projects in the Rust community, such as axum, sqlx, sea-orm, etc. https://d07riv.github.io/diabloweb/: Run Diablo I in your browser https://owickstrom.github.io/the-monospace-web/: The Monospace Web -https://www.howtocodeit.com/articles/master-hexagonal-architecture-rust: Master hexagonal architecture in Rust \ No newline at end of file +https://www.howtocodeit.com/articles/master-hexagonal-architecture-rust: Master hexagonal architecture in Rust +https://www.reddit.com/r/rust/comments/qpeblj/doctave_a_batteriesincluded_docs_site_generator/: MkDocs in Rust \ No newline at end of file diff --git a/Pages/Don’t build your castle in other people’s kingdoms – How To Market A Game.md b/Pages/Don’t build your castle in other people’s kingdoms – How To Market A Game.md new file mode 100644 index 0000000..d3cda32 --- /dev/null +++ b/Pages/Don’t build your castle in other people’s kingdoms – How To Market A Game.md @@ -0,0 +1,196 @@ + +[howtomarketagame.com](https://howtomarketagame.com/2021/11/01/dont-build-your-castle-in-other-peoples-kingdoms/) + +11–14 minutes + +--- + +![](https://howtomarketagame.com/wp-content/uploads/2021/11/1-solo-island.png) + +In the past couple of months a couple of big social media sites have changed their terms or introduced suspicious paid plans and it has caught content creators off guard. + +For instance, last week Twitch introduced a new “Boost” program where streamers can pay to get more viewers to see their stream. + +Read here + +- [Twitch Adds New Boost Stream Feature](https://www.dexerto.com/entertainment/twitch-adds-new-boost-stream-feature-but-it-comes-at-a-price-1686346/) +- [Twitch Tests Paid Boost Feature](https://www.eurogamer.net/articles/2021-10-29-twitch-tests-paid-boost-feature-despite-negative-feedback-from-streamers) + +[OnlyFans ALMOST banned porn which would have left content creators out of hundreds of thousands of dollars](https://www.cnbc.com/2021/08/25/onlyfans-says-it-will-no-longer-ban-porn-after-backlash-from-users.html). Only Fans (under heavy pressure) reversed the plan… for now.  + +Every single year one social network or another pulls a stunt like this. Somehow people are still shocked when it happens. + +Don’t be. + +It isn’t a matter of if, it is a matter of when. They don’t care about you. You must market your game as if the platform you are using will go away tomorrow and you will lose access to some or most of your followers. + +My basic point: DON’T BUILD YOUR CASTLE IN LAND YOU DON’T OWN! + +I got the pixelart itch again and have a blog to write so I thought I would combine the two. This week I created a little metaphorical fable about building your castle in other people’s kingdoms. [I am using the amazing RPG tileset from Pita Madgwick to do it.](https://assetstore.unity.com/packages/2d/environments/overworld-tileset-123120) + +Here goes: + +Your game studio is basically your land. You are the king. You can do whatever you want on this plot of land and kick out who you want, charge what you want. Set the rules.  + +Goal here: You want to grow from this little tiny hamlet to a giant castle. You also want a bunch of people in your kingdom living there (aka playing your games), and paying you taxes (buying your games) and telling you how brilliant of a leader you are (fan mail, fan art) and enjoying the company of your kingdom’s fellow citizens (community engagement).  + +That is your goal but for now you have no one in your kingdom. Your castle is tiny. But how do you get people to come to your crappy little kingdom? + +![](https://lh3.googleusercontent.com/B342AzyiJq1j8CzQ_LW12n61PhoLR8q6Xe8DDEZ6x1tkiB_1M36nK393uUKN9fsTV6ZBn1qS6d2preSjyU0wxoww9T5cHjzRu7a3q1Cz6_D0CjBT24MvSB27ZcAyN4tOFRzAlPqr) + +You look across the sea and you see another country run by another King. They have thousands of people. Everyone is rich. The king is giving away land for anyone to build their castle. + +![](https://lh6.googleusercontent.com/WsxJUTU3z8Hg9jPPdPsXIUkTtxjOMXcF0z8BvQZRso8TxVVqfBHZlyiO-Nvo_rWacyzd1BN2lTg9DreOYTcf6mcVfeLyyk4t39Y_RAhj4HoyPBOCwkzV5HOth4ltulZpWM_kWKmu) + +Let’s call it Myspacia  + +So you cross the sea and build a castle in their land. The King doesn’t even charge you for it. So you decide to spend a whole lot of time building a big beautiful castle. Everyone is there. Thousands of people come into your castle built on their land. Your home Kingdom is neglected but why worry?  + +![](https://lh3.googleusercontent.com/GmeyWq6DjDyui5ZbRiMPcwSmyN4JYVMG9_DhRSfsZQ_ecktVmi5pL94VS2YIt5kL5FKZAHznUbqCZGA4ZIax193WVMB1cRTQUOyOqfooyyVouSdME8zSybyRbFHYcTUbwRHsj-Ge) + +Then one day the rain dries up and the land isn’t fertile, and the king doesn’t do anything like invest in irrigation, and the infrastructure sucks. + +People move away and that castle that you spent so much time building is empty. Nobody is around to visit. It sucks. Your castle looks awesome but it sits empty not because of anything you did but because the landlord messed up and everyone left. + +![](https://lh3.googleusercontent.com/uc4dtpx3baqH8KgU5jG_JQYjbiEB1YzZqxpPdbcdgp48lMTcwYygGaUW88q0BlX9txL_jpI6HyxNpWyhP9pMIDCK4zkkbOaFwvIAJlQ-vGiH3VMsalKKmOzZ3AmTIlCShEpOOqGP) + +So you say, no matter. Let’s go over to this kingdom where everyone is setting up.  + +Let’s call it Facebookia  + +Again, just like Myspacia once was, it is bustling. It’s just like the good old days. + +You start a new castle! Sure you have to start all over but look at all the people! So you build and build and build. Your home Kingdom is still neglected but who cares! Facebookia is awesome! + +![](https://lh6.googleusercontent.com/fPhcFgwmQLqWvsyoJdXWWn2yRYybxUKclXZCTU-P1cwDWBPzeCvHsv0vlSYj7VB9hzFRwgBuOQXXq4bkWb8xTAuq6sQOf-i_lr46QKnq0DoDGsQtpVrILGoZKYIF3BelBfvswQEj) + +Then one day. The King of that land puts a giant wall around their kingdom. Even though you helped make his kingdom worth living in, you are not allowed to enter your castle. In fact, the King starts charging you a toll just to enter your own castle to talk to the people who are there! Why oh why did you spend so much time and money on this castle?!? + +![](https://lh3.googleusercontent.com/XgM75Xpnd9EAT_WipX5qhbAhri4i4eWtTzU1wxbByzqaDackayNLcBo42roM4nVo503HOIwGQZW0uMb_A4D_T1MYVgpf6SWxorADt-ca3ltWFFgWNB1LkDymtpewcKJRiENB-VnW) + +This happens every single day.  + +But there are thousands of variations + +**A hoard of racist barbarians invade** and nobody wants to visit your castle anymore because the king turns out to be a total creep and everyone is scared off because he just lets the racists hang out. (See Twitter and Facebook) + +![](https://lh4.googleusercontent.com/rrA_fTBIm_iRpjXs4tFDREVbewDkNmjQTOmU8CwG_i_dC6HvGb5AQtg4s4DNr4EzTBiaT8m-CXBbW0zEqhER-KUboFH_9NMsMG4MVsUZVI3WTcIlDvcDuvYhi9anIEytq8YuIyXO) + +**The kingdom gets absorbed** by another neighboring King that has no idea why people where there in the first place and starts changing the laws and everyone leaves. Example: Tumblr + +There are countless tales of this happening. EVERY SINGLE EXTERNAL KINGDOM will do this to you. No matter how “cool” or “hip” they are right now. They will let you down. + +So what do you do? How do you possibly fix this? Here are the rules. + +## Rule #1: Build your castle on land you own  + +Your kingdom is a platform that you own:  + +- A website on a domain you own +- Your blog hosted on your site (not on medium or Patreon) +- A mailing list +- Your Intellectual Property that you own and can license out +- Merch that you sell on your site +- Your own reputation + +But wait! Your mailing list is hosted by Mailchimp which is another company, and your website is hosted by GoDaddy or Squarespace? Aren’t they evil kingdoms too?  + +Not really. They are just hosting platforms that are invisible to your followers. + +The general public doesn’t have to go to Mailchimp.com to read your newsletter or squarespace to view your blog. Your readers go to your domain. + +If any of those companies changes their terms in a way that you don’t like, you can migrate to a new hosting platform and your followers will never have to change their behavior. They will still go to the same URL to get to your site, they will still receive your newsletter from your email address. It is completely transparent to them. + +Spend more resources on the castles on your land. + +![](https://lh3.googleusercontent.com/JNGGv26YNconZhLlROTMOHEeDhFql44h9V8mObSz2oCAj8lKNlcbSRZQfT-QpCb8iqXjK2263xG3BQ-8k-2IPlMIFUuJLzdqvcaNCiVFqUBtHh1alkzoSTVJyEhyaosB84JxWWrN) + +## Rule #2: SHAMELESSLY USE THE OTHER KINGDOMS JUST LIKE THEY ARE USING YOU!  + +I know Tiktok is the hot social media platform right now. It is so pleasant! It’s not like that last kingdom twitterlandia! All the cool kids are here! It is fun! There is dancing! And the traffic is huge! + +But just like Facebookia and Myspacia something will change with them. It will. [It almost did already](https://www.npr.org/2021/02/10/966584204/biden-administration-pauses-trumps-tiktok-ban-backs-off-pressure-for-tiktok-to-s). + +Your solution? Use them so hard. Don’t fall in love. Try to get every last follower off of their kingdom and back to your kingdom. Be shameless about it. + +It is true, you cannot get visibility without them, but don’t rely on them as your home base. Instead build the minimum following necessary. You are basically building a [Potemkin Castle](https://en.wikipedia.org/wiki/Potemkin_village).  + +Don’t feel guilty posting a Call to Action of “Join me on my mailing list.” You have every right to get people who want to hear from you out of their kingdom and into your kingdom. Don’t let people shame you that you are being to “salesy.” F**k em. You need to get people out of enemy territory and you gotta be clear. _[ALWAYS HAVE A CALL TO ACTION](https://youtu.be/EMGTcgsEN68?t=561)_.   + +Basically build a big tall temporary tower in that foreign kingdom that is loud and garish and attracts a lot of attention. When anyone enters you say “oh to really follow me, go across the sea and join my real kingdom.” + +![](https://lh4.googleusercontent.com/gXjiI78Axux6a5B4aCaDYgKi16xThuhfPwVZLmuklckfcjCC1Cxj51zxoRUENf9NeO9oib4KLWER1wUtdzS_3tPklNiVEn7FF-PhudvV4i3w1o8KGFX5YAS8MZoYvyInTbzuQUNg) + +Always try to build this bridge from a rival Kingdom back to your home Kingdom. Basically, when using any social media platform always ask yourself “How can I enrich my own kingdom with the people of this other land?” + +When constructing your path see my next rule… + +## Rule #3: Always move people back to your kingdom, never to another kingdom. + +It always kills me when I see a trailer on youtube go viral and the call to action at the end is “follow us on Discord!” or “Follow us on Twitter.” + +UGH! You went from Youtuberia to Discordia! Neither of which are owned by you. + +You are just sacrificing a bunch of traffic that could have been your own and you are giving it to another King who can take it from you. + +Always try to get people to follow you back to your kingdom. + +![](https://lh6.googleusercontent.com/hCjcUgpNcksuG7SlC27853Ep_sCmnpDHSTu-_7sYMG_E2JCC83S52gd7OzANwJK8u4HhcvG6Wenw6Dl5I1o37D6Bp1Z4qE9qQKEc6GFj0BhMpFtrmT4Ss94odbKQkUetLPk7hvVp) + +Why oh why are you spending time and money on a bridge that helps these two kings while neglecting your own land? + +## Rule #4: Operate like your castle can get shutdown tomorrow + +Do you have a huge following on Twitter? Is everyone who would buy something from you following you only on Youtube? What would happen if tomorrow Twitter didn’t exist? Or Youtube? How would you communicate to them? Do not over-invest in one platform. Always try to get them back to your kingdom. + +Don’t wait until it is too late. From the first day you start in that rival kingdom you should be pushing followers back to your home kingdom. + +Yes you can still build a little castle in their land, but build a cheap one that you can afford to lose. Then you should stand on the top of that cheap castle and say “follow me over there!” as you gesture wildly to platforms owned by you. + +## Rule #5: Be suspicious of new kingdoms that give away easy visibility + +Just about every 2 years there is some new “hot” social media site where you get tons of “engagement” and followers come easy? + +Remember Snapchat? + +The new guys are most definitely going to let you down soon.  + +Here is why. Social media platforms are capitalist enterprises and they are trying to steal market share away from rival social media sites. They are giving away so much free visibility, free engagement, cheap paid advertising to screw their rival social network. + +This is temporary. The party won’t last. They will eventually decide to stop giving away traffic and decide to monetize their subscribers (meaning charge you to reach your fans).  + +Right now I see there are 2 platforms that are new and very risky. + +#### **TikTok** + +I have to be honest, they are giving away a LOT of free traffic. [And it seems to be pretty good traffic](https://youtu.be/l_J22g4CaDs). It is attractive! I also hear their paid ad rates are very good. However this is because they are trying to grow as fast as they can. They are giving away free traffic right now to steal from other social networks. The good times will end. + +My recommendation is yes, try to go viral on TikTok then make a CTA to follow you on your mailing list. Also don’t set your profile link to a LinkTree (that is another platform you don’t own!) Instead, link to your own URL that is designed to quickly shuffle them to where you want them to. + +#### **Discord** + +Yes everyone is on Discord right now but the company is still privately held and doesn’t have a solid business model yet. [Also concerning is that they are explicitly anti-ads.](https://www.npr.org/2021/04/01/983159051/why-does-discord-not-use-ads-and-why-is-microsoft-interested-we-asked-discords-c) I know it sounds good to be anti ad but that means they have no clear way to monetize. That means they could stagnate without any clear growth model and a more aggressive rival will take market share from them (this is what happened to Myspace when Facebook showed up.) + +Discord is also run by a small company relative to the amount of mindshare they have. This means they are ripe to be purchased by a bigger company. And when a big company takes over, we will always get screwed by the changes they make (Just look at Tumblr and Instagram and WhatsApp and Twitch for examples of this). + +## Rule #6: Give good reasons to go back to the Castle in your Kingdom. And be persistent! + +It is hard to make people leave a social media site. But you need to work hard at it.  + +With every single person who enters your castle in a foreign land, tell them “welcome, yes my castle is nice here, but did you know I do better stuff over there in that Kingdom across the sea?” + +Always be working to get people over to your land.  + +Use lead magnets to entice people to follow you back to your kingdom. For example + +- Yes you went viral on TikTok but you should use that visibility to tell them that to learn more they should join your mailing list.  +- Announce you are holding a contest on all the social media platforms but to enter the contest they have to join your mailing list. +- Hold a beta but instead of running it on Discord (another kingdom) tell them to sign up for the beta on your mailing list.  +- Tease major reveals and exclusive news about your game on Social media but tell them that the announcement will be made on your website and mailing list first. +- Give exclusive codes out to your mailing list. + +You also need to be persistent. This isn’t a one time thing. Every week you should be telling them to follow you back to your kingdom. It will be slow, but building a TRUE following takes time. + +I am not the first one to point this out: + +![](https://lh3.googleusercontent.com/UWlxXfmI5Ss0hIwKsq80D_cHlQS4B4ra0YOJiWjb9hFV-ZVPatGv0NtMIG4MDubxp7iPgDyrHyKb6giHEKwMTcAtBTj23Vsmc1WOHqH_3tYxcqYukT5KsTM29W0mphv3xgd7WW8i) \ No newline at end of file diff --git a/Pages/The Ultimate Guide to Error Handling in Python.md b/Pages/The Ultimate Guide to Error Handling in Python.md new file mode 100644 index 0000000..1e7227e --- /dev/null +++ b/Pages/The Ultimate Guide to Error Handling in Python.md @@ -0,0 +1,333 @@ +[blog.miguelgrinberg.com](https://blog.miguelgrinberg.com/post/the-ultimate-guide-to-error-handling-in-python) + +Miguel Grinberg + +24–30 minutes + +--- + +I often come across developers who know the mechanics of Python error handling well, yet when I [review their code](https://blog.miguelgrinberg.com/post/hire-me) I find it to be far from good. Exceptions in Python is one of those areas that have a surface layer that most people know, and a deeper, almost secret one that a lot of developers don't even know exists. If you want to test yourself on this topic, see if you can answer the following questions: + +- When should you catch exceptions raised by functions you call, and when should you not? +- How can you know what exception classes to catch? +- When you catch an exception, what should you do to "handle" it? +- Why is catching all exceptions considered a bad practice, and when is it okay to do it? + +Are you ready to learn the secrets of error handling in Python? Let's go! + +## The Basics: Two Paths to Error Handling in Python + +I'm going to start with something that I believe many of my readers already know or have seen discussed elsewhere. In Python, there are two main styles of writing error handling code, often called by their unpronounceable acronyms of "LBYL" and "EAFP". Are you familiar with these? In case you are not, below is a quick introduction to them. + +### Look Before You Leap (LBYL) + +The "look before you leap" pattern for error handling says that you should check that the conditions for performing an action that can fail are proper before triggering the action itself. + +``` +if can_i_do_x(): + do_x() +else: + handle_error() +``` + +Consider as an example the task of deleting a file from disk. Using, LBYL this could be coded as follows: + +``` +if os.path.exists(file_path): + os.remove(file_path) +else: + print(f"Error: file {file_path} does not exist!") +``` + +While as a first impression it may appear that this code is fairly robust, in practice it isn't. + +The main problem here is that we need to know all the possible things that can go wrong with deleting a file so that we can check for them before we make the `remove()` call. It is obvious that the file must exist, but a missing file isn't the only reason why a deletion can fail. Here are just a few other reasons why a file may fail to delete: + +- The path could be of a directory instead of a file +- The file could be owned by a different user than the one attempting the deletion +- The file could have read-only permissions +- The disk on which the file is stored could be mounted as a read-only volume +- The file could be locked by another process, a common annoyance on Microsoft Windows + +How would the delete file example above look if we had to add checks for all these as well? + +As you see, it is quite difficult to write robust logic using LBYL, because you have to know all the possible ways in which the functions that you call can fail, and sometimes there are just too many. + +Another problem when using the LBYL pattern is that of race conditions. If you check for the failure conditions, and then execute the action, it is always possible for the conditions to change in the small window of time between when the checks were made and when the action was executed. + +### Easier to Ask Forgiveness than Permission (EAFP) + +I'm sure you realize that I don't have a very high opinion of the LBYL pattern (but in fact it is useful in some situations, as you will see later). The competing pattern says that it is "easier to ask forgiveness than permission". What does this mean? It means you should perform the action, and deal with any errors afterwards. + +In Python, EAFP is best implemented using exceptions: + +``` +try: + do_x() +except SomeError: + handle_error() +``` + +Here is how to delete a file using EAFP: + +``` +try: + os.remove(file_path) +except OSError as error: + print(f"Error deleting file: {error}") +``` + +I hope you agree that in most cases EAFP is preferable to LBYL. + +It is a big improvement that with this pattern the target function is tasked with checking and reporting errors, so we as callers can make the call and trust that the function will let us know if the action failed. + +On the other side, we need to know what exceptions to write down in the `except` clause, because any exception classes that we miss are going to bubble up and potentially cause the Python application to crash. For a file deletion it is safe to assume that any errors that are raised are going to be `OSError` or one of its subclasses, but in other cases knowing what exceptions a function could raise requires looking at documentation or source code. + +You may ask why not catch all possible exceptions to make sure none are missed. This is a bad pattern that causes more problems than it solves, so I do not recommend it except in a few very specific cases that I will discuss later. The problem is that usually bugs in your own code manifest themselves as unexpected exceptions. If you are catching and silencing all exceptions every time you call a function, you are likely to miss the exceptions that shouldn't have occurred, the ones that were caused by bugs that need to be fixed. + +To avoid the risk of missing application bugs that manifest as unexpected exceptions, you should always catch the smallest possible list of exception classes, and when it makes sense, don't catch any exceptions at all. Hold on to the thought of not catching exceptions as an error handling strategy. It may sound like a contradiction, but it isn't. I will come back to this. + +## Python Error Handling in the Real World + +Unfortunately the traditional error handling knowledge doesn't go very far. You can have a complete understanding of LBYL and EAFP and know how `try` and `except` work by heart, and still, many times you may not know what to do or feel that the way you write error handling code could be better. + +So now we are going to look at errors in a completely different way that is centered around the errors themselves, and not so much on the techniques to deal with them. I hope this is going to make it much easier for you to know what to do. + +### New Errors vs. Bubbled-Up Errors + +First, we need to classify the error based on its origin. There are two types to consider: + +- Your code found a problem and needs to generate an error. I'll call this type a "new error". +- Your code received an error from a function it called. I'll call this one a "bubbled-up error". + +When it comes down to it, these are really the two situations in which errors may come to exist, right? You either need to introduce a new error yourself and put it in the system for some other part of the application to handle, or you received an error from somewhere else and need to decide what to do with it. + +In case you are not familiar with the expression "bubbled-up", this is an attribute of exceptions. When a piece of code raises an exception, the caller of the errored function gets a chance to catch the exception in a `try`/`except` block. It the caller doesn't catch it, then the exception is offered to the next caller up the call stack, and this continues until some code decides to catch the exception and handle it. When the exception travels towards the top of the call stack it is said to be "bubbling up". If the exception isn't caught and bubbles up all the way to the top, then Python will interrupt the application, and this is when you see a stack trace with all the levels through which the error traveled, a very useful debugging aid. + +### Recoverable vs. Non-Recoverable Errors + +Aside from the error being new or bubbled-up, you need to decide if it is recoverable or not. A recoverable error is an error that the code dealing with it can correct before continuing. For example, if a piece of code tries to delete a file and finds that the file does not exist, it's not a big deal, it can just ignore the error and continue. + +A non-recoverable error is an error that the code in question cannot correct, or in other words, an error that makes it impossible for the code at this level to continue running. As an example, consider a function that needs to read some data from the database, modify it and save it back. If the reading fails, the function has to abort early, since it cannot do the rest of the work. + +Now you have an easy way to categorize an error based on its origin and its recoverable status, resulting in just four different error configurations that you need to know how to handle. In the following sections I will tell you exactly what you need to do for each of these four error types! + +### Type 1: Handling New Recoverable Errors + +This is an easy case. You have a piece of code in your own application that found an error condition. Luckily this code is able to recover from this error itself and continue. + +What do you think is the best way to handle this case? Well, recover from the error and continue, without bothering anyone else! + +Let's look at an example: + +``` +def add_song_to_database(song): + # ... + if song.year is None: + song.year = 'Unknown' + # ... +``` + +Here we have a function that writes a song to a database. Let's say that in the database schema the song's year cannot be null. + +Using ideas from the LBYL pattern we can check if the year attribute of the song is not set, to prevent a database write to fail. How do we recover from the error? In this case we set the year to unknown and we keep going, knowing that the database write is not going to fail (from this one reason, at least). + +Of course, how to recover from an error is going to be very specific to each application and error. In the example above I'm assuming that the song's year is stored as a string in the database. If it is stored as a number then maybe setting the year to `0` is an acceptable way to handle songs with an unknown year. In another application the year may be required, in which case this wouldn't be a recoverable error for that application. + +Makes sense? If you find a mistake or inconsistency in the current state of the application, and have a way to correct the state without raising an error, then no need to raise an error, just correct the state and keep going. + +### Type 2: Handling Bubbled-Up Recoverable Errors + +The second case is a variation of the first. Here the error is not a new error, it is an error that bubbles up from a function that was called. As in the previous case, the nature of the error is such that the code that receives the error knows how to recover from it and continue. + +How do we handle this case? We use EAFP to catch the error, and then we do whatever needs to be done to recover from it and continue. + +Here is another part of the `add_song_to_database()` function that demonstrates this case: + +``` +def add_song_to_database(song): + # ... + try: + artist = get_artist_from_database(song.artist) + except NotFound: + artist = add_artist_to_database(song.artist) + # ... +``` + +The function wants to retrieve the artist given with the song from the database, but this is something that may fail from time to time, for example when adding the first song of a given artist. The function uses EAFP to catch the `NotFound` error from the database, and then corrects the error by adding the unknown artist to the database before continuing. + +As with the first case, here the code that needs to handle the error knows how to adjust the state of the application to continue running, so it can consume the error and continue. None of the layers in the call stack above this code need to know that there was an error, so the bubbling up of this error ends at this point. + +### Type 3: Handling New Non-Recoverable Errors + +The third case is a bit more interesting. Now we have a new error of such severity that the code does not know what to do and cannot continue. The only reasonable action that can be taken is to stop the current function and alert one level up the call stack of the error, with the hope that the caller knows what to do. As discussed above, in Python the preferred mechanism to notify the caller of an error is to raise an exception, so this is what we'll do. + +This strategy works well because of an interesting property of non-recoverable errors. In most cases, a non-recoverable error will eventually become recoverable when it reaches a high enough position in the call stack. So the error can bubble up the call stack until it becomes recoverable, at which point it'll be a type 2 error, which we know how to handle. + +Let's revisit the `add_song_to_database()` function. We've seen that if the year of the song was missing, we decided that can recover and prevent a database error by setting the year to `'Unknown'`. If the song does not have a name, however, it is much harder to know what's the right thing to do at this level, so we can say that a missing name is a non-recoverable error for this function. Here is how we handle this error: + +``` +def add_song_to_database(song): + # ... + if song.name is None: + raise ValueError('The song must have a name') + # ... +``` + +The choice of what exception class to use really depends on the application and your personal taste. For many errors the exceptions that come with Python can be used, but if none of the built-in exceptions fit, then you can always create your own exception subclasses. Here is the same example implemented with a custom exception: + +``` +class ValidationError(Exception): + pass + +# ... + +def add_song_to_database(song): + # ... + if song.name is None: + raise ValidationError('The song must have a name') + # ... +``` + +The important thing to note here is that the `raise` keyword interrupts the function. This is necessary because we said that this error cannot be recovered, so the rest of the function after the error will not be able to do what it needs to do and should not run. Raising the exception interrupts the current function and starts the bubbling up of the error starting from the closest caller and continuing up the call stack until some code decides to catch the exception. + +### Type 4: Handling Bubbled-Up Non-Recoverable Errors + +Okay, we have one last error type to review, and this is actually the most interesting of all and also my favorite. + +Now we have a piece of code that called some function, the function raised an error, and we in our function have no idea how to fix things up so that we can continue, so we have to consider this error as non-recoverable. What do we do now? + +The answer is going to surprise you. In this case we do absolutely nothing! + +I've mentioned earlier that not handling errors can be a great error handling strategy, and this is exactly what I meant. +Let me show you an example of how it looks to handle an error by doing nothing: + +``` +def new_song(): + song = get_song_from_user() + add_song_to_database(song) +``` + +Let's say that both functions called in `new_song()` can fail and raise exceptions. Here are a couple of examples of things that can go wrong with these functions: + +- The user could press Ctrl-C while the application is waiting for input inside `get_song_from_user()`, or in the case of a GUI application, the user could click a Close or Cancel button. +- While inside either one of the functions, the database can go offline due to a cloud issue, causing all queries and commits to fail for some time. + +If we have no way to recover from these errors, then there is no point in catching them. Doing nothing is actually the most useful thing you can do, as it allows the exceptions to bubble up. Eventually the exceptions will reach a level at which the code knows how to do recovery, and at that point they will be considered type 2 errors, which are easily caught and handled. + +You may think that this is an exceptionally rare situation to be in. I think you are wrong. In fact, you should design your applications so that as much code as possible is in functions that do not need to concern themselves with error handling. Moving error handling code to higher-level functions is a very good strategy that helps you have clean and maintainable code. + +I expect some of you may disagree. Maybe you think that the `add_song()` function above should at least print an error message to inform the user that there was a failure. I don't disagree, but let's think about that for a bit. Can we be sure that we have a console to print on? Or is this a GUI application? GUIs do not have `stdout`, they present errors to users visually through some sort of alert or message box. Maybe this is a web application instead? In web apps you present errors by returning an HTTP error response to the user. Should this function know which type of application this is and how errors are to be presented to the user? The [separation of concerns](https://en.wikipedia.org/wiki/Separation_of_concerns) principle says that it should not. + +Once again, I'll reiterate that doing nothing in this function does not mean that the error is being ignored, it means that we are allowing the error to bubble up so that some other part of the application with more context can deal with it appropriately. + +### Catching All Exceptions + +One of the reasons you may be doubting that type 4 errors should be the most common in your application is that by letting exceptions bubble up freely they may go all the way to the top without being caught anywhere else, causing the application to crash. This is a valid concern that has an easy solution. + +You should design your applications so that it is impossible for an exception to ever reach the Python layer. And you do this by adding a `try`/`except` block at the highest level that catches the runaway exceptions. + +If you were writing a command line application, you could do this as follows: + +``` +import sys + +def my_cli() + # ... + +if __name__ == '__main__': + try: + my_cli() + except Exception as error: + print(f"Unexpected error: {error}") + sys.exit(1) +``` + +Here the top-level of this application is in the `if __name__ == '__main__'` conditional, and it considers any errors that reach this level as recoverable. The recovery mechanism is to show the error to the user and to exit the application with a exit code of `1`, which will inform the shell or the parent process that the application failed. With this logic the application knows how to exit with failure, so now there is no need to reimplement this anywhere else. The application can simply let errors bubble up, and they'll eventually be caught here, where the error message will be shown and the application will then exit with an error code. + +You may remember that I've mentioned above that catching all exceptions is a bad practice. Yet, that is exactly what I'm doing here! The reason is that at this level we really cannot let any exceptions reach Python because we do not want this program to ever crash, so this is the one situation in which it makes sense to catch all exceptions. This is the exception (pun intended) that proves the rule. + +Having a high-level catch-all exception block is actually a common pattern that is implemented by most application frameworks. Here are two examples: + +- **The [Flask](https://flask.pallets.com/) web framework**: Flask considers each request as a separate run of the application, with the `full_dispatch_request()` method as the top layer. The code that catches all exceptions is [here](https://github.com/pallets/flask/blob/2fec0b206c6e83ea813ab26597e15c96fab08be7/src/flask/app.py#L893-L900). +- **The Tkinter GUI toolkit** (part of the Python standard library): Tkinter considers each application event handler as separate little run of the application, and adds a generic catch-all exception block each time it calls a handler, to prevent faulty application handlers from ever crashing the GUI. See the code [here](https://github.com/python/cpython/blob/b3e2c0291595edddc968680689bec7707d27d2d1/Lib/tkinter/__init__.py#L1965-L1972). In this snippet note how Tkinter allows the `SystemExit` exception (indicating the application is exiting) to bubble up, but catches every other one to prevent a crash. + +### An Example + +I want to show you an example of how you can improve your code when using a smart design for error handling. For this I'm going to use Flask, but this applies to most other frameworks or application types as well. + +Let's say this is a database application that uses the Flask-SQLAlchemy extension. Through my consulting and code review work I see lots of developers coding database operations in Flask endpoints as follows: + +``` +# NOTE: this is an example of how NOT to do exception handling! +@app.route('/songs/', methods=['PUT']) +def update_song(id): + # ... + try: + db.session.add(song) + db.session.commit() + except SQLAlchemyError: + current_app.logger.error('failed to update song %s, %s', song.name, e) + try: + db.session.rollback() + except SQLAlchemyError as e: + current_app.logger.error('error rolling back failed update song, %s', e) + return 'Internal Service Error', 500 + return '', 204 +``` + +Here this route attempts to save a song to the database, and catches database errors, which are all subclasses of the `SQLAlchemyError` exception class. If the error occurs, it writes an explanatory message to the log, and then rolls back the database session. But of course, the rollback operation can also fail sometimes, so there is a second exception catching block to catch rollback errors and also log them. After all this, a 500 error is returned to the user so that they know that there was a server error. This pattern is repeated in every endpoint that writes to the database. + +This is a very bad solution. First of all, there is nothing that this function can do to recover a rollback error. If a rollback error occurs that means the database is in big trouble, so you will likely continue to see errors, and logging that there was a rollback error is not going to help you in any way. Second, logging an error message when a commit fails appears useful at first, but this particular log lacks information, especially the stack trace of the error, which is the most important debugging tool you will need later when figuring out what happened. At the very least, this code should use `logger.exception()` instead of `logger.error()`, since that will log an error message plus a stack trace. But we can do even better. + +This endpoint falls in the type 4 category, so it can be coded using the "doing nothing" approach, resulting in a much better implementation: + +``` +@app.route('/songs/', methods=['PUT']) +def update_song(id): + # ... + db.session.add(song) + db.session.commit() + return '', 204 +``` + +Why does this work? As you've seen before, Flask catches all errors, so your application will never crash due to missing to catch an error. As part of its handling, Flask will log the error message and the stack trace to the Flask log for you, which is exactly what we want, so no need to do this ourselves. Flask will also return a 500 error to the client, to indicate that an unexpected server error has occurred. In addition, the Flask-SQLAlchemy extension attaches to the exception handling mechanism in Flask and rolls back the session for you when a database error occurs, the last important thing that we need. There is really nothing left for us to do in the route! + +The recovery process for database errors is the same in most applications, so you should let the framework do the dirty work for you, while you benefit from much simpler logic in your own application code. + +### Errors in Production vs. Errors in Development + +I mentioned that one of the benefits of moving as much of the error handling logic as possible to the higher layers of the application call stack is that your application code can let those errors bubble up without having to catch them, resulting in much easier to maintain and readable code. + +Another benefit of moving the bulk of error handling code to a separate part of the application is that with the error handling code in a single place you have better control of how the application reacts to errors. The best example of this is how easy it becomes to change the error behavior on the production and development configurations of your application. + +During development, there is actually nothing wrong with the application crashing and showing a stack trace. In fact, this is a good thing, since you want errors and bugs to be noticed and fixed. But of course, the same application must be rock solid during production, with errors being logged and developers notified if feasible, without leaking any internal or private details of the error to the end user. + +This becomes much easier to implement when the error handling is in one place and separate from the application logic. Let's go back to the command line example I shared earlier, but now let's add development and production modes: + +``` +import sys + +mode = os.environ.get("APP_MODE", "production") + +def my_cli() + # ... + +if __name__ == '__main__': + try: + my_cli() + except Exception as error: + if mode == "development": + raise # in dev mode we let the app crash! + else: + print(f"Unexpected error: {error}") + sys.exit(1) +``` + +Isn't this wonderful? When we are running in development mode we now re-raise the exceptions to cause the application to crash, so that we can see the errors and the stack traces while working. But we do this without compromising the robustness of the production version, which continues to catch all errors and prevent crashes. More importantly, the application logic does not need to know of these configuration differences. + +Does this remind of you of anything Flask, Django and other web frameworks do? Many web frameworks have a development or debug mode, which shows you crashes in your console and sometimes even in your web browser. Exactly the same solution I'm showing you on a made-up CLI application, but applied to a web application! + +## Conclusion + +I hope you've learned a thing or two from this article and as a result you are able to write better error handling code for your projects! If there are any questions that you aren't clear on, feel free to write them below and I'll do my best to address them. \ No newline at end of file diff --git a/Pages/Tracking the music I listen to.md b/Pages/Tracking the music I listen to.md new file mode 100644 index 0000000..d29ff9f --- /dev/null +++ b/Pages/Tracking the music I listen to.md @@ -0,0 +1,502 @@ +[coryd.dev](https://coryd.dev/posts/2024/tracking-the-music-i-listen-to/) + +20–25 minutes + + + +[I've talked about building my own music scrobbler](https://coryd.dev/posts/2024/building-a-scrobbler-using-plex-webhooks-edge-functions-and-blob-storage/), [I've talked about improving it](https://coryd.dev/posts/2024/improving-my-self-hosted-scrobbling-implementation/) and [I've complained about wanting to stream my own music](https://coryd.dev/posts/2023/i-dont-want-streaming-music-i-just-want-to-stream-my-music/) and then [I wrote a retrospective about it](https://coryd.dev/posts/2024/a-retrospective-on-a-year-without-streaming-music/). I've settled on something that works, so let's look at it in a bit more detail. + +## [Table of contents](about:reader?url=https%3A%2F%2Fcoryd.dev%2Fposts%2F2024%2Ftracking-the-music-i-listen-to%2F#table-of-contents) + +1. [Why build this?](about:reader?url=https%3A%2F%2Fcoryd.dev%2Fposts%2F2024%2Ftracking-the-music-i-listen-to%2F#why-build-this) +2. [Data collection](about:reader?url=https%3A%2F%2Fcoryd.dev%2Fposts%2F2024%2Ftracking-the-music-i-listen-to%2F#data-collection) +3. [Displaying the data](about:reader?url=https%3A%2F%2Fcoryd.dev%2Fposts%2F2024%2Ftracking-the-music-i-listen-to%2F#displaying-the-data) +4. [The rest of the data](about:reader?url=https%3A%2F%2Fcoryd.dev%2Fposts%2F2024%2Ftracking-the-music-i-listen-to%2F#the-rest-of-the-data) +5. [Ok that's enough / Conclusion](about:reader?url=https%3A%2F%2Fcoryd.dev%2Fposts%2F2024%2Ftracking-the-music-i-listen-to%2F#conclusion) + +## [Why build this?](about:reader?url=https%3A%2F%2Fcoryd.dev%2Fposts%2F2024%2Ftracking-the-music-i-listen-to%2F#why-build-this%3F) + +I can, so I must.[[1]](about:reader?url=https%3A%2F%2Fcoryd.dev%2Fposts%2F2024%2Ftracking-the-music-i-listen-to%2F#fn1) + +I get to own the data: it sits there, pristine in a table over at [Supabase](https://supabase.com/). I can add properties, run migrations, count the rows by hand — all the things you'd normally do with raw data. + +My metadata remains exactly as I've defined it. It lines up from [Plex](https://plex.tv/) directly to what's displayed on my site. Genres, artist images — a little tedious at first, but I tend to be pretty neurotic and find the consistency satisfying. + +Routing my listening through [Plex](https://plex.tv/) means I get the benefits of streaming music without the restrictions. Apple's not deduplicating my tracks and making a mess of the genres. The album I was listening to yesterday isn't gone today because a licensing deal changed. I'm listening to _You'd Prefer an Astronaut_ not _You'd Prefer an Astronaut (Deluxe) (Remastered + Bonus tracks - 2022)_. + +I own all the music I listen to. I pay artists for it (directly — whenever I can). It's on a hard drive that's backed up to B2 and GCP. The music directory is linked to Google Drive too. + +[Last.fm](https://last.fm/) _does_ exist. I've got nothing but fond memories and I still reference it's recommendations sometimes. It feels abandoned (or — at the very least — neglected). I'm also wary of the fact that it gets dragged around with Paramount as they continue to change hands (are they going to unplug that server rack? Use it for AI training data? Do the latter then the former? Who knows). + +[ListenBrainz](https://listenbrainz.org/) exists too. It's quite nice and it's operated by the fine folks at [MusicBrainz](https://musicbrainz.org/). They're wonderful and I send my listen data their way too. + +I own the files, I own the data and I control the experience. On we go. + +## [Data collection](about:reader?url=https%3A%2F%2Fcoryd.dev%2Fposts%2F2024%2Ftracking-the-music-i-listen-to%2F#data-collection) + +When I started out with this I was collecting listening data into giant JSON blobs stored in [Netlify's blob storage](https://docs.netlify.com/blobs/overview/). I'm a front end developer. I like JSON. _This was a terrible idea._ I mean, it worked but it's one of those things you force to work and then realize you've made a terrible mistake. + +A good friend had been recommending [Supabase](https://supabase.com/). I didn't think I had a use for it. Turns out I was wrong. + +I'd been mirroring listens to ListenBrainz, so I dumped my history from there to JSON, wrote a node script I'm not proud of and imported it into a proper database. Phew. I was nowhere near done (I was near the entrance of the rabbit hole at that point). + +Again — me, front end developer — I got to learn about foreign key relationships and all sorts of database mechanics. **It's funny how trying to build something you're passionate about makes learning fun**. It's also worth noting that working with (making mistakes with) and configuring [Directus](https://directus.io/) on top of [Supabase](https://supabase.com/) was a helpful experience. Seeing how the former configured the latter provided quite a bit of insight I'm still grateful for. + +**Databases are cool.** Use the right tool for the right job and all that. Just because you're good with a hammer doesn't mean it should replace a table saw. Whatever. + +Alright, we're storing data. Plex makes sending it convenient because [Plex](https://plex.tv/) offers [webhooks](https://github.com/plexinc/webhooks-notifications) and _they even tag the event type_. Webhooks get sent off to a Cloudflare Worker, the worker watches for `media.scrobble` events (see — super handy). When I get a `media.scrobble` event, I write a insert a listen row to my `listens` table. Listens are connected to the albums table using a key that consists of `artist-name-album-name`. They're connected to the artist table by explicitly matching the artist name. Fragile? Maybe, but that's the metadata available in the webhook payload. + +Let's step through the worker (I've added handy dandy comments to the code — or ["Cory I don't care about code, skip this part"](about:reader?url=https%3A%2F%2Fcoryd.dev%2Fposts%2F2024%2Ftracking-the-music-i-listen-to%2F#displaying-the-data)): + +``` +import { createClient } from '@supabase/supabase-js' +import { DateTime } from 'luxon' +import slugify from 'slugify' + +/* + We use the native normalize method to get strings with diacritics and other characters removed that make constructing keys difficult. We replace problematic characters with some poorly written regexes and then slugify the string (with some more sanitization — redundant? Perhaps). +*/ +const sanitizeMediaString = (str) => { + const sanitizedString = str + .normalize('NFD') + .replace(/[\u0300-\u036f\u2010\-\.\?\(\)\[\]\{\}]/g, '') + .replace(/\.{3}/g, '') + return slugify(sanitizedString, { + replacement: '-', + remove: /[#,&,+()$~%.'":*?<>{}]/g, + lower: true, + }) +} + +/* + I route email notifications through forwardemail.net. My email addresses are named creatively. +*/ +const sendEmail = async (subject, text, authHeader, maxRetries = 3) => { + const emailData = new URLSearchParams({ + from: 'hi@admin.coryd.dev', + to: 'hi@coryd.dev', + subject: subject, + text: text, + }).toString() + + let attempt = 0 + let success = false + + while (attempt < maxRetries && !success) { + attempt++ + try { + const response = await fetch('https://api.forwardemail.net/v1/emails', { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'Authorization': authHeader, + }, + body: emailData, + }) + + if (!response.ok) { + const responseText = await response.text() + console.error(`Attempt ${attempt}: Email API response error:`, response.status, responseText) + throw new Error(`Failed to send email: ${responseText}`) + } + + console.log('Email sent successfully on attempt', attempt) + success = true + } catch (error) { + console.error(`Attempt ${attempt}: Error sending email:`, error.message) + + if (attempt < maxRetries) { + console.log(`Retrying email send (attempt ${attempt + 1}/${maxRetries})...`) + } else { + console.error('All attempts to send email failed.') + } + } + } + + return success +} + +export default { + async fetch(request, env) { + const SUPABASE_URL = env.SUPABASE_URL + const SUPABASE_KEY = env.SUPABASE_KEY + const FORWARDEMAIL_API_KEY = env.FORWARDEMAIL_API_KEY + const ACCOUNT_ID_PLEX = env.ACCOUNT_ID_PLEX + const supabase = createClient(SUPABASE_URL, SUPABASE_KEY) + const authHeader = 'Basic ' + btoa(`${FORWARDEMAIL_API_KEY}:`) + const url = new URL(request.url) + const params = url.searchParams + const id = params.get('id') + + if (!id) return new Response(JSON.stringify({ status: 'Bad request' }), { + headers: { 'Content-Type': 'application/json' }, + }) + + if (id !== ACCOUNT_ID_PLEX) return new Response(JSON.stringify({ status: 'Forbidden' }), { + headers: { 'Content-Type': 'application/json' }, + }) + + const contentType = request.headers.get('Content-Type') || '' + if (!contentType.includes('multipart/form-data')) return new Response( + JSON.stringify({ + status: 'Bad request', + message: 'Invalid Content-Type. Expected multipart/form-data.', + }), + { headers: { 'Content-Type': 'application/json' } } + ) + + try { + const data = await request.formData() + const payload = JSON.parse(data.get('payload')) + + /* + There's that event we talked about earlier. + */ + if (payload?.event === 'media.scrobble') { + const artistName = payload['Metadata']['grandparentTitle'] + const albumName = payload['Metadata']['parentTitle'] + const trackName = payload['Metadata']['title'] + const listenedAt = Math.floor(DateTime.now().toSeconds()) + const artistKey = sanitizeMediaString(artistName) + const albumKey = `${artistKey}-${sanitizeMediaString(albumName)}` + + /* + Use the payload data to see if the artist for the listen exists. If it doesn't, email myself with the artist name and pertinent metadata that I'll need to correct the record or populate it. + */ + let { data: artistData, error: artistError } = await supabase + .from('artists') + .select('*') + .ilike('name_string', artistName) + .single() + + if (artistError && artistError.code === 'PGRST116') { + const { error: insertArtistError } = await supabase + .from('artists') + .insert([ + { + mbid: null, + art: '4cef75db-831f-4f5d-9333-79eaa5bb55ee', + name: artistName, + tentative: true, + total_plays: 0, + }, + ]) + + if (insertArtistError) { + console.error('Error inserting artist: ', insertArtistError.message) + return new Response( + JSON.stringify({ + status: 'error', + message: insertArtistError.message, + }), + { headers: { 'Content-Type': 'application/json' } } + ) + } + + await sendEmail( + 'New tentative artist record', + `A new tentative artist record was inserted:\n\nArtist: ${artistName}\nKey: ${artistKey}`, + authHeader + ) + + ;({ data: artistData, error: artistError } = await supabase + .from('artists') + .select('*') + .ilike('name_string', artistName) + .single()) + } + + if (artistError) { + console.error('Error fetching artist:', artistError.message) + return new Response( + JSON.stringify({ status: 'error', message: artistError.message }), + { headers: { 'Content-Type': 'application/json' } } + ) + } + + /* + The same thing we did for artists, but for albums. The value assigned to `art` is a placeholder image so that these temporary records don't yield broken images. + */ + let { data: albumData, error: albumError } = await supabase + .from('albums') + .select('*') + .ilike('key', albumKey) + .single() + + if (albumError && albumError.code === 'PGRST116') { + const { error: insertAlbumError } = await supabase + .from('albums') + .insert([ + { + mbid: null, + art: '4cef75db-831f-4f5d-9333-79eaa5bb55ee', + key: albumKey, + name: albumName, + tentative: true, + total_plays: 0, + artist: artistData.id, + }, + ]) + + if (insertAlbumError) { + console.error('Error inserting album:', insertAlbumError.message) + return new Response( + JSON.stringify({ + status: 'error', + message: insertAlbumError.message, + }), + { headers: { 'Content-Type': 'application/json' } } + ) + } + + await sendEmail( + 'New tentative album record', + `A new tentative album record was inserted:\n\nAlbum: ${albumName}\nKey: ${albumKey}\nArtist: ${artistName}`, + authHeader + ) + + ;({ data: albumData, error: albumError } = await supabase + .from('albums') + .select('*') + .ilike('key', albumKey) + .single()) + } + + if (albumError) { + console.error('Error fetching album:', albumError.message) + return new Response( + JSON.stringify({ status: 'error', message: albumError.message }), + { headers: { 'Content-Type': 'application/json' } } + ) + } + + /* + Insert the listen. Finally. + */ + const { error: listenError } = await supabase.from('listens').insert([ + { + artist_name: artistData['name_string'] || artistName, + album_name: albumData['name'] || albumName, + track_name: trackName, + listened_at: listenedAt, + album_key: albumKey, + }, + ]) + + if (listenError) { + console.error('Error inserting listen:', listenError.message) + return new Response( + JSON.stringify({ status: 'error', message: listenError.message }), + { headers: { 'Content-Type': 'application/json' } } + ) + } + + console.log('Listen record inserted successfully') + } + + return new Response(JSON.stringify({ status: 'success' }), { + headers: { 'Content-Type': 'application/json' }, + }) + } catch (e) { + console.error('Error processing request:', e.message) + return new Response( + JSON.stringify({ status: 'error', message: e.message }), + { headers: { 'Content-Type': 'application/json' } } + ) + } + }, +} +``` + +## [Displaying the data](about:reader?url=https%3A%2F%2Fcoryd.dev%2Fposts%2F2024%2Ftracking-the-music-i-listen-to%2F#displaying-the-data) + +I display a bunch (but not all) of this data. I stop displaying charts at 3 months. I _could_ display — say — all time artist, album and track views, but I'd rather keep build times predictable (rather than letting them slowly grow forever). The root music page is available by clicking the headphones in my site's primary navigation. [Or you can click here](https://coryd.dev/music/). Don't scroll up — just go. + +We've got some text, data's interpolated into it — artists, albums, tracks, genres and some calls to action. The last line with the emoji is the only dynamic part — the last played or track currently being played. + +Below that, we've got grids of artists, albums, track charts and albums I'm looking forward to[[2]](about:reader?url=https%3A%2F%2Fcoryd.dev%2Fposts%2F2024%2Ftracking-the-music-i-listen-to%2F#fn2). + +So, how is all of this fetched? Well, I use [Directus](https://directus.io/) to manage my site, so I use the API for that, right? No, no — it's all queries from Supabase directly using their SDK. + +The frontend of my site is built using [11ty](https://www.11ty.dev/) and all of the queries are performed in data files at build time. They _were_ querying the tables directly (I'm a frontend developer — sorry), they _are now_ querying optimized views. Let's look at `artists.js` (or don't): + +``` +import { createClient } from '@supabase/supabase-js' +import { sanitizeMediaString, parseCountryField } from '../../config/utilities/index.js' + +const SUPABASE_URL = process.env.SUPABASE_URL +const SUPABASE_KEY = process.env.SUPABASE_KEY +const supabase = createClient(SUPABASE_URL, SUPABASE_KEY) +const PAGE_SIZE = 1000 + +/* + Page through the artist results until we've queried all of the records. Right now, there are 613 (less than a page for our purposes). We get nearly all of the data associated with the artists and return it. +*/ +const fetchAllArtists = async () => { + let artists = [] + let rangeStart = 0 + + while (true) { + const { data, error } = await supabase + .from('optimized_artists') + .select(` + id, + mbid, + name_string, + tentative, + total_plays, + country, + description, + favorite, + genre, + emoji, + tattoo, + art, + albums, + concerts, + books, + movies, + posts, + related_artists, + shows + `) + .range(rangeStart, rangeStart + PAGE_SIZE - 1) + + if (error) { + console.error('Error fetching artists:', error) + break + } + + artists = artists.concat(data) + if (data.length < PAGE_SIZE) break + rangeStart += PAGE_SIZE + } + + return artists +} + +/* + This used to be more concise, but I went down the road of connecting media types together. Artists were — as previously discussed — connected to albums. But I read books about music, musicians show up in movies, or TV shows, or posts — or they've got related artists (every black metal band in Iceland shares members with one another.) +*/ +const processArtists = (artists) => { + return artists.map(artist => ({ + id: artist['id'], + mbid: artist['mbid'], + name: artist['name_string'], + tentative: artist['tentative'], + totalPlays: artist['total_plays'], + country: parseCountryField(artist['country']), + description: artist['description'], + favorite: artist['favorite'], + genre: artist['genre'], + emoji: artist['emoji'], + tattoo: artist['tattoo'], + image: artist['art'] ? `/${artist['art']}` : '', + url: `/music/artists/${sanitizeMediaString(artist['name_string'])}-${sanitizeMediaString(parseCountryField(artist['country']))}`, + albums: (artist['albums'] || []).map(album => ({ + id: album['id'], + name: album['name'], + releaseYear: album['release_year'], + totalPlays: album['total_plays'], + art: album.art ? `/${album['art']}` : '' + })).sort((a, b) => a['release_year'] - b['release_year']), + concerts: artist['concerts']?.[0]?.['id'] ? artist['concerts'].sort((a, b) => new Date(b['date']) - new Date(a['date'])) : null, + books: artist['books']?.[0]?.['id'] ? artist['books'].map(book => ({ + title: book['title'], + author: book['author'], + isbn: book['isbn'], + description: book['description'], + url: `/books/${book['isbn']}`, + })).sort((a, b) => a['title'].localeCompare(b['title'])) : null, + movies: artist['movies']?.[0]?.['id'] ? artist['movies'].map(movie => ({ + title: movie['title'], + year: movie['year'], + tmdb_id: movie['tmdb_id'], + url: `/watching/movies/${movie['tmdb_id']}`, + })).sort((a, b) => b['year'] - a['year']) : null, + shows: artist['shows']?.[0]?.['id'] ? artist['shows'].map(show => ({ + title: show['title'], + year: show['year'], + tmdb_id: show['tmdb_id'], + url: `/watching/shows/${show['tmdb_id']}`, + })).sort((a, b) => b['year'] - a['year']) : null, + posts: artist['posts']?.[0]?.['id'] ? artist['posts'].map(post => ({ + id: post['id'], + title: post['title'], + date: post['date'], + slug: post['slug'], + url: post['slug'], + })).sort((a, b) => new Date(b['date']) - new Date(a['date'])) : null, + relatedArtists: artist['related_artists']?.[0]?.['id'] ? artist['related_artists'].map(relatedArtist => { + relatedArtist['url'] = `/music/artists/${sanitizeMediaString(relatedArtist['name'])}-${sanitizeMediaString(parseCountryField(relatedArtist['country']))}` + return relatedArtist + }).sort((a, b) => a['name'].localeCompare(b['name'])) : null, + })) +} + +export default async function () { + try { + const artists = await fetchAllArtists() + return processArtists(artists) + } catch (error) { + console.error('Error fetching and processing artists data:', error) + return [] + } +} +``` + +My whole site has slowly been integrated with [Directus](https://directus.io/) and [Supabase](https://supabase.com/). If you'd like to see all of the data files, take a look at [the source for my site's frontend](https://github.com/cdransf/coryd.dev/tree/main/src/data). The `artists.js` is _relatively_ concise but — I hope — illustrative. + +_Plex to Cloudflare, Cloudflare to Supabase, Supabase to Directus, Supabase to 11ty. Round and round we go._ + +I started out looking to track music and I ended up moving all of the content for my site into a [CMS](https://coryd.dev/posts/2024/it-turns-out-a-cms-can-be-pretty-awesome/). I've got music in here — why not books? Movies? TV? Posts? Pages? `robots.txt`? Uhhh...links? Yeah links. + +_Anyways._ All of this is built hourly over at [Cloudflare](https://cloudflare.com/). The only network call is for the `now playing` web component. If you've got JavaScript disabled it'll show the last played track at the time the site was built. Why make it static? Well — I don't _need_ to see this data live. I don't think any visitors do either. If I need to review something I'll pop into [Directus](https://directus.io/). + +## [The rest of the data](about:reader?url=https%3A%2F%2Fcoryd.dev%2Fposts%2F2024%2Ftracking-the-music-i-listen-to%2F#the-rest-of-the-data) + +Artist bios have been bootstrapped and painstakingly edited to include links, markdown formatting and media relationships. I like links (they're all over this post). Genre data is sourced from [Wikipedia](https://wikipedia.org/) and each description links out to the appropriate page. + +Artist images are from — all over — I guess. + +Album images are dumped from file tags. I throw them in a shared album that I point Apple's photos screensaver at. Shifting tiles of album art. + +When I add a new artist, it goes like this: + +1. Add their name. +2. Associate the appropriate genre. +3. Add an artist image. +4. Enter the appropriate country code. +5. Add a bio and format it. +6. Add their [MusicBrainz](https://musicbrainz.org/) ID — this is an autocomplete field in [Directus](https://directus.io/) that queries the [MusicBrainz](https://musicbrainz.org/) API. +7. Optionally: mark them as a favorite, add an emoji (or combination thereof) that show up when I'm listening to them, indicate whether I've got a tattoo inspired by them and associate any related media on the site. +8. Tag the artist's music using [Meta](https://www.nightbirdsevolve.com/meta/). +9. Export the album art. +10. Add the album art to my screensaver. +11. Add the artist's music to Plex. +12. Add the artist image and genre's that match my site. +13. Add albums for the artist. + 1. Add an album name. + 2. Add the album key. + 3. Add the release [MusicBrainz](https://musicbrainz.org/) ID (this field also queries their API). + 4. Add album art. + 5. Add the artist's name string that connects to the `listens` table. + 6. Add the release year. + 7. Repeat for the next album. +14. Enjoy. + +## [Conclusion](about:reader?url=https%3A%2F%2Fcoryd.dev%2Fposts%2F2024%2Ftracking-the-music-i-listen-to%2F#conclusion) + +Is this worth doing? Should I do it? I dunno — does it sound worth it? I'm _thrilled_ with it. It's not a novel application so much as it is a composition of parts. My site's primarily a blog but it also has, well, _all of this_ built into it. It's very much a personal site. It's also a sisyphean task. + +Spend time on what you enjoy — I've spent a lot on this, I've learned a whole bunch and I'm quite happy with the result. I'm paying for some more infrastructure, but I'm also _not_ paying for The Storygraph (which _is a very nice service_), Trakt, [Last.fm](http://last.fm/), Letterboxd etc. etc. Tradeoffs. + +If I've enjoyed something, it lands in a section in my site. If I share it, I share the link. It's on me to keep that link alive and preserve that. I like that quite a bit too. + +I love music. I've built a site that reflects that. Thanks for reading and happy listening. + +--- + +1. I'm kidding — really. [↩︎](about:reader?url=https%3A%2F%2Fcoryd.dev%2Fposts%2F2024%2Ftracking-the-music-i-listen-to%2F#fnref1) + +2. These are associated with artists, tagged with a release date and link and rendered — [I generate a calendar subscription and feeds for them too](https://coryd.dev/feeds). [↩︎](about:reader?url=https%3A%2F%2Fcoryd.dev%2Fposts%2F2024%2Ftracking-the-music-i-listen-to%2F#fnref2) \ No newline at end of file