Switched to using beautifulsoup directly

README.md

@@ -0,0 +1,53 @@
+Logs into Nebula as the configured user, fetches available videos, and generates an RSS feed. No feed will be generated, if less than *feed_cache_time_seconds* seconds (set in config.json) have elapsed since last running this script.
+
+Requirements
+============
+
+The following are needed to run this script:
+- Python, version 3.8 to 3.10: I have tested 3.8.10 and 3.10.0. Older versions might work, but 3.6 is a hard minimum
+- Firefox: It should be fairly easy to switch to Chrome instead, `nebula_loader.py` has a few references to Firefox which one would need to alter
+- [Geckodriver](https://github.com/mozilla/geckodriver/releases/) is needed to remote-control Firefox. The executable should be in your path, it's possible to specify the path to the binary in the config file. Replace with ChromeDriver for Chrome
+- [Poetry](https://python-poetry.org): Not a hard requirement, I have included a requirements.txt which can be used to install dependencies with `pip install -r requirements.txt`
+. I haven't tested this, and the list is auto-generated by poetry, using python 3.10
+
+Installation
+============
+
+- Download or clone the repository.
+- If you are using poetry, run `poetry install` in the `nebula-rss` directory.
+- If you are not using poetry, run `pip install -r requirements.txt`. If that doesn't work, try manually installing the dependencies listed in `pyproject.toml`
+- Copy `config_default.json` to `config.json`
+
+Usage
+=====
+
+This assume you are inside the folder `nebula-rss`.
+
+First, configure your username and password in `config.json`. You should also set your feed url and possibly adjust the path to the output file which will be generated.
+Run `poetry run python main.py` to generate your feed. Alternatively, run `chmod +x ./main.py` to mark it as executable and then run `poetry run ./main.py`.
+If you a not using poetry, run `python main.py` directly, or mark `main.py` as executable and run `./main.py`.
+
+You will likely want to run the script regularly, to update the feed. I'm using the following crontab `* 0 * * * cd $HOME/nebula-rss; $HOME/.local/bin/poetry run ./main.py`, so that it runs once an hour.
+
+I have not tested this on windows, but it should work without issues.
+
+Notes
+=====
+
+This is an extremely hacky solution and basically loads the Nebula homepage, logs in, goes to "My Library" and loads the list of videos. Naturally, this may break at any time, e.g. every time Nebula changes their layout slightly.
+Right after finishing this, I found a way to directly load the list of videos:
+
+    curl 'https://content.watchnebula.com/library/video/?page=1' \
+    -X 'GET' \
+    -H 'Accept: application/json, text/plain, */*' \
+    -H 'Origin: https://nebula.app' \
+    -H 'Referer: https://nebula.app/' \
+    -H 'Accept-Language: en-GB,en;q=0.9' \
+    -H 'Host: content.watchnebula.com' \
+    -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.2 Safari/605.1.15' \
+    -H 'Authorization: Bearer ADD_TOKEN_HERE' \
+    -H 'Connection: keep-alive' \
+    -H 'Nebula-Platform: web'
+
+
+When I get around to implementing this (that is: logging in, reading the authorisation token, and then parsing the result) I might switch to this instead, should be more stable.

main.py

@@ -28,4 +28,4 @@ if __name__ == '__main__':
         output_file.write(feed.generate())
     last_generation_time = config['last_generated_ts'] = current_ts
     with open(config_file_path, 'w') as config_file:
-        json.dump(config, config_file)
+        json.dump(config, config_file, indent=4)

nebula.rss

@@ -2,7 +2,7 @@
 <feed xmlns="http://www.w3.org/2005/Atom" xml:lang="en">
   <id>https://phlaym.net/feeds/nebula.rss</id>
   <title>Nebula subscriptions</title>
-  <updated>2022-01-11T15:58:08.097960+00:00</updated>
+  <updated>2022-01-11T19:55:44.781950+00:00</updated>
   <link href="https://phlaym.net/feeds/nebula.rss" rel="self"/>
   <generator uri="https://lkiesow.github.io/python-feedgen" version="0.9.0">python-feedgen</generator>
   <logo>https://nebula.app/apple-touch-icon.png</logo>
@@ -10,7 +10,7 @@
   <entry>
     <id>https://nebula.app/videos/sarah-z-what-went-wrong-with-dear-evan-hansen</id>
     <title>What Went Wrong With Dear Evan Hansen</title>
-    <updated>2022-01-11T15:58:08.098619+00:00</updated>
+    <updated>2022-01-11T19:55:44.782675+00:00</updated>
     <author>
       <name>SarahZ</name>
       <email>Unknown</email>
@@ -23,7 +23,7 @@
   <entry>
     <id>https://nebula.app/videos/half-as-interesting-how-one-terrible-trader-bankrupted-barings-bank</id>
     <title>How One Terrible Trader Bankrupted Barings Bank</title>
-    <updated>2022-01-11T15:58:08.098602+00:00</updated>
+    <updated>2022-01-11T19:55:44.782661+00:00</updated>
     <author>
       <name>Half as Interesting</name>
       <email>Unknown</email>
@@ -36,7 +36,7 @@
   <entry>
     <id>https://nebula.app/videos/legaleagle-real-lawyer-reacts-to-its-always-sunny-hero-or-hate-crime</id>
     <title>Real Lawyer Reacts to It's Always Sunny (Hero or Hate Crime)</title>
-    <updated>2022-01-11T15:58:08.098585+00:00</updated>
+    <updated>2022-01-11T19:55:44.782646+00:00</updated>
     <author>
       <name>Legal Eagle</name>
       <email>Unknown</email>
@@ -49,7 +49,7 @@
   <entry>
     <id>https://nebula.app/videos/half-as-interesting-why-planes-fly-over-the-north-pole-but-not-the-south-pole</id>
     <title>Why Planes Fly Over The North Pole But Not The South Pole</title>
-    <updated>2022-01-11T15:58:08.098568+00:00</updated>
+    <updated>2022-01-11T19:55:44.782631+00:00</updated>
     <author>
       <name>Half as Interesting</name>
       <email>Unknown</email>
@@ -62,7 +62,7 @@
   <entry>
     <id>https://nebula.app/videos/legaleagle-are-the-crumbley-parents-responsible-for-the-shooting-too</id>
     <title>Are The Crumbley Parents Responsible for the Shooting Too?</title>
-    <updated>2022-01-11T15:58:08.098500+00:00</updated>
+    <updated>2022-01-11T19:55:44.782556+00:00</updated>
     <author>
       <name>Legal Eagle</name>
       <email>Unknown</email>
@@ -75,7 +75,7 @@
   <entry>
     <id>https://nebula.app/videos/people-make-games-roblox-pressured-us-to-delete-our-video-so-we-dug-deeper</id>
     <title>Roblox Pressured Us to Delete Our Video. So We Dug Deeper.</title>
-    <updated>2022-01-11T15:58:08.098482+00:00</updated>
+    <updated>2022-01-11T19:55:44.782541+00:00</updated>
     <author>
       <name>People Make Games</name>
       <email>Unknown</email>
@@ -88,7 +88,7 @@
   <entry>
     <id>https://nebula.app/videos/wendover-productions-how-airlines-quietly-became-banks</id>
     <title>How Airlines Quietly Became Banks</title>
-    <updated>2022-01-11T15:58:08.098461+00:00</updated>
+    <updated>2022-01-11T19:55:44.782522+00:00</updated>
     <author>
       <name>Wendover</name>
       <email>Unknown</email>
@@ -101,7 +101,7 @@
   <entry>
     <id>https://nebula.app/videos/uno-dos-of-trace-do-dads-even-matter</id>
     <title>Do Dads Even Matter?</title>
-    <updated>2022-01-11T15:58:08.098444+00:00</updated>
+    <updated>2022-01-11T19:55:44.782507+00:00</updated>
     <author>
       <name>Uno Dos of Trace</name>
       <email>Unknown</email>
@@ -114,7 +114,7 @@
   <entry>
     <id>https://nebula.app/videos/half-as-interesting-why-sweden-is-moving-its-northernmost-town-2-miles-east</id>
     <title>Why Sweden Is Moving Its Northernmost Town 2 Miles East</title>
-    <updated>2022-01-11T15:58:08.098427+00:00</updated>
+    <updated>2022-01-11T19:55:44.782493+00:00</updated>
     <author>
       <name>Half as Interesting</name>
       <email>Unknown</email>
@@ -127,7 +127,7 @@
   <entry>
     <id>https://nebula.app/videos/up-and-atom-this-paradox-proves-einsteins-special-relativity</id>
     <title>This Paradox Proves Einstein's Special Relativity</title>
-    <updated>2022-01-11T15:58:08.098410+00:00</updated>
+    <updated>2022-01-11T19:55:44.782478+00:00</updated>
     <author>
       <name>Up and Atom</name>
       <email>Unknown</email>
@@ -140,7 +140,7 @@
   <entry>
     <id>https://nebula.app/videos/mia-mulder-were-elagabalus-and-queen-kristina-transgender-mia-mulder</id>
     <title>Were Elagabalus And Queen Kristina Transgender? | Mia Mulder</title>
-    <updated>2022-01-11T15:58:08.098393+00:00</updated>
+    <updated>2022-01-11T19:55:44.782463+00:00</updated>
     <author>
       <name>Mia Mulder</name>
       <email>Unknown</email>
@@ -153,7 +153,7 @@
   <entry>
     <id>https://nebula.app/videos/real-engineering-the-insane-engineering-of-the-james-webb-telescope</id>
     <title>The Insane Engineering of the James Webb Telescope</title>
-    <updated>2022-01-11T15:58:08.098376+00:00</updated>
+    <updated>2022-01-11T19:55:44.782445+00:00</updated>
     <author>
       <name>Real Engineering</name>
       <email>Unknown</email>
@@ -166,7 +166,7 @@
   <entry>
     <id>https://nebula.app/videos/real-science-the-insane-biology-of-the-orca</id>
     <title>The Insane Biology of: The Orca</title>
-    <updated>2022-01-11T15:58:08.098357+00:00</updated>
+    <updated>2022-01-11T19:55:44.782428+00:00</updated>
     <author>
       <name>Real Science</name>
       <email>Unknown</email>
@@ -179,7 +179,7 @@
   <entry>
     <id>https://nebula.app/videos/techaltar-how-911-google-changed-cinema</id>
     <title>How 9/11 &amp; Google Changed Cinema</title>
-    <updated>2022-01-11T15:58:08.098340+00:00</updated>
+    <updated>2022-01-11T19:55:44.782412+00:00</updated>
     <author>
       <name>TechAltar</name>
       <email>Unknown</email>
@@ -192,7 +192,7 @@
   <entry>
     <id>https://nebula.app/videos/legaleagle-the-most-bizarre-legal-defenses-that-worked</id>
     <title>The Most Bizarre Legal Defenses (That Worked!)</title>
-    <updated>2022-01-11T15:58:08.098323+00:00</updated>
+    <updated>2022-01-11T19:55:44.782396+00:00</updated>
     <author>
       <name>Legal Eagle</name>
       <email>Unknown</email>
@@ -205,7 +205,7 @@
   <entry>
     <id>https://nebula.app/videos/sarahz-why-the-mean-girls-musical-doesnt-work</id>
     <title>Why The Mean Girls Musical Doesn't Work</title>
-    <updated>2022-01-11T15:58:08.098303+00:00</updated>
+    <updated>2022-01-11T19:55:44.782378+00:00</updated>
     <author>
       <name>SarahZ</name>
       <email>Unknown</email>
@@ -218,7 +218,7 @@
   <entry>
     <id>https://nebula.app/videos/hai-why-almost-every-town-in-america-has-a-thai-restaurant</id>
     <title>Why Almost Every Town In America Has a Thai Restaurant</title>
-    <updated>2022-01-11T15:58:08.098286+00:00</updated>
+    <updated>2022-01-11T19:55:44.782358+00:00</updated>
     <author>
       <name>Half as Interesting</name>
       <email>Unknown</email>
@@ -231,7 +231,7 @@
   <entry>
     <id>https://nebula.app/videos/techaltar-the-phone-company-tier-list-2021</id>
     <title>The phone company tier list - 2021</title>
-    <updated>2022-01-11T15:58:08.098268+00:00</updated>
+    <updated>2022-01-11T19:55:44.782340+00:00</updated>
     <author>
       <name>TechAltar</name>
       <email>Unknown</email>
@@ -244,7 +244,7 @@
   <entry>
     <id>https://nebula.app/videos/hai-the-real-reason-nyc-is-always-covered-in-scaffolding</id>
     <title>The Real Reason NYC Is Always Covered In Scaffolding</title>
-    <updated>2022-01-11T15:58:08.098248+00:00</updated>
+    <updated>2022-01-11T19:55:44.782324+00:00</updated>
     <author>
       <name>Half as Interesting</name>
       <email>Unknown</email>
@@ -257,7 +257,7 @@
   <entry>
     <id>https://nebula.app/videos/legaleagle-why-alex-jones-lost-the-sandy-hook-suits</id>
     <title>Why Alex Jones Lost the Sandy Hook Suits</title>
-    <updated>2022-01-11T15:58:08.098231+00:00</updated>
+    <updated>2022-01-11T19:55:44.782305+00:00</updated>
     <author>
       <name>Legal Eagle</name>
       <email>Unknown</email>
@@ -270,7 +270,7 @@
   <entry>
     <id>https://nebula.app/videos/tracedominguez-are-lgbtq-better-parents</id>
     <title>Are LGBTQ+ Better Parents?</title>
-    <updated>2022-01-11T15:58:08.098214+00:00</updated>
+    <updated>2022-01-11T19:55:44.782289+00:00</updated>
     <author>
       <name>Uno Dos of Trace</name>
       <email>Unknown</email>
@@ -283,7 +283,7 @@
   <entry>
     <id>https://nebula.app/videos/wendover-the-news-you-missed-in-2021-from-every-country-in-the-world-part-1</id>
     <title>The News You Missed in 2021, From Every Country in the World (Part 1)</title>
-    <updated>2022-01-11T15:58:08.098197+00:00</updated>
+    <updated>2022-01-11T19:55:44.782272+00:00</updated>
     <author>
       <name>Wendover</name>
       <email>Unknown</email>
@@ -296,7 +296,7 @@
   <entry>
     <id>https://nebula.app/videos/wendover-the-news-you-missed-in-2021-from-every-country-in-the-world-part-2</id>
     <title>The News You Missed in 2021, From Every Country in the World (Part 2)</title>
-    <updated>2022-01-11T15:58:08.098179+00:00</updated>
+    <updated>2022-01-11T19:55:44.782257+00:00</updated>
     <author>
       <name>Wendover</name>
       <email>Unknown</email>
@@ -309,7 +309,7 @@
   <entry>
     <id>https://nebula.app/videos/tracedominguez-how-babies-rewire-your-brain</id>
     <title>How Babies Rewire Your Brain</title>
-    <updated>2022-01-11T15:58:08.098162+00:00</updated>
+    <updated>2022-01-11T19:55:44.782241+00:00</updated>
     <author>
       <name>Uno Dos of Trace</name>
       <email>Unknown</email>
@@ -322,7 +322,7 @@
   <entry>
     <id>https://nebula.app/videos/tracedominguez-3-stereotypes-that-hurt-kids-and-how-to-stop-them</id>
     <title>3 Stereotypes that Hurt Kids (and How to Stop Them)</title>
-    <updated>2022-01-11T15:58:08.098145+00:00</updated>
+    <updated>2022-01-11T19:55:44.782221+00:00</updated>
     <author>
       <name>Uno Dos of Trace</name>
       <email>Unknown</email>
@@ -335,7 +335,7 @@
   <entry>
     <id>https://nebula.app/videos/hai-who-officially-decides-what-time-it-is</id>
     <title>Who Officially Decides What Time It Is?</title>
-    <updated>2022-01-11T15:58:08.098127+00:00</updated>
+    <updated>2022-01-11T19:55:44.782204+00:00</updated>
     <author>
       <name>Half as Interesting</name>
       <email>Unknown</email>
@@ -348,7 +348,7 @@
   <entry>
     <id>https://nebula.app/videos/hai-why-dont-artificial-flavors-taste-accurate</id>
     <title>Why Don’t Artificial Flavors Taste Accurate?</title>
-    <updated>2022-01-11T15:58:08.098108+00:00</updated>
+    <updated>2022-01-11T19:55:44.782187+00:00</updated>
     <author>
       <name>Half as Interesting</name>
       <email>Unknown</email>
@@ -361,7 +361,7 @@
   <entry>
     <id>https://nebula.app/videos/legaleagle-celebrities-sued-for-posting-photos-of-themselves</id>
     <title>Celebrities Sued For Posting Photos of Themselves</title>
-    <updated>2022-01-11T15:58:08.098089+00:00</updated>
+    <updated>2022-01-11T19:55:44.782167+00:00</updated>
     <author>
       <name>Legal Eagle</name>
       <email>Unknown</email>
@@ -374,7 +374,7 @@
   <entry>
     <id>https://nebula.app/videos/hai-the-bug-that-created-free-public-wifi-networks-that-didnt-work</id>
     <title>The Bug That Created “Free Public Wifi” Networks That Didn’t Work</title>
-    <updated>2022-01-11T15:58:08.098065+00:00</updated>
+    <updated>2022-01-11T19:55:44.782142+00:00</updated>
     <author>
       <name>Half as Interesting</name>
       <email>Unknown</email>
@@ -387,7 +387,7 @@
   <entry>
     <id>https://nebula.app/videos/wendover-electric-vehicles-battery-problem</id>
     <title>Electric Vehicles' Battery Problem</title>
-    <updated>2022-01-11T15:58:08.098012+00:00</updated>
+    <updated>2022-01-11T19:55:44.782069+00:00</updated>
     <author>
       <name>Wendover</name>
       <email>Unknown</email>

poetry.lock

@@ -43,17 +43,6 @@ soupsieve = ">1.2"
 html5lib = ["html5lib"]
 lxml = ["lxml"]
 
-[[package]]
-name = "bs4"
-version = "0.0.1"
-description = "Dummy package for Beautiful Soup"
-category = "main"
-optional = false
-python-versions = "*"
-
-[package.dependencies]
-beautifulsoup4 = "*"
-
 [[package]]
 name = "certifi"
 version = "2021.10.8"
@@ -375,7 +364,7 @@ h11 = ">=0.9.0,<1"
 [metadata]
 lock-version = "1.1"
 python-versions = ">=3.8, <4"
-content-hash = "a5539b66162dc6e970156af8476cd35f92d70aa91166b89a3ff14de775dae24a"
+content-hash = "ba3ecd85d591822347c2548818869e64748cef7a83caf43f656e036e739184be"
 
 [metadata.files]
 async-generator = [
@@ -394,9 +383,6 @@ beautifulsoup4 = [
     {file = "beautifulsoup4-4.10.0-py3-none-any.whl", hash = "sha256:9a315ce70049920ea4572a4055bc4bd700c940521d36fc858205ad4fcde149bf"},
     {file = "beautifulsoup4-4.10.0.tar.gz", hash = "sha256:c23ad23c521d818955a4151a67d81580319d4bf548d3d49f4223ae041ff98891"},
 ]
-bs4 = [
-    {file = "bs4-0.0.1.tar.gz", hash = "sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a"},
-]
 certifi = [
     {file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"},
     {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"},

pyproject.toml

@@ -9,7 +9,7 @@ python = ">=3.8, <4"
 selenium = "^4.1.0"
 feedgen = "^0.9.0"
 lxml = "^4.7.1"
-bs4 = "^0.0.1"
+beautifulsoup4 = "^4.10.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "^6.2"